diff --git a/.env.template b/.env.template index 6f6e393d..ec982907 100644 --- a/.env.template +++ b/.env.template @@ -12,7 +12,7 @@ SIGNOZ_BASE_URL= SIGNOZ_SERVICE_NAME=fastgpt-plugin # S3 Config -# Customize S3 Endpoint. Such as https://example.com The URL returned will be rewrite by this Endpoint: https://example.com/{{filename}} +# Customize S3 Base URL, for example: https://s3.example.com, make sure you can access your files via https://s3.example.com/[bucket]/[objectname] S3_EXTERNAL_BASE_URL= S3_ENDPOINT=localhost S3_PORT=9000 @@ -20,7 +20,7 @@ S3_USE_SSL=false S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin S3_UPLOAD_BUCKET=files # 系统工具,创建的临时文件,存储的桶,要求公开读私有写。 -S3_PLUGIN_BUCKET=fastgpt-plugin # 系统插件热安装文件都桶,私有读写。 +S3_PLUGIN_BUCKET=fastgpt-plugin # 系统插件热安装文件的桶,私有读写。 # Signoz SIGNOZ_BASE_URL= diff --git a/modules/tool/api/upload/confirmUpload.ts b/modules/tool/api/upload/confirmUpload.ts index 90726f80..b282e6fe 100644 --- a/modules/tool/api/upload/confirmUpload.ts +++ b/modules/tool/api/upload/confirmUpload.ts @@ -13,6 +13,7 @@ export default s.route(contract.tool.upload.confirmUpload, async ({ body }) => { await mongoSessionRun(async (session) => { const toolId = await downloadTool(objectName); + if (!toolId) return Promise.reject('Can not parse ToolId from the tool, installation failed.'); const oldTool = await MongoPluginModel.findOneAndUpdate( { toolId diff --git a/modules/tool/controller.ts b/modules/tool/controller.ts index 32259a0a..5baecd2d 100644 --- a/modules/tool/controller.ts +++ b/modules/tool/controller.ts @@ -27,6 +27,16 @@ export function getToolType(): z.infer { })); } +const removeFile = async (file: string) => { + try { + if (fs.existsSync(file)) { + await fs.promises.unlink(file); + } + } catch (err) { + addLog.warn(` delele File Error, ${getErrText(err)} `); + } +}; + export async function refreshUploadedTools() { addLog.info('Refreshing uploaded tools'); const existsFiles = uploadedTools.map((item) => item.toolDirName); @@ -41,12 +51,6 @@ export async function refreshUploadedTools() { const newFiles = tools.filter((item) => !existsFiles.includes(item.objectName.split('/').pop()!)); - const removeFile = async (file: string) => { - if (fs.existsSync(file)) { - await fs.promises.unlink(file); - } - }; - // merge remove and download steps into one Promise.all await Promise.all([ ...deleteFiles.map((item) => @@ -85,9 +89,8 @@ export async function downloadTool(objectName: string) { return Promise.reject(err); }); addLog.debug(`Downloaded tool: ${toolId}`); + return toolId; } catch { - if (fs.existsSync(filepath)) { - await fs.promises.unlink(filepath); - } + await removeFile(filepath); } } diff --git a/modules/tool/init.ts b/modules/tool/init.ts index 9af0658e..e3221543 100644 --- a/modules/tool/init.ts +++ b/modules/tool/init.ts @@ -6,6 +6,7 @@ import fs from 'fs'; import { addLog } from '@/utils/log'; import { ToolTypeEnum } from './type/tool'; import { BuiltInToolBaseURL, UploadedToolBaseURL } from './utils'; +import { refreshUploadedTools } from './controller'; const filterToolList = ['.DS_Store', '.git', '.github', 'node_modules', 'dist', 'scripts']; @@ -131,4 +132,4 @@ export async function initUploadedTool() { ); } -export const initTools = () => Promise.all([initBuiltInTools(), initUploadedTool()]); +export const initTools = async () => Promise.all([initBuiltInTools(), refreshUploadedTools()]); diff --git a/src/cache/index.ts b/src/cache/index.ts index 6772afcd..53906a5b 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -1,9 +1,9 @@ import type { SystemCacheKeyEnum } from './type'; import { randomUUID } from 'node:crypto'; import { initCache } from './init'; -import { FASTGPT_REDIS_PREFIX, getGlobalRedisConnection } from '@/redis'; +import { getGlobalRedisConnection } from '@/redis'; -const cachePrefix = `${FASTGPT_REDIS_PREFIX}:VERSION_KEY:`; +const cachePrefix = `VERSION_KEY:`; const getVersionKey = async (key: `${SystemCacheKeyEnum}`) => { if (!global.systemCache) initCache(); diff --git a/src/s3/controller.ts b/src/s3/controller.ts index a3d9ea1a..838dcc65 100644 --- a/src/s3/controller.ts +++ b/src/s3/controller.ts @@ -16,6 +16,7 @@ import { export class S3Service { private client: Minio.Client; + private externalClient?: Minio.Client; private config: S3ConfigType; constructor(config: Partial) { @@ -32,6 +33,22 @@ export class S3Service { secretKey: this.config.secretKey, transportAgent: this.config.transportAgent }); + + this.externalClient = this.config.externalBaseUrl + ? (() => { + const urlObj = new URL(this.config.externalBaseUrl); + const endPoint = urlObj.hostname; + const useSSL = urlObj.protocol === 'https'; + return new Minio.Client({ + endPoint, + port: urlObj.port ? parseInt(urlObj.port) : useSSL ? 443 : 80, + useSSL, + accessKey: this.config.accessKey, + secretKey: this.config.secretKey, + transportAgent: this.config.transportAgent + }); + })() + : undefined; } async initialize() { @@ -290,8 +307,10 @@ export class S3Service { const name = this.generateFileId(); const objectName = `${filepath}/${name}`; + const client = this.externalClient ?? this.client; + try { - const policy = this.client.newPostPolicy(); + const policy = client.newPostPolicy(); policy.setBucket(this.config.bucket); policy.setKey(objectName); if (contentType) { @@ -308,7 +327,19 @@ export class S3Service { ...metadata }); - return { ...(await this.client.presignedPostPolicy(policy)), objectName }; + const res = await client.presignedPostPolicy(policy); + const postURL = (() => { + if (this.config.externalBaseUrl) { + return `${this.config.externalBaseUrl}/${this.config.bucket}`; + } else { + return res.postURL; + } + })(); + return { + postURL, + formData: res.formData, + objectName + }; } catch (error) { addLog.error('Failed to generate Upload Presigned URL', error); return Promise.reject(`Failed to generate Upload Presigned URL: ${getErrText(error)}`); diff --git a/src/s3/index.ts b/src/s3/index.ts index 5fa186d5..120a709c 100644 --- a/src/s3/index.ts +++ b/src/s3/index.ts @@ -18,7 +18,8 @@ export const pluginFileS3Server = (() => { if (!global._pluginFileS3Server) { global._pluginFileS3Server = new S3Service({ maxFileSize: 50 * 1024 * 1024, // 默认 50MB - bucket: process.env.S3_PLUGIN_BUCKET || 'plugin_files' + bucket: process.env.S3_PLUGIN_BUCKET || 'plugin_files', + externalBaseUrl: process.env.S3_EXTERNAL_BASE_URL }); } return global._pluginFileS3Server;