feat: migrate to fastgpt storage sdk

.env.marketplace.template

@@ -4,3 +4,15 @@ S3_ACCESS_KEY=minioadmin
 S3_SECRET_KEY=minioadmin
 MARKETPLACE_BASE_URL=http://localhost:3100
 MARKETPLACE_AUTH_TOKEN=token
+
+# 对象存储
+STORAGE_VENDOR=minio # 如果是 Sealos 的对象存储填 aws-s3
+STORAGE_REGION=us-east-1
+STORAGE_ACCESS_KEY_ID=minioadmin
+STORAGE_SECRET_ACCESS_KEY=minioadmin
+STORAGE_PUBLIC_BUCKET=fastgpt-public
+STORAGE_PRIVATE_BUCKET=fastgpt-private
+STORAGE_EXTERNAL_ENDPOINT=
+STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
+STORAGE_S3_FORCE_PATH_STYLE=true
+STORAGE_S3_MAX_RETRIES=3

.github/workflows/deploy-marketplace.yml

@@ -24,10 +24,23 @@ jobs:
 
       - name: Build and deploy to marketplace
         env:
-          S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
-          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
-          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
-          S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          STORAGE_VENDOR: ${{ secrets.STORAGE_VENDOR }}
+          STORAGE_REGION: ${{ secrets.STORAGE_REGION }}
+          STORAGE_ACCESS_KEY_ID: ${{ secrets.STORAGE_ACCESS_KEY_ID }}
+          STORAGE_SECRET_ACCESS_KEY: ${{ secrets.STORAGE_SECRET_ACCESS_KEY }}
+          STORAGE_PUBLIC_BUCKET: ${{ secrets.STORAGE_PUBLIC_BUCKET }}
+          STORAGE_PRIVATE_BUCKET: ${{ secrets.STORAGE_PRIVATE_BUCKET }}
+          STORAGE_S3_ENDPOINT: ${{ secrets.STORAGE_S3_ENDPOINT }}
+          STORAGE_S3_FORCE_PATH_STYLE: ${{ secrets.STORAGE_S3_FORCE_PATH_STYLE }}
+          STORAGE_S3_MAX_RETRIES: ${{ secrets.STORAGE_S3_MAX_RETRIES }}
+          STORAGE_OSS_ENDPOINT: ${{ secrets.STORAGE_OSS_ENDPOINT }}
+          STORAGE_OSS_CNAME: ${{ secrets.STORAGE_OSS_CNAME }}
+          STORAGE_OSS_SECURE: ${{ secrets.STORAGE_OSS_SECURE }}
+          STORAGE_OSS_INTERNAL: ${{ secrets.STORAGE_OSS_INTERNAL }}
+          STORAGE_COS_PROTOCOL: ${{ secrets.STORAGE_COS_PROTOCOL }}
+          STORAGE_COS_USE_ACCELERATE: ${{ secrets.STORAGE_COS_USE_ACCELERATE }}
+          STORAGE_COS_CNAME_DOMAIN: ${{ secrets.STORAGE_COS_CNAME_DOMAIN }}
+          STORAGE_COS_PROXY: ${{ secrets.STORAGE_COS_PROXY }}
         run: bun ./scripts/deploy-marketplace.ts
 
       - name: Notify deployment success

lib/package.json

@@ -4,6 +4,7 @@
   "version": "1.0.0",
   "description": "FastGPT Plugins",
   "dependencies": {
+    "@fastgpt-sdk/storage": "0.5.8",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/api-logs": "^0.203.0",
     "@opentelemetry/exporter-logs-otlp-http": "^0.203.0",

lib/s3/config.ts

@@ -1,29 +1,11 @@
 import { z } from 'zod';
-import type { ClientOptions } from 'minio';
-import { HttpProxyAgent } from 'http-proxy-agent';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-
-export type S3ConfigType = {
-  maxFileSize?: number; // 文件大小限制（字节）
-  externalBaseURL?: string; // 自定义域名
-  bucket: string; // 存储桶名称
-  isPublicRead: boolean;
-} & ClientOptions;
-
-export const commonS3Config: Partial<S3ConfigType> = {
-  endPoint: process.env.S3_ENDPOINT || 'localhost',
-  port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000,
-  useSSL: process.env.S3_USE_SSL === 'true',
-  accessKey: process.env.S3_ACCESS_KEY || 'minioadmin',
-  secretKey: process.env.S3_SECRET_KEY || 'minioadmin',
-  transportAgent: process.env.HTTP_PROXY
-    ? new HttpProxyAgent(process.env.HTTP_PROXY)
-    : process.env.HTTPS_PROXY
-      ? new HttpsProxyAgent(process.env.HTTPS_PROXY)
-      : undefined,
-  pathStyle: process.env.S3_PATH_STYLE === 'false' ? false : true,
-  region: process.env.S3_REGION || undefined
-} as const;
+import {
+  type IAwsS3CompatibleStorageOptions,
+  type ICosStorageOptions,
+  type IOssStorageOptions,
+  type IStorageOptions
+} from '@fastgpt-sdk/storage';
+import { addLog } from '@/utils/log';
 
 export const FileMetadataSchema = z.object({
   originalFilename: z.string(),
@@ -35,3 +17,106 @@ export const FileMetadataSchema = z.object({
 });
 
 export type FileMetadata = z.infer<typeof FileMetadataSchema>;
+
+export function createDefaultStorageOptions() {
+  const vendor = (process.env.STORAGE_VENDOR || 'minio') as IStorageOptions['vendor'];
+
+  addLog.debug(`Load configuration of '${vendor}' Vendor`);
+
+  switch (vendor) {
+    case 'minio': {
+      return {
+        vendor: 'minio',
+        forcePathStyle: true,
+        externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
+        endpoint: process.env.STORAGE_S3_ENDPOINT || 'http://localhost:9000',
+        region: process.env.STORAGE_REGION || 'us-east-1',
+        publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
+        privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
+        credentials: {
+          accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || 'minioadmin',
+          secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || 'minioadmin'
+        },
+        maxRetries: process.env.STORAGE_S3_MAX_RETRIES
+          ? parseInt(process.env.STORAGE_S3_MAX_RETRIES)
+          : 3
+      } satisfies Omit<IAwsS3CompatibleStorageOptions, 'bucket'> & {
+        publicBucket: string;
+        privateBucket: string;
+        externalBaseUrl?: string;
+      };
+    }
+
+    case 'aws-s3': {
+      return {
+        vendor: 'aws-s3',
+        forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : false,
+        externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
+        endpoint: process.env.STORAGE_S3_ENDPOINT || '',
+        region: process.env.STORAGE_REGION || 'us-east-1',
+        publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
+        privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
+        credentials: {
+          accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || '',
+          secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || ''
+        },
+        maxRetries: process.env.STORAGE_S3_MAX_RETRIES
+          ? parseInt(process.env.STORAGE_S3_MAX_RETRIES)
+          : 3
+      } satisfies Omit<IAwsS3CompatibleStorageOptions, 'bucket'> & {
+        publicBucket: string;
+        privateBucket: string;
+        externalBaseUrl?: string;
+      };
+    }
+
+    case 'cos': {
+      return {
+        vendor: 'cos',
+        externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
+        region: process.env.STORAGE_REGION || 'ap-shanghai',
+        publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
+        privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
+        credentials: {
+          accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || '',
+          secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || ''
+        },
+        protocol: (process.env.STORAGE_COS_PROTOCOL as 'https:' | 'http:' | undefined) || 'https:',
+        useAccelerate: process.env.STORAGE_COS_USE_ACCELERATE === 'true' ? true : false,
+        domain: process.env.STORAGE_COS_CNAME_DOMAIN || undefined,
+        proxy: process.env.STORAGE_COS_PROXY || undefined
+      } satisfies Omit<ICosStorageOptions, 'bucket'> & {
+        publicBucket: string;
+        privateBucket: string;
+        externalBaseUrl?: string;
+      };
+    }
+
+    case 'oss': {
+      return {
+        vendor: 'oss',
+        externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
+        endpoint: process.env.STORAGE_OSS_ENDPOINT || '',
+        region: process.env.STORAGE_REGION || 'oss-cn-hangzhou',
+        publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
+        privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
+        credentials: {
+          accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || '',
+          secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || ''
+        },
+        cname: process.env.STORAGE_OSS_CNAME === 'true' ? true : false,
+        internal: process.env.STORAGE_OSS_INTERNAL === 'true' ? true : false,
+        secure: process.env.STORAGE_OSS_SECURE === 'true' ? true : false,
+        enableProxy: process.env.STORAGE_OSS_ENABLE_PROXY === 'false' ? false : true
+      } satisfies Omit<IOssStorageOptions, 'bucket'> & {
+        publicBucket: string;
+        privateBucket: string;
+        externalBaseUrl?: string;
+      };
+    }
+
+    default: {
+      throw new Error(`Unsupported storage vendor: ${vendor}`);
+    }
+  }
+}

lib/s3/const.ts

@@ -19,7 +19,3 @@ export const mimeMap: Record<string, string> = {
   '.js': 'application/javascript',
   '.md': 'text/markdown'
 };
-
-export const PublicBucketBaseURL = process.env.S3_EXTERNAL_BASE_URL
-  ? `${process.env.S3_EXTERNAL_BASE_URL}/${process.env.S3_PUBLIC_BUCKET}`
-  : `${process.env.S3_USE_SSL === 'true' ? 'https' : 'http'}://${process.env.S3_ENDPOINT}:${process.env.S3_PORT}/${process.env.S3_PUBLIC_BUCKET}`;