Skip to content

Commit 248d864

Browse files
xqvvuc121914yu
xqvvu
authored and
c121914yu
committed
feat: migrate to fastgpt storage sdk (#324)
* chore: more logs * feat: migrate to fastgpt storage sdk * chore: add systemVar types * chore: rename env variable * fix: oracle db operation * fix: global proxy agent * fix: incorrect config load timing * fix: incorrect fastgpt storage sdk version * fix: update COS proxy * chore: upgrade tools' version which using 'uploadFile' function * clean code
1 parent 9534dcd commit 248d864

File tree

38 files changed

+1165
-390
lines changed

38 files changed

+1165
-390
lines changed

‎.env.marketplace.template‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,15 @@ S3_ACCESS_KEY=minioadmin
44
S3_SECRET_KEY=minioadmin
55
MARKETPLACE_BASE_URL=http://localhost:3100
66
MARKETPLACE_AUTH_TOKEN=token
7+
8+
# 对象存储
9+
STORAGE_VENDOR=minio # 如果是 Sealos 的对象存储填 aws-s3
10+
STORAGE_REGION=us-east-1
11+
STORAGE_ACCESS_KEY_ID=minioadmin
12+
STORAGE_SECRET_ACCESS_KEY=minioadmin
13+
STORAGE_PUBLIC_BUCKET=fastgpt-public
14+
STORAGE_PRIVATE_BUCKET=fastgpt-private
15+
STORAGE_EXTERNAL_ENDPOINT=
16+
STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
17+
STORAGE_S3_FORCE_PATH_STYLE=true
18+
STORAGE_S3_MAX_RETRIES=3

‎.github/workflows/deploy-marketplace.yml‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,23 @@ jobs:
2424

2525
- name: Build and deploy to marketplace
2626
env:
27-
S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
28-
S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
29-
S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
30-
S3_BUCKET: ${{ secrets.S3_BUCKET }}
27+
STORAGE_VENDOR: ${{ secrets.STORAGE_VENDOR }}
28+
STORAGE_REGION: ${{ secrets.STORAGE_REGION }}
29+
STORAGE_ACCESS_KEY_ID: ${{ secrets.STORAGE_ACCESS_KEY_ID }}
30+
STORAGE_SECRET_ACCESS_KEY: ${{ secrets.STORAGE_SECRET_ACCESS_KEY }}
31+
STORAGE_PUBLIC_BUCKET: ${{ secrets.STORAGE_PUBLIC_BUCKET }}
32+
STORAGE_PRIVATE_BUCKET: ${{ secrets.STORAGE_PRIVATE_BUCKET }}
33+
STORAGE_S3_ENDPOINT: ${{ secrets.STORAGE_S3_ENDPOINT }}
34+
STORAGE_S3_FORCE_PATH_STYLE: ${{ secrets.STORAGE_S3_FORCE_PATH_STYLE }}
35+
STORAGE_S3_MAX_RETRIES: ${{ secrets.STORAGE_S3_MAX_RETRIES }}
36+
STORAGE_OSS_ENDPOINT: ${{ secrets.STORAGE_OSS_ENDPOINT }}
37+
STORAGE_OSS_CNAME: ${{ secrets.STORAGE_OSS_CNAME }}
38+
STORAGE_OSS_SECURE: ${{ secrets.STORAGE_OSS_SECURE }}
39+
STORAGE_OSS_INTERNAL: ${{ secrets.STORAGE_OSS_INTERNAL }}
40+
STORAGE_COS_PROTOCOL: ${{ secrets.STORAGE_COS_PROTOCOL }}
41+
STORAGE_COS_USE_ACCELERATE: ${{ secrets.STORAGE_COS_USE_ACCELERATE }}
42+
STORAGE_COS_CNAME_DOMAIN: ${{ secrets.STORAGE_COS_CNAME_DOMAIN }}
43+
STORAGE_COS_PROXY: ${{ secrets.STORAGE_COS_PROXY }}
3144
run: bun ./scripts/deploy-marketplace.ts
3245

3346
- name: Notify deployment success

‎bun.lock‎

Lines changed: 543 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"version": "1.0.0",
55
"description": "FastGPT Plugins",
66
"dependencies": {
7+
"@fastgpt-sdk/storage": "0.5.8",
78
"@opentelemetry/api": "^1.9.0",
89
"@opentelemetry/api-logs": "^0.203.0",
910
"@opentelemetry/exporter-logs-otlp-http": "^0.203.0",

‎lib/s3/config.ts‎

Lines changed: 110 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,11 @@
11
import { z } from 'zod';
2-
import type { ClientOptions } from 'minio';
3-
import { HttpProxyAgent } from 'http-proxy-agent';
4-
import { HttpsProxyAgent } from 'https-proxy-agent';
5-
6-
export type S3ConfigType = {
7-
maxFileSize?: number; // 文件大小限制（字节）
8-
externalBaseURL?: string; // 自定义域名
9-
bucket: string; // 存储桶名称
10-
isPublicRead: boolean;
11-
} & ClientOptions;
12-
13-
export const commonS3Config: Partial<S3ConfigType> = {
14-
endPoint: process.env.S3_ENDPOINT || 'localhost',
15-
port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000,
16-
useSSL: process.env.S3_USE_SSL === 'true',
17-
accessKey: process.env.S3_ACCESS_KEY || 'minioadmin',
18-
secretKey: process.env.S3_SECRET_KEY || 'minioadmin',
19-
transportAgent: process.env.HTTP_PROXY
20-
? new HttpProxyAgent(process.env.HTTP_PROXY)
21-
: process.env.HTTPS_PROXY
22-
? new HttpsProxyAgent(process.env.HTTPS_PROXY)
23-
: undefined,
24-
pathStyle: process.env.S3_PATH_STYLE === 'false' ? false : true,
25-
region: process.env.S3_REGION || undefined
26-
} as const;
2+
import {
3+
type IAwsS3CompatibleStorageOptions,
4+
type ICosStorageOptions,
5+
type IOssStorageOptions,
6+
type IStorageOptions
7+
} from '@fastgpt-sdk/storage';
8+
import { addLog } from '@/utils/log';
279

2810
export const FileMetadataSchema = z.object({
2911
originalFilename: z.string(),
@@ -35,3 +17,106 @@ export const FileMetadataSchema = z.object({
3517
});
3618

3719
export type FileMetadata = z.infer<typeof FileMetadataSchema>;
20+
21+
export function createDefaultStorageOptions() {
22+
const vendor = (process.env.STORAGE_VENDOR || 'minio') as IStorageOptions['vendor'];
23+
24+
addLog.debug(`Load configuration of '${vendor}' Vendor`);
25+
26+
switch (vendor) {
27+
case 'minio': {
28+
return {
29+
vendor: 'minio',
30+
forcePathStyle: true,
31+
externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
32+
endpoint: process.env.STORAGE_S3_ENDPOINT || 'http://localhost:9000',
33+
region: process.env.STORAGE_REGION || 'us-east-1',
34+
publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
35+
privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
36+
credentials: {
37+
accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || 'minioadmin',
38+
secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || 'minioadmin'
39+
},
40+
maxRetries: process.env.STORAGE_S3_MAX_RETRIES
41+
? parseInt(process.env.STORAGE_S3_MAX_RETRIES)
42+
: 3
43+
} satisfies Omit<IAwsS3CompatibleStorageOptions, 'bucket'> & {
44+
publicBucket: string;
45+
privateBucket: string;
46+
externalBaseUrl?: string;
47+
};
48+
}
49+
50+
case 'aws-s3': {
51+
return {
52+
vendor: 'aws-s3',
53+
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : false,
54+
externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
55+
endpoint: process.env.STORAGE_S3_ENDPOINT || '',
56+
region: process.env.STORAGE_REGION || 'us-east-1',
57+
publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
58+
privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
59+
credentials: {
60+
accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || '',
61+
secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || ''
62+
},
63+
maxRetries: process.env.STORAGE_S3_MAX_RETRIES
64+
? parseInt(process.env.STORAGE_S3_MAX_RETRIES)
65+
: 3
66+
} satisfies Omit<IAwsS3CompatibleStorageOptions, 'bucket'> & {
67+
publicBucket: string;
68+
privateBucket: string;
69+
externalBaseUrl?: string;
70+
};
71+
}
72+
73+
case 'cos': {
74+
return {
75+
vendor: 'cos',
76+
externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
77+
region: process.env.STORAGE_REGION || 'ap-shanghai',
78+
publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
79+
privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
80+
credentials: {
81+
accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || '',
82+
secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || ''
83+
},
84+
protocol: (process.env.STORAGE_COS_PROTOCOL as 'https:' | 'http:' | undefined) || 'https:',
85+
useAccelerate: process.env.STORAGE_COS_USE_ACCELERATE === 'true' ? true : false,
86+
domain: process.env.STORAGE_COS_CNAME_DOMAIN || undefined,
87+
proxy: process.env.STORAGE_COS_PROXY || undefined
88+
} satisfies Omit<ICosStorageOptions, 'bucket'> & {
89+
publicBucket: string;
90+
privateBucket: string;
91+
externalBaseUrl?: string;
92+
};
93+
}
94+
95+
case 'oss': {
96+
return {
97+
vendor: 'oss',
98+
externalBaseUrl: process.env.STORAGE_EXTERNAL_ENDPOINT || undefined,
99+
endpoint: process.env.STORAGE_OSS_ENDPOINT || '',
100+
region: process.env.STORAGE_REGION || 'oss-cn-hangzhou',
101+
publicBucket: process.env.STORAGE_PUBLIC_BUCKET || 'fastgpt-public',
102+
privateBucket: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private',
103+
credentials: {
104+
accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || '',
105+
secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || ''
106+
},
107+
cname: process.env.STORAGE_OSS_CNAME === 'true' ? true : false,
108+
internal: process.env.STORAGE_OSS_INTERNAL === 'true' ? true : false,
109+
secure: process.env.STORAGE_OSS_SECURE === 'true' ? true : false,
110+
enableProxy: process.env.STORAGE_OSS_ENABLE_PROXY === 'false' ? false : true
111+
} satisfies Omit<IOssStorageOptions, 'bucket'> & {
112+
publicBucket: string;
113+
privateBucket: string;
114+
externalBaseUrl?: string;
115+
};
116+
}
117+
118+
default: {
119+
throw new Error(`Unsupported storage vendor: ${vendor}`);
120+
}
121+
}
122+
}

‎lib/s3/const.ts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,3 @@ export const mimeMap: Record<string, string> = {
1919
'.js': 'application/javascript',
2020
'.md': 'text/markdown'
2121
};
22-
23-
export const PublicBucketBaseURL = process.env.S3_EXTERNAL_BASE_URL
24-
? `${process.env.S3_EXTERNAL_BASE_URL}/${process.env.S3_PUBLIC_BUCKET}`
25-
: `${process.env.S3_USE_SSL === 'true' ? 'https' : 'http'}://${process.env.S3_ENDPOINT}:${process.env.S3_PORT}/${process.env.S3_PUBLIC_BUCKET}`;

0 commit comments

Comments
 (0)