diff --git a/.env.template b/.env.template index 5e6dd2ff..5b097ef2 100644 --- a/.env.template +++ b/.env.template @@ -7,16 +7,28 @@ LOG_LEVEL=info # Model provider sort MODEL_PROVIDER_PRIORITY= -# MinIO Config -# Customize MinIO Endpoint. Such as https://example.com The URL returned will be rewrite by this Endpoint: https://example.com/{{filename}} -MINIO_CUSTOM_ENDPOINT= -MINIO_ENDPOINT=localhost -MINIO_PORT=9000 -MINIO_USE_SSL=false -MINIO_ACCESS_KEY=minioadmin -MINIO_SECRET_KEY=minioadmin -MINIO_BUCKET=files +# Signoz +SIGNOZ_BASE_URL= +SIGNOZ_SERVICE_NAME=fastgpt-plugin + +# S3 Config +# Customize S3 Base URL, for example: https://s3.example.com, make sure you can access your files via https://s3.example.com/[bucket]/[objectname] +S3_EXTERNAL_BASE_URL= +S3_ENDPOINT=localhost +S3_PORT=9000 +S3_USE_SSL=false +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +S3_TOOL_BUCKET=fastgpt-tool # 系统工具，创建的临时文件，存储的桶，要求公开读私有写。 +S3_PLUGIN_BUCKET=fastgpt-plugin # 系统插件热安装文件的桶，私有读写。 +RETENTION_DAYS=15 # 系统工具临时文件保存天数 # Signoz SIGNOZ_BASE_URL= SIGNOZ_SERVICE_NAME=fastgpt-plugin + +# MongoDB connection string +# Replace 'myusername' and 'mypassword' with your actual MongoDB credentials and ensure the database 'fastgpt' exists. +MONGODB_URI=mongodb://myusername:mypassword@localhost:27017/fastgpt?authSource=admin&directConnection=true + +REDIS_URL=redis://default:mypassword@127.0.0.1:6379 diff --git a/.gitignore b/.gitignore index a00cd6f6..0136e00e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ node_modules public/imgs/ .local pnpm-lock.yaml +local/ \ No newline at end of file diff --git a/bun.lock b/bun.lock index a99575cd..fa3dbd57 100644 --- a/bun.lock +++ b/bun.lock @@ -25,9 +25,12 @@ "express": "^5.1.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", + "ioredis": "^5.7.0", "json5": "^2.2.3", "minio": "^8.0.5", + "mongoose": "^8.16.4", "nanoid": "^5.1.5", + "node-cron": "^4.2.1", "undici": "^7.13.0", "uuid": "^11.1.0", "zod": "^3.24.3", @@ -391,6 +394,18 @@ "typescript": "^5.0.0", }, }, + "modules/tool/packages/searchInfinity": { + "name": "@fastgpt-plugins/tool-search-infinity", + "dependencies": { + "zod": "^3.24.2", + }, + "devDependencies": { + "@types/bun": "latest", + }, + "peerDependencies": { + "typescript": "^5.0.0", + }, + }, "modules/tool/packages/searchXNG": { "name": "fastgpt-tools-searchXNG", "dependencies": { @@ -445,7 +460,7 @@ }, "sdk": { "name": "@fastgpt-sdk/plugin", - "version": "0.1.16", + "version": "0.1.18", "dependencies": { "@fortaine/fetch-event-source": "^3.0.6", "@ts-rest/core": "^3.52.1", @@ -635,6 +650,8 @@ "@fastgpt-plugins/tool-mineru": ["@fastgpt-plugins/tool-mineru@workspace:modules/tool/packages/mineru"], + "@fastgpt-plugins/tool-search-infinity": ["@fastgpt-plugins/tool-search-infinity@workspace:modules/tool/packages/searchInfinity"], + "@fastgpt-sdk/plugin": ["@fastgpt-sdk/plugin@workspace:sdk"], "@fortaine/fetch-event-source": ["@fortaine/fetch-event-source@3.0.6", "", {}, "sha512-621GAuLMvKtyZQ3IA6nlDWhV1V/7PGOTNIGLUifxt0KzM+dZIweJ6F3XvQF3QnqeNfS1N7WQ0Kil1Di/lhChEw=="], @@ -677,6 +694,8 @@ "@inquirer/type": ["@inquirer/type@3.0.8", "", { "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-lg9Whz8onIHRthWaN1Q9EGLa/0LFJjyM8mEUbL1eTi6yMGvBf8gvyDLtxSXztQsxMvhxxNpJYrwa1YHdq+w4Jw=="], + "@ioredis/commands": ["@ioredis/commands@1.4.0", "", {}, "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ=="], + "@isaacs/cliui": ["@isaacs/cliui@8.0.2", "", { "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="], "@istanbuljs/schema": ["@istanbuljs/schema@0.1.3", "", {}, "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA=="], @@ -697,6 +716,8 @@ "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="], + "@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.3.0", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-zlayKCsIjYb7/IdfqxorK5+xUMyi4vOKcFy10wKJYc63NSdKI8mNME+uJqfatkPmOSMMUiojrL58IePKBm3gvQ=="], + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], @@ -873,6 +894,10 @@ "@types/turndown": ["@types/turndown@5.0.5", "", {}, "sha512-TL2IgGgc7B5j78rIccBtlYAnkuv8nUQqhQc+DSYV5j9Be9XOcm/SKOVRuA47xAVI3680Tk9B1d8flK2GWT2+4w=="], + "@types/webidl-conversions": ["@types/webidl-conversions@7.0.3", "", {}, "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA=="], + + "@types/whatwg-url": ["@types/whatwg-url@11.0.5", "", { "dependencies": { "@types/webidl-conversions": "*" } }, "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ=="], + "@types/xml2js": ["@types/xml2js@0.4.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ=="], "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.38.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.38.0", "@typescript-eslint/type-utils": "8.38.0", "@typescript-eslint/utils": "8.38.0", "@typescript-eslint/visitor-keys": "8.38.0", "graphemer": "^1.4.0", "ignore": "^7.0.0", "natural-compare": "^1.4.0", "ts-api-utils": "^2.1.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.38.0", "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <5.9.0" } }, "sha512-CPoznzpuAnIOl4nhj4tRr4gIPj5AfKgkiJmGQDaq+fQnRJTYlcBjbX3wbciGmpoPf8DREufuPRe1tNMZnGdanA=="], @@ -983,6 +1008,8 @@ "browserslist": ["browserslist@4.25.1", "", { "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", "node-releases": "^2.0.19", "update-browserslist-db": "^1.1.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw=="], + "bson": ["bson@6.10.4", "", {}, "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng=="], + "buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], "buffer-crc32": ["buffer-crc32@1.0.0", "", {}, "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w=="], @@ -1039,6 +1066,8 @@ "cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], + "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], @@ -1395,6 +1424,8 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + "ioredis": ["ioredis@5.7.0", "", { "dependencies": { "@ioredis/commands": "^1.3.0", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-NUcA93i1lukyXU+riqEyPtSEkyFq8tX90uL659J+qpCZ3rEdViB/APC58oAhIh3+bJln2hzdlZbBZsGNrlsR8g=="], + "ipaddr.js": ["ipaddr.js@2.2.0", "", {}, "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA=="], "is-arguments": ["is-arguments@1.2.0", "", { "dependencies": { "call-bound": "^1.0.2", "has-tostringtag": "^1.0.2" } }, "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA=="], @@ -1469,6 +1500,8 @@ "jws": ["jws@3.2.2", "", { "dependencies": { "jwa": "^1.4.1", "safe-buffer": "^5.0.1" } }, "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA=="], + "kareem": ["kareem@2.6.3", "", {}, "sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q=="], + "keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="], "lazystream": ["lazystream@1.0.1", "", { "dependencies": { "readable-stream": "^2.0.5" } }, "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw=="], @@ -1505,6 +1538,8 @@ "lodash.includes": ["lodash.includes@4.3.0", "", {}, "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w=="], + "lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="], + "lodash.isboolean": ["lodash.isboolean@3.0.3", "", {}, "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg=="], "lodash.isequal": ["lodash.isequal@4.5.0", "", {}, "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ=="], @@ -1555,6 +1590,8 @@ "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + "memory-pager": ["memory-pager@1.5.0", "", {}, "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg=="], + "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], @@ -1581,6 +1618,16 @@ "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], + "mongodb": ["mongodb@6.18.0", "", { "dependencies": { "@mongodb-js/saslprep": "^1.1.9", "bson": "^6.10.4", "mongodb-connection-string-url": "^3.0.0" }, "peerDependencies": { "@aws-sdk/credential-providers": "^3.188.0", "@mongodb-js/zstd": "^1.1.0 || ^2.0.0", "gcp-metadata": "^5.2.0", "kerberos": "^2.0.1", "mongodb-client-encryption": ">=6.0.0 <7", "snappy": "^7.2.2", "socks": "^2.7.1" }, "optionalPeers": ["@aws-sdk/credential-providers", "@mongodb-js/zstd", "gcp-metadata", "kerberos", "mongodb-client-encryption", "snappy", "socks"] }, "sha512-fO5ttN9VC8P0F5fqtQmclAkgXZxbIkYRTUi1j8JO6IYwvamkhtYDilJr35jOPELR49zqCJgXZWwCtW7B+TM8vQ=="], + + "mongodb-connection-string-url": ["mongodb-connection-string-url@3.0.2", "", { "dependencies": { "@types/whatwg-url": "^11.0.2", "whatwg-url": "^14.1.0 || ^13.0.0" } }, "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA=="], + + "mongoose": ["mongoose@8.18.1", "", { "dependencies": { "bson": "^6.10.4", "kareem": "2.6.3", "mongodb": "~6.18.0", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", "sift": "17.1.3" } }, "sha512-K0RfrUXXufqNRZZjvAGdyjydB91SnbWxlwFYi5t7zN2DxVWFD3c6puia0/7xfBwZm6RCpYOVdYFlRFpoDWiC+w=="], + + "mpath": ["mpath@0.9.0", "", {}, "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew=="], + + "mquery": ["mquery@5.0.0", "", { "dependencies": { "debug": "4.x" } }, "sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "mssql": ["mssql@11.0.1", "", { "dependencies": { "@tediousjs/connection-string": "^0.5.0", "commander": "^11.0.0", "debug": "^4.3.3", "rfdc": "^1.3.0", "tarn": "^3.0.2", "tedious": "^18.2.1" }, "bin": { "mssql": "bin/mssql" } }, "sha512-KlGNsugoT90enKlR8/G36H0kTxPthDhmtNUCwEHvgRza5Cjpjoj+P2X6eMpFUDN7pFrJZsKadL4x990G8RBE1w=="], @@ -1601,6 +1648,8 @@ "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + "node-cron": ["node-cron@4.2.1", "", {}, "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg=="], + "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], @@ -1735,6 +1784,10 @@ "readdir-glob": ["readdir-glob@1.1.3", "", { "dependencies": { "minimatch": "^5.1.0" } }, "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA=="], + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], + + "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], "require-in-the-middle": ["require-in-the-middle@7.5.2", "", { "dependencies": { "debug": "^4.3.5", "module-details-from-path": "^1.0.3", "resolve": "^1.22.8" } }, "sha512-gAZ+kLqBdHarXB64XpAe2VCjB7rIRv+mU8tfRWziHRJ5umKsIHN2tLLv6EtMw7WCdP19S0ERVMldNvxYCHnhSQ=="], @@ -1799,6 +1852,8 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "sift": ["sift@17.1.3", "", {}, "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ=="], + "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], "signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], @@ -1807,6 +1862,8 @@ "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "sparse-bitfield": ["sparse-bitfield@3.0.3", "", { "dependencies": { "memory-pager": "^1.0.2" } }, "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ=="], + "split-on-first": ["split-on-first@1.1.0", "", {}, "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw=="], "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], @@ -1817,6 +1874,8 @@ "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], "std-env": ["std-env@3.9.0", "", {}, "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw=="], @@ -1879,7 +1938,7 @@ "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], - "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], + "tr46": ["tr46@5.1.1", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw=="], "traverse": ["traverse@0.3.9", "", {}, "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="], @@ -1939,13 +1998,13 @@ "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="], - "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], + "webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="], "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="], "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], - "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + "whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="], "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], @@ -2051,6 +2110,8 @@ "@fastgpt-plugins/tool-mineru/@types/bun": ["@types/bun@1.2.21", "", { "dependencies": { "bun-types": "1.2.21" } }, "sha512-NiDnvEqmbfQ6dmZ3EeUO577s4P5bf4HCTXtI6trMc6f6RzirY5IrF3aIookuSpyslFzrnvv2lmEWv5HyC1X79A=="], + "@fastgpt-plugins/tool-search-infinity/@types/bun": ["@types/bun@1.2.22", "", { "dependencies": { "bun-types": "1.2.22" } }, "sha512-5A/KrKos2ZcN0c6ljRSOa1fYIyCKhZfIVYeuyb4snnvomnpFqC0tTsEkdqNxbAgExV384OETQ//WAjl3XbYqQA=="], + "@humanfs/node/@humanwhocodes/retry": ["@humanwhocodes/retry@0.3.1", "", {}, "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA=="], "@inquirer/core/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], @@ -2213,6 +2274,8 @@ "@fastgpt-plugins/tool-mineru/@types/bun/bun-types": ["bun-types@1.2.21", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-sa2Tj77Ijc/NTLS0/Odjq/qngmEPZfbfnOERi0KRUYhT9R8M4VBioWVmMWE5GrYbKMc+5lVybXygLdibHaqVqw=="], + "@fastgpt-plugins/tool-search-infinity/@types/bun/bun-types": ["bun-types@1.2.22", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-hwaAu8tct/Zn6Zft4U9BsZcXkYomzpHJX28ofvx7k0Zz2HNz54n1n+tDgxoWFGB4PcFvJXJQloPhaV2eP3Q6EA=="], + "@inquirer/core/wrap-ansi/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], @@ -2237,6 +2300,8 @@ "crc32-stream/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + "cross-fetch/node-fetch/whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + "docx/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], "exceljs/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], @@ -2301,6 +2366,10 @@ "@inquirer/core/wrap-ansi/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "cross-fetch/node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], + + "cross-fetch/node-fetch/whatwg-url/webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], + "fastgpt-tools-metaso/vitest/@vitest/spy/tinyspy": ["tinyspy@2.2.1", "", {}, "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A=="], "fastgpt-tools-metaso/vitest/@vitest/utils/loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], diff --git a/env.d.ts b/env.d.ts index c7b52568..11b6e4f9 100644 --- a/env.d.ts +++ b/env.d.ts @@ -1,12 +1,12 @@ declare namespace NodeJS { interface ProcessEnv { - MINIO_CUSTOM_ENDPOINT: string; - MINIO_ENDPOINT: string; - MINIO_PORT: string; - MINIO_USE_SSL: string; - MINIO_ACCESS_KEY: string; - MINIO_SECRET_KEY: string; - MINIO_BUCKET: string; + S3_EXTERNAL_BASE_URL: string; + S3_ENDPOINT: string; + S3_PORT: string; + S3_USE_SSL: string; + S3_ACCESS_KEY: string; + S3_SECRET_KEY: string; + S3_BUCKET: string; MAX_FILE_SIZE: string; RETENTION_DAYS: string; } diff --git a/modules/model/init.ts b/modules/model/init.ts index 401ada75..eeb9f845 100644 --- a/modules/model/init.ts +++ b/modules/model/init.ts @@ -1,32 +1,36 @@ -import alicloud from './provider/AliCloud/index'; -import baai from './provider/BAAI/index'; -import baichuan from './provider/Baichuan/index'; -import chatglm from './provider/ChatGLM/index'; -import claude from './provider/Claude/index'; -import deepseek from './provider/DeepSeek/index'; -import doubao from './provider/Doubao/index'; -import ernie from './provider/Ernie/index'; -import fishaudio from './provider/FishAudio/index'; -import gemini from './provider/Gemini/index'; -import grok from './provider/Grok/index'; -import groq from './provider/Groq/index'; -import hunyuan from './provider/Hunyuan/index'; -import intern from './provider/InternLM/index'; -import jina from './provider/Jina/index'; -import meta from './provider/Meta/index'; -import minimax from './provider/MiniMax/index'; -import mistralai from './provider/MistralAI/index'; -import moka from './provider/Moka/index'; -import moonshot from './provider/Moonshot/index'; -import ollama from './provider/Ollama/index'; -import openai from './provider/OpenAI/index'; -import other from './provider/Other/index'; -import ppio from './provider/PPIO/index'; -import qwen from './provider/Qwen/index'; -import siliconflow from './provider/Siliconflow/index'; -import sparkdesk from './provider/SparkDesk/index'; -import stepfun from './provider/StepFun/index'; -import yi from './provider/Yi/index'; +import alicloud from './provider/AliCloud'; +import baai from './provider/BAAI'; +import baichuan from './provider/Baichuan'; +import chatglm from './provider/ChatGLM'; +import claude from './provider/Claude'; +import deepseek from './provider/DeepSeek'; +import doubao from './provider/Doubao'; +import ernie from './provider/Ernie'; +import fishaudio from './provider/FishAudio'; +import gemini from './provider/Gemini'; +import grok from './provider/Grok'; +import groq from './provider/Groq'; +import hunyuan from './provider/Hunyuan'; +import intern from './provider/InternLM'; +import jina from './provider/Jina'; +import meta from './provider/Meta'; +import minimax from './provider/MiniMax'; +import mistralai from './provider/MistralAI'; +import moka from './provider/Moka'; +import moonshot from './provider/Moonshot'; +import ollama from './provider/Ollama'; +import openai from './provider/OpenAI'; +import other from './provider/Other'; +import ppio from './provider/PPIO'; +import qwen from './provider/Qwen'; +import siliconflow from './provider/Siliconflow'; +import sparkdesk from './provider/SparkDesk'; +import stepfun from './provider/StepFun'; +import yi from './provider/Yi'; +import ai360 from './provider/ai360'; +import huggingface from './provider/HuggingFace'; +import novita from './provider/novita'; +import openrouter from './provider/OpenRouter'; import { ModelItemSchema, ModelTypeEnum, type ProviderConfigType } from './type'; import { modelsBuffer } from './constants'; @@ -34,6 +38,7 @@ import { addLog } from '@/utils/log'; // All providers array in alphabetical order const allProviders: ProviderConfigType[] = [ + ai360, alicloud, baai, baichuan, @@ -46,6 +51,7 @@ const allProviders: ProviderConfigType[] = [ gemini, grok, groq, + huggingface, hunyuan, intern, jina, @@ -54,8 +60,10 @@ const allProviders: ProviderConfigType[] = [ mistralai, moka, moonshot, + novita, ollama, openai, + openrouter, other, ppio, qwen, diff --git a/modules/tool/api/list.ts b/modules/tool/api/list.ts index 0aa678d1..adfea20d 100644 --- a/modules/tool/api/list.ts +++ b/modules/tool/api/list.ts @@ -1,11 +1,15 @@ import { s } from '@/router/init'; import { contract } from '@/contract'; -import { tools } from '@tool/constants'; import { formatToolList } from '@tool/utils/tool'; +import { builtinTools } from '@tool/constants'; +import { getCachedData } from '@/cache'; +import { SystemCacheKeyEnum } from '@/cache/type'; export const getToolsHandler = s.route(contract.tool.list, async () => { + // this list will only be called when syncKey is changed. + const uploadedTools = await getCachedData(SystemCacheKeyEnum.systemTool); return { status: 200, - body: formatToolList(tools) + body: formatToolList([...builtinTools, ...uploadedTools]) }; }); diff --git a/modules/tool/api/upload/confirmUpload.ts b/modules/tool/api/upload/confirmUpload.ts new file mode 100644 index 00000000..b282e6fe --- /dev/null +++ b/modules/tool/api/upload/confirmUpload.ts @@ -0,0 +1,41 @@ +import { s } from '@/router/init'; +import { contract } from '@/contract'; +import { mongoSessionRun } from '@/mongo/utils'; +import { downloadTool } from '@tool/controller'; +import { MongoPluginModel, pluginTypeEnum } from '@/mongo/models/plugins'; +import { refreshVersionKey } from '@/cache'; +import { SystemCacheKeyEnum } from '@/cache/type'; +import { addLog } from '@/utils/log'; +import { pluginFileS3Server } from '@/s3'; + +export default s.route(contract.tool.upload.confirmUpload, async ({ body }) => { + const { objectName } = body; + + await mongoSessionRun(async (session) => { + const toolId = await downloadTool(objectName); + if (!toolId) return Promise.reject('Can not parse ToolId from the tool, installation failed.'); + const oldTool = await MongoPluginModel.findOneAndUpdate( + { + toolId + }, + { + objectName, + type: pluginTypeEnum.Enum.tool + }, + { + session, + upsert: true + } + ); + if (oldTool?.objectName) pluginFileS3Server.removeFile(oldTool.objectName); + await refreshVersionKey(SystemCacheKeyEnum.systemTool); + addLog.info(`Upload tool success: ${toolId}`); + }); + + return { + status: 200, + body: { + message: 'ok' + } + }; +}); diff --git a/modules/tool/api/upload/delete.ts b/modules/tool/api/upload/delete.ts new file mode 100644 index 00000000..e87c3e65 --- /dev/null +++ b/modules/tool/api/upload/delete.ts @@ -0,0 +1,31 @@ +import { contract } from '@/contract'; +import { MongoPluginModel } from '@/mongo/models/plugins'; +import { mongoSessionRun } from '@/mongo/utils'; +import { s } from '@/router/init'; +import { pluginFileS3Server } from '@/s3'; +import { refreshVersionKey } from '@/cache'; +import { SystemCacheKeyEnum } from '@/cache/type'; + +export default s.route(contract.tool.upload.delete, async ({ query: { toolId: rawToolId } }) => { + const toolId = rawToolId.split('-').slice(1).join('-'); + await mongoSessionRun(async (session) => { + const result = await MongoPluginModel.findOneAndDelete({ toolId }).session(session); + if (!result) { + return { + status: 404, + body: { + error: `Tool with toolId ${toolId} not found in MongoDB` + } + }; + } + await pluginFileS3Server.removeFile(result.objectName); + await refreshVersionKey(SystemCacheKeyEnum.systemTool); + }); + + return { + status: 200, + body: { + message: 'Tool deleted successfully' + } + }; +}); diff --git a/modules/tool/api/upload/getUploadURL.ts b/modules/tool/api/upload/getUploadURL.ts new file mode 100644 index 00000000..c55598d6 --- /dev/null +++ b/modules/tool/api/upload/getUploadURL.ts @@ -0,0 +1,16 @@ +import { s } from '@/router/init'; +import { contract } from '@/contract'; +import { pluginFileS3Server } from '@/s3'; +import { UploadToolsS3Path } from '@tool/constants'; +import { mimeMap } from '@/s3/const'; + +export default s.route(contract.tool.upload.getUploadURL, async ({ query: { filename } }) => { + return { + status: 200, + body: await pluginFileS3Server.generateUploadPresignedURL({ + filepath: UploadToolsS3Path, + contentType: mimeMap['.js'], + filename + }) + }; +}); diff --git a/modules/tool/api/upload/router.ts b/modules/tool/api/upload/router.ts new file mode 100644 index 00000000..9c2e2e87 --- /dev/null +++ b/modules/tool/api/upload/router.ts @@ -0,0 +1,11 @@ +import { contract } from '@/contract'; +import { s } from '@/router/init'; +import confirmUpload from './confirmUpload'; +import getUploadURL from './getUploadURL'; +import deleteHandler from './delete'; + +export default s.router(contract.tool.upload, { + confirmUpload, + getUploadURL, + delete: deleteHandler +}); diff --git a/modules/tool/constants.ts b/modules/tool/constants.ts index 144f8736..33232c87 100644 --- a/modules/tool/constants.ts +++ b/modules/tool/constants.ts @@ -1,3 +1,6 @@ import type { ToolType } from './type'; -export const tools: ToolType[] = []; +export const uploadedTools: ToolType[] = []; +export const builtinTools: ToolType[] = []; + +export const UploadToolsS3Path = 'system/tools'; diff --git a/modules/tool/contract.ts b/modules/tool/contract.ts index b3f2b8ef..69a62fe2 100644 --- a/modules/tool/contract.ts +++ b/modules/tool/contract.ts @@ -1,7 +1,61 @@ import z from 'zod'; import { c } from '@/contract/init'; -import { ToolListItemSchema, type ToolListItemType } from './type/api'; -import { ToolTypeListSchema } from './controller'; +import { ToolListItemSchema, type ToolListItemType, ToolTypeListSchema } from './type/api'; + +export const toolUploadContract = c.router( + { + getUploadURL: { + path: '/getUploadURL', + query: z.object({ + filename: z.string() + }), + responses: { + 200: z.object({ + postURL: z.string(), + formData: z.record(z.any()), + objectName: z.string() + }) + }, + method: 'GET', + description: 'Get presigned upload URL' + }, + delete: { + path: '/delete', + method: 'DELETE', + description: 'Delete a tool', + query: z.object({ + toolId: z.string() + }), + responses: { + 200: z.object({ + message: z.string() + }), + 400: z.object({ + error: z.string() + }), + 404: z.object({ + error: z.string() + }) + } + }, + confirmUpload: { + path: '/confirmUpload', + method: 'POST', + description: 'Upload and install a tool plugin', + body: z.object({ + objectName: z.string() + }), + responses: { + 200: z.object({ + message: z.string() + }) + } + } + }, + { + pathPrefix: '/upload' + } +); export const toolContract = c.router( { @@ -31,7 +85,8 @@ export const toolContract = c.router( responses: { 200: ToolTypeListSchema } - } + }, + upload: toolUploadContract }, { pathPrefix: '/tool' diff --git a/modules/tool/controller.ts b/modules/tool/controller.ts index a1958ea1..5baecd2d 100644 --- a/modules/tool/controller.ts +++ b/modules/tool/controller.ts @@ -1,18 +1,22 @@ -import type { ToolType } from './type'; -import { tools } from './constants'; import { ToolTypeEnum } from './type/tool'; import { ToolTypeMap } from './type/tool'; import z from 'zod'; -import { I18nStringStrictSchema } from '@/type/i18n'; - -export const ToolTypeListSchema = z.array( - z.object({ - type: z.nativeEnum(ToolTypeEnum), - name: I18nStringStrictSchema - }) -); +import { ToolTypeListSchema } from './type/api'; +import { MongoPluginModel, pluginTypeEnum } from '@/mongo/models/plugins'; +import { builtinTools, uploadedTools } from './constants'; +import type { ToolSetType, ToolType } from './type'; +import { pipeline } from 'stream/promises'; +import { createWriteStream } from 'fs'; +import * as fs from 'fs'; +import { initUploadedTool } from '@tool/init'; +import path from 'path'; +import { addLog } from '@/utils/log'; +import { getErrText } from './utils/err'; +import { pluginFileS3Server } from '@/s3'; +import { UploadedToolBaseURL } from './utils'; export function getTool(toolId: string): ToolType | undefined { + const tools = [...builtinTools, ...uploadedTools]; return tools.find((tool) => tool.toolId === toolId); } @@ -22,3 +26,71 @@ export function getToolType(): z.infer { name })); } + +const removeFile = async (file: string) => { + try { + if (fs.existsSync(file)) { + await fs.promises.unlink(file); + } + } catch (err) { + addLog.warn(` delele File Error, ${getErrText(err)} `); + } +}; + +export async function refreshUploadedTools() { + addLog.info('Refreshing uploaded tools'); + const existsFiles = uploadedTools.map((item) => item.toolDirName); + + const tools = await MongoPluginModel.find({ + type: pluginTypeEnum.Enum.tool + }).lean(); + + const deleteFiles = existsFiles.filter( + (item) => !tools.find((tool) => tool.objectName.split('/').pop() === item.split('/').pop()) + ); + + const newFiles = tools.filter((item) => !existsFiles.includes(item.objectName.split('/').pop()!)); + + // merge remove and download steps into one Promise.all + await Promise.all([ + ...deleteFiles.map((item) => + removeFile(path.join(UploadedToolBaseURL, item.split('/').pop()!)) + ), + ...newFiles.map((tool) => downloadTool(tool.objectName)) + ]); + + await initUploadedTool(); + return uploadedTools; +} + +export async function downloadTool(objectName: string) { + const filename = objectName.split('/').pop() as string; + async function extractToolIdFromFile(filePath: string) { + const rootMod = (await import(filePath)).default as ToolSetType; + return rootMod.toolId; + } + + const uploadPath = path.join(process.cwd(), 'dist', 'tools', 'uploaded'); + if (!fs.existsSync(uploadPath)) { + fs.mkdirSync(uploadPath, { recursive: true }); + } + + const filepath = path.join(uploadPath, filename); + + try { + await pipeline(await pluginFileS3Server.getFile(objectName), createWriteStream(filepath)).catch( + (err) => { + addLog.warn(`Download plugin file: ${objectName} from S3 error: ${getErrText(err)}`); + return Promise.reject(err); + } + ); + const toolId = await extractToolIdFromFile(filepath).catch((err) => { + addLog.warn(`Can not parse the tool file: ${filepath}, ${getErrText(err)}`); + return Promise.reject(err); + }); + addLog.debug(`Downloaded tool: ${toolId}`); + return toolId; + } catch { + await removeFile(filepath); + } +} diff --git a/modules/tool/cronTask/cleanOrphanPlugins.ts b/modules/tool/cronTask/cleanOrphanPlugins.ts new file mode 100644 index 00000000..ecc72a4f --- /dev/null +++ b/modules/tool/cronTask/cleanOrphanPlugins.ts @@ -0,0 +1,24 @@ +import { MongoPluginModel, pluginTypeEnum } from '@/mongo/models/plugins'; +import { lockEnum, withLock } from '@/redis/lock'; +import { pluginFileS3Server } from '@/s3'; +import { addLog } from '@/utils/log'; +import { UploadToolsS3Path } from '@tool/constants'; + +// Remove invalid s3 files +export default async () => { + try { + await withLock(lockEnum.Enum.cleanOrphanPlugin, 60000, async () => { + const tools = await MongoPluginModel.find({ + type: pluginTypeEnum.Enum.tool + }).lean(); + + const objectNames = tools.reduce((acc, tool) => acc.add(tool.objectName), new Set()); + const files = await pluginFileS3Server.getFiles(UploadToolsS3Path); + + const orphans = files.filter((file) => !objectNames.has(file)); + await pluginFileS3Server.removeFiles(orphans); + }); + } catch { + addLog.info('Acquire Lock failed, other task is running'); + } +}; diff --git a/modules/tool/cronTask/index.ts b/modules/tool/cronTask/index.ts new file mode 100644 index 00000000..b7beffbe --- /dev/null +++ b/modules/tool/cronTask/index.ts @@ -0,0 +1,3 @@ +import cron from 'node-cron'; +import cleanOrphanPlugins from './cleanOrphanPlugins'; +cron.schedule('0 0 * * *', cleanOrphanPlugins); diff --git a/modules/tool/init.ts b/modules/tool/init.ts index 31e84ae0..e3221543 100644 --- a/modules/tool/init.ts +++ b/modules/tool/init.ts @@ -1,36 +1,30 @@ import path from 'path'; import { isProd } from '@/constants'; import type { ToolType, ToolConfigWithCbType, ToolSetType } from './type'; -import { tools } from './constants'; +import { builtinTools, uploadedTools } from './constants'; import fs from 'fs'; import { addLog } from '@/utils/log'; import { ToolTypeEnum } from './type/tool'; +import { BuiltInToolBaseURL, UploadedToolBaseURL } from './utils'; +import { refreshUploadedTools } from './controller'; const filterToolList = ['.DS_Store', '.git', '.github', 'node_modules', 'dist', 'scripts']; -const saveFile = async (url: string, path: string) => { - const response = await fetch(url); - if (!response.ok) { - throw new Error(`Failed to download file: ${response.statusText}`); - } - const buffer = await response.arrayBuffer(); - fs.writeFileSync(path, Buffer.from(buffer)); - return buffer; -}; - // Load tool or toolset and its children export const LoadToolsByFilename = async ( - basePath: string, - filename: string + filename: string, + toolSource: 'built-in' | 'uploaded' = 'built-in' ): Promise => { const tools: ToolType[] = []; - const toolRootPath = path.join(basePath, filename); + const basepath = toolSource === 'uploaded' ? UploadedToolBaseURL : BuiltInToolBaseURL; + const toolRootPath = path.join(basepath, filename); const rootMod = (await import(toolRootPath)).default as ToolSetType; const defaultIcon = `/imgs/tools/${filename.split('.')[0]}.svg`; + // Tool set if ('children' in rootMod || fs.existsSync(path.join(toolRootPath, 'children'))) { - const toolsetId = isProd ? rootMod.toolId! : filename; + const toolsetId = isProd || toolSource === 'uploaded' ? rootMod.toolId! : filename; const icon = rootMod.icon || defaultIcon; // is toolSet @@ -39,7 +33,8 @@ export const LoadToolsByFilename = async ( type: rootMod.type || ToolTypeEnum.other, toolId: toolsetId, icon, - toolDirName: filename, + toolDirName: `${toolSource}/${filename}`, + toolSource, cb: () => Promise.resolve({}), versionList: [] }); @@ -60,7 +55,8 @@ export const LoadToolsByFilename = async ( return children; }; - const children = isProd ? rootMod.children : await getChildren(toolRootPath); + const children = + isProd || toolSource === 'uploaded' ? rootMod.children : await getChildren(toolRootPath); for (const child of children) { const toolId = child.toolId!; @@ -73,7 +69,8 @@ export const LoadToolsByFilename = async ( courseUrl: rootMod.courseUrl, author: rootMod.author, icon, - toolDirName: filename + toolDirName: `${toolSource}/${filename}`, + toolSource }); } } else { @@ -84,23 +81,55 @@ export const LoadToolsByFilename = async ( type: tool.type || ToolTypeEnum.tools, icon: tool.icon || defaultIcon, toolId: tool.toolId || filename, - toolDirName: filename + toolDirName: `${toolSource}/${filename}`, + toolSource }); } return tools; }; -export async function initTool() { - const basePath = isProd - ? process.env.TOOLS_DIR || path.join(process.cwd(), 'dist', 'tools') - : path.join(__dirname, 'packages'); +async function initBuiltInTools() { + // Create directory if it doesn't exist + if (!fs.existsSync(BuiltInToolBaseURL)) { + addLog.info(`Creating built-in tools directory: ${BuiltInToolBaseURL}`); + fs.mkdirSync(BuiltInToolBaseURL, { recursive: true }); + } - const toolDirs = fs.readdirSync(basePath).filter((file) => !filterToolList.includes(file)); + builtinTools.length = 0; + const toolDirs = fs + .readdirSync(BuiltInToolBaseURL) + .filter((file) => !filterToolList.includes(file)); for (const tool of toolDirs) { - const tmpTools = await LoadToolsByFilename(basePath, tool); - tools.push(...tmpTools); + const tmpTools = await LoadToolsByFilename(tool, 'built-in'); + builtinTools.push(...tmpTools); } - addLog.info(`Load tools in ${isProd ? 'production' : 'development'} env, total: ${tools.length}`); + addLog.info( + `Load builtin tools in ${isProd ? 'production' : 'development'} env, total: ${toolDirs.length}` + ); } + +export async function initUploadedTool() { + // Create directory if it doesn't exist + if (!fs.existsSync(UploadedToolBaseURL)) { + addLog.info(`Creating uploaded tools directory: ${UploadedToolBaseURL}`); + fs.mkdirSync(UploadedToolBaseURL, { recursive: true }); + } + + uploadedTools.length = 0; + + const toolDirs = fs + .readdirSync(UploadedToolBaseURL) + .filter((file) => !filterToolList.includes(file)); + for (const tool of toolDirs) { + const tmpTools = await LoadToolsByFilename(tool, 'uploaded'); + uploadedTools.push(...tmpTools); + } + + addLog.info( + `Load uploaded tools in ${isProd ? 'production' : 'development'} env, total: ${toolDirs.length}` + ); +} + +export const initTools = async () => Promise.all([initBuiltInTools(), refreshUploadedTools()]); diff --git a/modules/tool/packages/fetchUrl/src/index.ts b/modules/tool/packages/fetchUrl/src/index.ts index e81df55d..4787e998 100644 --- a/modules/tool/packages/fetchUrl/src/index.ts +++ b/modules/tool/packages/fetchUrl/src/index.ts @@ -7,7 +7,14 @@ import TurndownService from 'turndown'; // @ts-ignore const turndownPluginGfm = require('joplin-turndown-plugin-gfm'); +// Update content size limits +const MAX_CONTENT_SIZE = 20 * 1024 * 1024; // 20MB limit +const MAX_TEXT_LENGTH = 100 * 1000; // 100k characters limit + export const html2md = (html: string) => { + if (html.length > MAX_TEXT_LENGTH) { + html = html.slice(0, MAX_TEXT_LENGTH); + } const turndownService = new TurndownService({ headingStyle: 'atx', bulletListMarker: '-', @@ -25,27 +32,15 @@ export const html2md = (html: string) => { const md = turndownService.turndown(html); - const formatMd = md - // Remove line breaks within link alt text: [alt text with\nline breaks](url) - .replace(/\[([^\]]*)\]\([^)]*\)/g, (match) => { - const altMatch = match.match(/\[([^\]]*)\]/); - const urlMatch = match.match(/\(([^)]*)\)/); - if (altMatch && urlMatch) { - const cleanAltText = altMatch[1].replace(/\n+/g, ' ').trim(); - return `[${cleanAltText}]${urlMatch[0]}`; - } - return match; - }) - // Remove line breaks within image alt text: ![alt text with\nline breaks](url) - .replace(/!\[([^\]]*)\]\([^)]*\)/g, (match) => { - const altMatch = match.match(/\[([^\]]*)\]/); - const urlMatch = match.match(/\(([^)]*)\)/); - if (altMatch && urlMatch) { - const cleanAltText = altMatch[1].replace(/\n+/g, ' ').trim(); - return `![${cleanAltText}]${urlMatch[0]}`; - } - return match; - }); + const formatMd = md.replace( + /(!\[([^\]]*)\]|\[([^\]]*)\])(\([^)]*\))/g, + (match, prefix, imageAlt, linkAlt, url) => { + const altText = imageAlt !== undefined ? imageAlt : linkAlt; + const cleanAltText = altText.replace(/\n+/g, ' ').trim(); + + return imageAlt !== undefined ? `![${cleanAltText}]${url}` : `[${cleanAltText}]${url}`; + } + ); return formatMd; }; @@ -209,9 +204,16 @@ export const urlsFetch = async ({ try { const fetchRes = await axios.get(url, { - timeout: 30000 + timeout: 30000, + maxContentLength: MAX_CONTENT_SIZE, // 20MB limit + maxBodyLength: MAX_CONTENT_SIZE, + responseType: 'text' }); + if (fetchRes.data && fetchRes.data.length > MAX_CONTENT_SIZE) { + return Promise.reject('Content size exceeds 20MB limit'); + } + const $ = cheerio.load(fetchRes.data); const { title, html, usedSelector } = cheerioToHtml({ fetchUrl: url, diff --git a/modules/tool/packages/metaso/children/metasoSearch/config.ts b/modules/tool/packages/metaso/children/metasoSearch/config.ts index 966cd718..70ba8d9b 100644 --- a/modules/tool/packages/metaso/children/metasoSearch/config.ts +++ b/modules/tool/packages/metaso/children/metasoSearch/config.ts @@ -1,9 +1,5 @@ import { defineTool } from '@tool/type'; -import { - FlowNodeInputTypeEnum, - WorkflowIOValueTypeEnum, - SystemInputKeyEnum -} from '@tool/type/fastgpt'; +import { FlowNodeInputTypeEnum, WorkflowIOValueTypeEnum } from '@tool/type/fastgpt'; import { ToolTypeEnum } from '@tool/type/tool'; export default defineTool({ diff --git a/modules/tool/router.ts b/modules/tool/router.ts index 769404fe..c522cf0f 100644 --- a/modules/tool/router.ts +++ b/modules/tool/router.ts @@ -3,9 +3,11 @@ import { getToolHandler } from './api/getTool'; import { getToolsHandler } from './api/list'; import { getTypeHandler } from './api/getType'; import { contract } from '@/contract'; +import uploadToolRouter from './api/upload/router'; export const toolRouter = s.router(contract.tool, { getTool: getToolHandler, list: getToolsHandler, - getType: getTypeHandler + getType: getTypeHandler, + upload: uploadToolRouter }); diff --git a/modules/tool/type/api.ts b/modules/tool/type/api.ts index d9ba2952..b10f8c0c 100644 --- a/modules/tool/type/api.ts +++ b/modules/tool/type/api.ts @@ -1,8 +1,16 @@ -import { I18nStringSchema } from '@/type/i18n'; +import { I18nStringSchema, I18nStringStrictSchema } from '@/type/i18n'; import { z } from 'zod'; import { ToolTypeEnum, VersionListItemSchema } from './tool'; import { InputConfigSchema } from './fastgpt'; +// Tool Type List Schema - 移动到这里以避免导入 controller（controller 中有 mongo 依赖） +export const ToolTypeListSchema = z.array( + z.object({ + type: z.nativeEnum(ToolTypeEnum), + name: I18nStringStrictSchema + }) +); + export const ToolListItemSchema = z.object({ id: z.string().describe('The unique id of the tool'), parentId: z.string().optional().describe('The parent id of the tool'), @@ -28,7 +36,8 @@ export const ToolListItemSchema = z.object({ secretInputConfig: z .array(InputConfigSchema) .optional() - .describe('The secret input list of the tool') + .describe('The secret input list of the tool'), + toolSource: z.enum(['built-in', 'uploaded']).optional().describe('The source of the tool') }); export type ToolListItemType = z.infer; diff --git a/modules/tool/type/tool.ts b/modules/tool/type/tool.ts index eb0ef8a2..c7ea2205 100644 --- a/modules/tool/type/tool.ts +++ b/modules/tool/type/tool.ts @@ -1,7 +1,6 @@ import { z } from 'zod'; import { I18nStringSchema } from '@/type/i18n'; import { InputConfigSchema, InputSchema, OutputSchema } from './fastgpt'; -import type { ToolTypeListSchema } from '@tool/controller'; /* Call back type */ export const SystemVarSchema = z.object({ @@ -212,6 +211,7 @@ export const ToolSchema = toolConfigWithCbSchema.merge( parentId: z.string().optional().describe('The parent id of the tool'), toolDirName: z.string(), + toolSource: z.enum(['built-in', 'uploaded']).optional().describe('The source of the tool'), // ToolSet Parent secretInputConfig: z .array(InputConfigSchema) diff --git a/modules/tool/utils.ts b/modules/tool/utils.ts new file mode 100644 index 00000000..067a273c --- /dev/null +++ b/modules/tool/utils.ts @@ -0,0 +1,7 @@ +import { isProd } from '@/constants'; +import path from 'path'; + +export const UploadedToolBaseURL = path.join(process.cwd(), 'dist', 'tools', 'uploaded'); +export const BuiltInToolBaseURL = isProd + ? path.join(process.cwd(), 'dist', 'tools', 'built-in') + : path.join(process.cwd(), 'modules', 'tool', 'packages'); diff --git a/modules/tool/utils/tool.ts b/modules/tool/utils/tool.ts index f48d0861..a0a85a65 100644 --- a/modules/tool/utils/tool.ts +++ b/modules/tool/utils/tool.ts @@ -48,11 +48,11 @@ export const exportToolSet = ({ config }: { config: ToolSetConfigType }) => { export function formatToolList(list: z.infer[]): ToolListItemType[] { return list.map((item, index) => ({ - id: item.toolId, - parentId: item.parentId, author: item.author, - courseUrl: item.courseUrl, name: item.name, + parentId: item.parentId, + courseUrl: item.courseUrl, + id: item.toolId, avatar: item.icon, versionList: item.versionList, description: item.description, @@ -64,6 +64,7 @@ export function formatToolList(list: z.infer[]): ToolListItem originCost: 0, currentCost: 0, hasTokenFee: false, - secretInputConfig: item.secretInputConfig + secretInputConfig: item.secretInputConfig, + toolSource: item.toolSource })); } diff --git a/modules/tool/utils/uploadFile.ts b/modules/tool/utils/uploadFile.ts index 0eaee748..2d509929 100644 --- a/modules/tool/utils/uploadFile.ts +++ b/modules/tool/utils/uploadFile.ts @@ -1,5 +1,5 @@ import type { FileMetadata } from '@/s3/config'; -import type { FileInput } from '@/s3/controller'; +import type { FileInput } from '@/s3/type'; import { parentPort } from 'worker_threads'; export const uploadFile = async (data: FileInput) => { diff --git a/package.json b/package.json index 0d283809..e741e970 100644 --- a/package.json +++ b/package.json @@ -38,9 +38,12 @@ "express": "^5.1.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", + "ioredis": "^5.7.0", "json5": "^2.2.3", "minio": "^8.0.5", + "mongoose": "^8.16.4", "nanoid": "^5.1.5", + "node-cron": "^4.2.1", "undici": "^7.13.0", "uuid": "^11.1.0", "zod": "^3.24.3" diff --git a/scripts/build.ts b/scripts/build.ts index 2a26092f..7a3f2504 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -23,8 +23,9 @@ const copyDir = async (sourceDir: string, targetDir: string) => { const toolsDir = path.join(__dirname, '..', 'modules', 'tool', 'packages'); const distDir = path.join(__dirname, '..', 'dist'); -const distToolDir = path.join(distDir, 'tools'); +const distToolDir = path.join(distDir, 'tools', 'built-in'); const tools = fs.readdirSync(toolsDir).filter((item) => !['.DS_Store'].includes(item)); + export const buildATool = async (tool: string, dist: string = distToolDir) => { const filepath = path.join(toolsDir, tool); Bun.build({ diff --git a/scripts/dev.ts b/scripts/dev.ts index 26b7128b..b60c5609 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -4,6 +4,7 @@ import path from 'path'; import { watch } from 'fs/promises'; import { $ } from 'bun'; import { addLog } from '@/utils/log'; +import { DevServer } from './devServer'; async function copyDevIcons() { if (isProd) return; @@ -30,17 +31,26 @@ async function copyDevIcons() { await copyDevIcons(); // watch the worker.ts change and build it -const workerPath = path.join(__dirname, '..', 'src', 'worker', 'worker.ts'); -const watcher = watch(workerPath); -(async () => { - for await (const _ of watcher) { - addLog.debug(`Worker file changed, rebuilding...`); - await $`bun run build:worker`; - } -})(); +// (async () => { +// const watcher = watch(path.join(__dirname, '..', 'src', 'worker', 'worker.ts')); +// for await (const _event of watcher) { +// addLog.debug(`Worker file changed, rebuilding...`); +// } +// })(); // build the worker -await $`bun run build:worker`; +// await $`bun run build:worker`; // run the main server -await $`bun run --hot src/index.ts`; +// (async () => { +// const watcher = watch(path.join(__dirname, '..', 'src')); +// for await (const _event of watcher) { +// addLog.debug(`Worker file changed, rebuilding...`); +// // rerun the server +// await $`bun run build:worker`; +// await $`bun run src/index.ts`; +// } +// })(); + +const server = new DevServer(); +await server.start(); diff --git a/scripts/devServer.ts b/scripts/devServer.ts new file mode 100644 index 00000000..c71bae4a --- /dev/null +++ b/scripts/devServer.ts @@ -0,0 +1,104 @@ +import path from 'path'; +import { watch } from 'fs/promises'; +import { $ } from 'bun'; +import { addLog } from '@/utils/log'; +import { spawn, type Subprocess } from 'bun'; + +// DevServer 类管理整个开发环境 +export class DevServer { + private serverProcess: Subprocess | null = null; + private isRestarting = false; + private debounceTimer: Timer | null = null; + + // 启动开发环境 + async start() { + await this.buildWorker(); + await this.startServer(); + await this.startWatching(); + } + + // 构建 worker + private async buildWorker() { + try { + addLog.info('Building worker...'); + await $`bun run build:worker`; + addLog.info('Worker built successfully'); + } catch (error) { + addLog.error('Failed to build worker:', error); + } + } + + // 启动服务器进程 + private async startServer() { + if (this.serverProcess) { + await this.stopServer(); + } + + this.serverProcess = spawn({ + cmd: ['bun', 'run', path.join(__dirname, '..', 'src', 'index.ts')], + stdout: 'inherit', + stderr: 'inherit', + stdin: 'inherit' + }); + } + + // 停止服务器进程 + private async stopServer() { + if (this.serverProcess) { + this.serverProcess.kill(); + this.serverProcess = null; + } + } + + /** + * 开始监听文件变化 + */ + private async startWatching() { + const workpaths = [ + { path: path.join(__dirname, '..', 'src'), name: 'src' }, + { path: path.join(__dirname, '..', 'modules'), name: 'modules' } + ]; + + // 为每个目录启动监听 + for (const { path: watchPath, name } of workpaths) { + this.watchDirectory(watchPath, name); + } + + addLog.info('文件监听已启动'); + } + + /** + * 监听指定目录 + */ + private watchDirectory(dirPath: string, dirName: string) { + (async () => { + try { + const watcher = watch(dirPath, { recursive: true }); + + for await (const event of watcher) { + if (event.filename) { + addLog.debug(`检测到 ${dirName} 目录文件变化: ${event.filename}`); + this.restart(); + } + } + } catch (error) { + addLog.error(`监听 ${dirName} 目录时出错:`, error); + // 可以在这里添加重试逻辑 + } + })(); + } + private async restart() { + addLog.debug(`Worker file changed, rebuilding...`); + // 如果正在重启，则忽略此次重启 + if (this.isRestarting) return; + // 如果正在重启，则忽略此次重启 + if (this.debounceTimer) clearTimeout(this.debounceTimer); + this.debounceTimer = setTimeout(() => { + this.debounceTimer = null; + this.isRestarting = true; + this.stopServer(); + this.startServer(); + this.isRestarting = false; + }, 1000); + } +} diff --git a/sdk/client.ts b/sdk/client.ts index 3d02db33..86b12c52 100644 --- a/sdk/client.ts +++ b/sdk/client.ts @@ -6,6 +6,7 @@ import type { StreamMessageType } from '@tool/type/tool'; export type { SystemVarType, StreamMessageType as StreamMessage }; export { RunToolWithStream } from './runToolStream'; export { StreamDataAnswerTypeEnum } from '@tool/type/tool'; +export { UploadToolsS3Path } from '@tool/constants'; export { ToolTypeEnum }; export { ModelProviders } from '@model/constants'; diff --git a/sdk/package.json b/sdk/package.json index e9823c96..81824b2a 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@fastgpt-sdk/plugin", - "version": "0.1.16", + "version": "0.1.19", "description": "fastgpt-plugin sdk", "main": "dist/client.js", "types": "dist/client.d.ts", diff --git a/src/cache/index.ts b/src/cache/index.ts new file mode 100644 index 00000000..53906a5b --- /dev/null +++ b/src/cache/index.ts @@ -0,0 +1,40 @@ +import type { SystemCacheKeyEnum } from './type'; +import { randomUUID } from 'node:crypto'; +import { initCache } from './init'; +import { getGlobalRedisConnection } from '@/redis'; + +const cachePrefix = `VERSION_KEY:`; + +const getVersionKey = async (key: `${SystemCacheKeyEnum}`) => { + if (!global.systemCache) initCache(); + const redis = getGlobalRedisConnection(); + const syncKey = `${cachePrefix}${key}`; + const val = await redis.get(syncKey); + if (val) return val; + const newVal = randomUUID(); + await redis.set(syncKey, newVal); + return newVal; +}; + +export const refreshVersionKey = async (key: `${SystemCacheKeyEnum}`) => { + if (!global.systemCache) initCache(); + const val = randomUUID(); + const redis = getGlobalRedisConnection(); + await redis.set(`${cachePrefix}${key}`, val); +}; + +export const getCachedData = async (key: `${SystemCacheKeyEnum}`) => { + if (!global.systemCache) initCache(); + + const versionKey = await getVersionKey(key); + const isDisableCache = process.env.DISABLE_CACHE === 'true'; + + if (global.systemCache[key].versionKey === versionKey && !isDisableCache) { + return global.systemCache[key].data; + } + + global.systemCache[key].versionKey = versionKey; + global.systemCache[key].data = await global.systemCache[key].refreshFunc(); + + return global.systemCache[key].data; +}; diff --git a/src/cache/init.ts b/src/cache/init.ts new file mode 100644 index 00000000..ce690549 --- /dev/null +++ b/src/cache/init.ts @@ -0,0 +1,12 @@ +import { refreshUploadedTools } from '@tool/controller'; +import { SystemCacheKeyEnum } from './type'; + +export const initCache = () => { + global.systemCache = { + [SystemCacheKeyEnum.systemTool]: { + versionKey: '', + data: [], + refreshFunc: refreshUploadedTools + } + }; +}; diff --git a/src/cache/type.ts b/src/cache/type.ts new file mode 100644 index 00000000..3b1b86b0 --- /dev/null +++ b/src/cache/type.ts @@ -0,0 +1,21 @@ +import type { ToolType } from '@tool/type'; + +export enum SystemCacheKeyEnum { + systemTool = 'systemTool' +} + +export type SystemCacheDataType = { + [SystemCacheKeyEnum.systemTool]: ToolType[]; +}; + +type SystemCacheType = { + [K in SystemCacheKeyEnum]: { + versionKey: string; + data: SystemCacheDataType[K]; + refreshFunc: () => Promise; + }; +}; + +declare global { + var systemCache: SystemCacheType; +} diff --git a/src/index.ts b/src/index.ts index 22c8d471..06e6602b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,14 +1,14 @@ import express from 'express'; import { initOpenAPI } from './contract/openapi'; import { initRouter } from './router'; -import { initTool } from '@tool/init'; +import { initTools } from '@tool/init'; import { addLog } from './utils/log'; import { isProd } from './constants'; -import { initS3Server } from './s3/config'; import { connectSignoz } from './utils/signoz'; import { initModels } from '@model/init'; import { setupProxy } from './utils/setupProxy'; import { initWorkflowTemplates } from '@workflow/init'; +import { connectMongo, connectionMongo, MONGO_URL } from '@/mongo'; const app = express().use( express.json(), @@ -25,14 +25,14 @@ setupProxy(); // DB try { - await initS3Server(); + await connectMongo(connectionMongo, MONGO_URL); } catch (error) { - addLog.error('Failed to initialize S3 server:', error); + addLog.error('Failed to initialize services:', error); process.exit(1); } // Modules -await Promise.all([initTool(), initModels(), initWorkflowTemplates()]); +await Promise.all([initTools(), initModels(), initWorkflowTemplates()]); const PORT = parseInt(process.env.PORT || '3000'); const server = app.listen(PORT, (error?: Error) => { diff --git a/src/mongo/index.ts b/src/mongo/index.ts new file mode 100644 index 00000000..c5b111f2 --- /dev/null +++ b/src/mongo/index.ts @@ -0,0 +1,135 @@ +import { isProd } from '../constants'; +import { addLog } from '../utils/log'; +import mongoose, { type Mongoose, type Model } from 'mongoose'; + +export const MONGO_URL = process.env.MONGODB_URI ?? ''; + +declare global { + var mongodb: Mongoose | undefined; +} + +export const connectionMongo = (() => { + if (!global.mongodb) { + global.mongodb = new mongoose.Mongoose(); + } + return global.mongodb; +})(); + +const addCommonMiddleware = (schema: mongoose.Schema) => { + const operations = [/^find/, 'save', 'create', /^update/, /^delete/]; + + operations.forEach((op: any) => { + schema.pre(op, function (this: any, next: () => void) { + this._startTime = Date.now(); + next(); + }); + + schema.post(op, function (this: any, result: any, next: () => void) { + if (this._startTime) { + const duration = Date.now() - this._startTime; + if (duration > 1000) { + addLog.warn(`Slow operation ${duration}ms on ${this.collection?.name}`); + } + } + next(); + }); + }); + + return schema; +}; + +export const getMongoModel = (name: string, schema: mongoose.Schema) => { + if (connectionMongo.models[name]) return connectionMongo.models[name] as Model; + if (!isProd) addLog.info(`Load model: ${name}`); + addCommonMiddleware(schema); + + const model = connectionMongo.model(name, schema); + + syncMongoIndex(model); + + return model; +}; + +const syncMongoIndex = async (model: Model) => { + if (process.env.SYNC_INDEX !== '0' && process.env.NODE_ENV !== 'test') { + try { + model.syncIndexes({ background: true }); + } catch (error: any) { + addLog.error('Create index error', error); + } + } +}; + +export const ReadPreference = connectionMongo.mongo.ReadPreference; + +export async function connectMongo(db: Mongoose, url: string): Promise { + if (db.connection.readyState !== 0) { + return db; + } + + if (!url || typeof url !== 'string') { + throw new Error(`Invalid MongoDB connection URL: ${url}`); + } + + addLog.info(`connecting to ${isProd ? 'MongoDB' : url}`); + + try { + db.connection.removeAllListeners('error'); + db.connection.removeAllListeners('disconnected'); + db.set('strictQuery', 'throw'); + + db.connection.on('error', async (error: any) => { + addLog.error('mongo error', error); + try { + if (db.connection.readyState !== 0) { + await db.disconnect(); + await delay(1000); + await connectMongo(db, url); + } + } catch (_error) { + addLog.error('Error during reconnection:', _error); + } + }); + + db.connection.on('disconnected', async () => { + addLog.warn('mongo disconnected'); + try { + if (db.connection.readyState !== 0) { + await db.disconnect(); + await delay(1000); + await connectMongo(db, url); + } + } catch (_error) { + addLog.error('Error during reconnection:', _error); + } + }); + + const options = { + bufferCommands: true, + maxPoolSize: Math.max(30, Number(process.env.MONGO_MAX_LINK || 20)), + minPoolSize: 20, + connectTimeoutMS: 60000, + waitQueueTimeoutMS: 60000, + socketTimeoutMS: 60000, + maxIdleTimeMS: 300000, + retryWrites: true, + retryReads: true, + serverSelectionTimeoutMS: 60000, + heartbeatFrequencyMS: 20000, + maxStalenessSeconds: 120 + }; + + await db.connect(url, options); + addLog.info('mongo connected'); + return db; + } catch (error) { + addLog.error('Mongo connect error', error); + await db.disconnect(); + await delay(1000); + return connectMongo(db, url); + } +} + +export async function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/mongo/models/plugins.ts b/src/mongo/models/plugins.ts new file mode 100644 index 00000000..3a613c4f --- /dev/null +++ b/src/mongo/models/plugins.ts @@ -0,0 +1,32 @@ +import { getMongoModel } from '..'; +import { z } from 'zod'; +import { Schema } from 'mongoose'; + +export const pluginTypeEnum = z.enum(['tool']); + +export const PluginZodSchema = z + .object({ + objectName: z.string() + }) + .merge( + z.object({ + type: z.literal('tool'), + toolId: z.string() + }) + ); + +export type MongoPluginSchemaType = z.infer; + +const pluginMongooseSchema = new Schema({ + toolId: { type: String }, + objectName: { type: String, required: true, unique: true }, + type: { type: String, required: true, enum: Object.values(pluginTypeEnum.Enum) } +}); + +pluginMongooseSchema.index({ toolId: 1 }, { unique: true, sparse: true }); +pluginMongooseSchema.index({ type: 1 }); + +export const MongoPluginModel = getMongoModel( + 'fastgpt_plugins', + pluginMongooseSchema +); diff --git a/src/mongo/utils.ts b/src/mongo/utils.ts new file mode 100644 index 00000000..0cf30c0c --- /dev/null +++ b/src/mongo/utils.ts @@ -0,0 +1,29 @@ +import type { ClientSession } from 'mongoose'; +import { connectionMongo } from './index'; +import { addLog } from '@/utils/log'; + +const timeout = 60000; + +export const mongoSessionRun = async (fn: (session: ClientSession) => Promise) => { + const session = await connectionMongo.startSession(); + + try { + session.startTransaction({ + maxCommitTimeMS: timeout + }); + const result = await fn(session); + + await session.commitTransaction(); + + return result as T; + } catch (error) { + if (!session.inTransaction()) { + await session.abortTransaction(); + } else { + addLog.warn('Uncaught mongo session error', { error }); + } + return Promise.reject(error); + } finally { + await session.endSession(); + } +}; diff --git a/src/redis/index.ts b/src/redis/index.ts new file mode 100644 index 00000000..96bae63e --- /dev/null +++ b/src/redis/index.ts @@ -0,0 +1,56 @@ +import { addLog } from '@/utils/log'; +import Redis from 'ioredis'; + +const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379'; + +export const newQueueRedisConnection = () => { + const redis = new Redis(REDIS_URL); + redis.on('connect', () => { + console.log('Redis connected'); + }); + redis.on('error', (error) => { + console.error('Redis connection error', error); + }); + return redis; +}; + +export const newWorkerRedisConnection = () => { + const redis = new Redis(REDIS_URL, { + maxRetriesPerRequest: null + }); + redis.on('connect', () => { + console.log('Redis connected'); + }); + redis.on('error', (error) => { + console.error('Redis connection error', error); + }); + return redis; +}; + +export const FASTGPT_REDIS_PREFIX = 'fastgpt:'; +export const getGlobalRedisConnection = () => { + if (global.redisClient) return global.redisClient; + + global.redisClient = new Redis(REDIS_URL, { keyPrefix: FASTGPT_REDIS_PREFIX }); + + global.redisClient.on('connect', () => { + addLog.info('Redis connected'); + }); + global.redisClient.on('error', (error) => { + addLog.error('Redis connection error', error); + }); + + return global.redisClient; +}; + +export const getAllKeysByPrefix = async (key: string) => { + const redis = getGlobalRedisConnection(); + const keys = (await redis.keys(`${FASTGPT_REDIS_PREFIX}${key}:*`)).map((key) => + key.replace(FASTGPT_REDIS_PREFIX, '') + ); + return keys; +}; + +declare global { + var redisClient: Redis; +} diff --git a/src/redis/lock.ts b/src/redis/lock.ts new file mode 100644 index 00000000..3f3f9eee --- /dev/null +++ b/src/redis/lock.ts @@ -0,0 +1,55 @@ +import z from 'zod'; +import { FASTGPT_REDIS_PREFIX, getGlobalRedisConnection } from '.'; + +export const lockEnum = z.enum(['cleanOrphanPlugin']); + +const lockPrefix = `${FASTGPT_REDIS_PREFIX}LOCK:`; + +export const acquireLock = async (key: z.infer, timeoutMs: number) => { + const redis = getGlobalRedisConnection(); + const lockKey = `${lockPrefix}${key}`; + + // Try to set the lock with NX (only set if not exists) and PX (expiry) + const result = await redis.set(lockKey, 'NX', 'PX', timeoutMs); + + // If result is 'OK', we acquired the lock + if (result === 'OK') { + return true; + } + + // Otherwise, lock not acquired + return false; +}; + +export const releaseLock = async (key: z.infer) => { + const redis = getGlobalRedisConnection(); + const lockKey = `${lockPrefix}${key}`; + + // Delete the lock key + const result = await redis.del(lockKey); + + // Return true if key was deleted (existed), false otherwise + return result === 1; +}; + +export const withLock = async ( + key: z.infer, + timeoutMs: number, + callback: () => Promise +): Promise => { + // Try to acquire the lock + const lockAcquired = await acquireLock(key, timeoutMs); + + if (!lockAcquired) { + throw new Error(`Failed to acquire lock for key: ${key}`); + } + + try { + // Execute the callback function + const result = await callback(); + return result; + } finally { + // Always release the lock in the finally block + await releaseLock(key); + } +}; diff --git a/src/s3/config.ts b/src/s3/config.ts index 7e02b82f..73bf6a30 100644 --- a/src/s3/config.ts +++ b/src/s3/config.ts @@ -1,28 +1,27 @@ import { z } from 'zod'; -import { S3Service } from './controller'; +import type { ClientOptions } from 'minio'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; -export type FileConfig = { - maxFileSize: number; // 文件大小限制（字节） - retentionDays: number; // 保留天数（由 MinIO 生命周期策略自动管理） - endpoint: string; // MinIO endpoint - port?: number; // MinIO port - useSSL: boolean; // 是否使用SSL - accessKey: string; // MinIO access key - secretKey: string; // MinIO secret key +export type S3ConfigType = { + maxFileSize?: number; // 文件大小限制（字节） + retentionDays?: number; // 保留天数（由 S3 生命周期策略自动管理） + externalBaseUrl?: string; // 自定义域名 bucket: string; // 存储桶名称 -}; +} & ClientOptions; -// 默认配置（动态从环境变量读取） -export const defaultFileConfig: FileConfig = { - maxFileSize: process.env.MAX_FILE_SIZE ? parseInt(process.env.MAX_FILE_SIZE) : 20 * 1024 * 1024, // 默认 20MB - retentionDays: process.env.RETENTION_DAYS ? parseInt(process.env.RETENTION_DAYS) : 15, // 默认保留15天 - endpoint: process.env.MINIO_ENDPOINT || 'localhost', - port: process.env.MINIO_PORT ? parseInt(process.env.MINIO_PORT) : 9000, - useSSL: process.env.MINIO_USE_SSL === 'true', - accessKey: process.env.MINIO_ACCESS_KEY || 'minioadmin', - secretKey: process.env.MINIO_SECRET_KEY || 'minioadmin', - bucket: process.env.MINIO_BUCKET || 'files' -}; +export const commonS3Config: Partial = { + endPoint: process.env.S3_ENDPOINT || 'localhost', + port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, + useSSL: process.env.S3_USE_SSL === 'true', + accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', + secretKey: process.env.S3_SECRET_KEY || 'minioadmin', + transportAgent: process.env.HTTP_PROXY + ? new HttpProxyAgent(process.env.HTTP_PROXY) + : process.env.HTTPS_PROXY + ? new HttpsProxyAgent(process.env.HTTPS_PROXY) + : undefined +} as const; export const FileMetadataSchema = z.object({ fileId: z.string(), @@ -34,12 +33,3 @@ export const FileMetadataSchema = z.object({ }); export type FileMetadata = z.infer; - -export const initS3Server = () => { - global.s3Server = new S3Service(defaultFileConfig); - return global.s3Server.initialize(); -}; - -declare global { - var s3Server: S3Service; -} diff --git a/src/s3/const.ts b/src/s3/const.ts new file mode 100644 index 00000000..c91043dc --- /dev/null +++ b/src/s3/const.ts @@ -0,0 +1,20 @@ +export const mimeMap: Record = { + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + '.pdf': 'application/pdf', + '.txt': 'text/plain', + '.json': 'application/json', + '.csv': 'text/csv', + '.zip': 'application/zip', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.doc': 'application/msword', + '.xls': 'application/vnd.ms-excel', + '.ppt': 'application/vnd.ms-powerpoint', + '.js': 'application/javascript' +}; diff --git a/src/s3/controller.ts b/src/s3/controller.ts index b062304a..838dcc65 100644 --- a/src/s3/controller.ts +++ b/src/s3/controller.ts @@ -1,116 +1,106 @@ import * as Minio from 'minio'; import { randomBytes } from 'crypto'; -import { defaultFileConfig, type FileConfig, type FileMetadata } from './config'; +import { type S3ConfigType, type FileMetadata, commonS3Config } from './config'; import * as fs from 'fs'; import * as path from 'path'; -import { z } from 'zod'; import { addLog } from '@/utils/log'; import { getErrText } from '@tool/utils/err'; import { catchError } from '@/utils/catch'; -import { HttpProxyAgent } from 'http-proxy-agent'; -import { HttpsProxyAgent } from 'https-proxy-agent'; - -export const FileInputSchema = z - .object({ - url: z.string().url('Invalid URL format').optional(), - path: z.string().min(1, 'File path cannot be empty').optional(), - base64: z.string().min(1, 'Base64 data cannot be empty').optional(), - buffer: z - .union([ - z.instanceof(Buffer, { message: 'Buffer is required' }), - z.instanceof(Uint8Array, { message: 'Uint8Array is required' }) - ]) - .transform((data) => { - if (data instanceof Uint8Array && !(data instanceof Buffer)) { - return Buffer.from(data); - } - return data; - }) - .optional(), - defaultFilename: z.string().optional() - }) - .refine( - (data) => { - const inputMethods = [data.url, data.path, data.base64, data.buffer].filter(Boolean); - return inputMethods.length === 1 && (!(data.base64 || data.buffer) || data.defaultFilename); - }, - { - message: 'Provide exactly one input method. Filename required for base64/buffer inputs.' - } - ); -export type FileInput = z.infer; - -type GetUploadBufferResponse = { buffer: Buffer; filename: string }; +import { mimeMap } from './const'; +import { + FileInputSchema, + type FileInput, + type GetUploadBufferResponse, + type PresignedUrlInputType +} from './type'; export class S3Service { - private minioClient: Minio.Client; - private config: FileConfig; + private client: Minio.Client; + private externalClient?: Minio.Client; + private config: S3ConfigType; - constructor(config?: Partial) { - this.config = { ...defaultFileConfig, ...config }; + constructor(config: Partial) { + this.config = { + ...commonS3Config, + ...config + } as S3ConfigType; - this.minioClient = new Minio.Client({ - endPoint: this.config.endpoint, + this.client = new Minio.Client({ + endPoint: this.config.endPoint, port: this.config.port, useSSL: this.config.useSSL, accessKey: this.config.accessKey, secretKey: this.config.secretKey, - transportAgent: process.env.HTTP_PROXY - ? new HttpProxyAgent(process.env.HTTP_PROXY) - : process.env.HTTPS_PROXY - ? new HttpsProxyAgent(process.env.HTTPS_PROXY) - : undefined + transportAgent: this.config.transportAgent }); + + this.externalClient = this.config.externalBaseUrl + ? (() => { + const urlObj = new URL(this.config.externalBaseUrl); + const endPoint = urlObj.hostname; + const useSSL = urlObj.protocol === 'https'; + return new Minio.Client({ + endPoint, + port: urlObj.port ? parseInt(urlObj.port) : useSSL ? 443 : 80, + useSSL, + accessKey: this.config.accessKey, + secretKey: this.config.secretKey, + transportAgent: this.config.transportAgent + }); + })() + : undefined; } async initialize() { const [, err] = await catchError(async () => { addLog.info(`Checking bucket: ${this.config.bucket}`); - const bucketExists = await this.minioClient.bucketExists(this.config.bucket); + const bucketExists = await this.client.bucketExists(this.config.bucket); if (!bucketExists) { addLog.info(`Creating bucket: ${this.config.bucket}`); - const [, err] = await catchError(() => this.minioClient.makeBucket(this.config.bucket)); + const [, err] = await catchError(() => this.client.makeBucket(this.config.bucket)); if (err) { - addLog.error(`Failed to create bucket: ${this.config.bucket}`); + addLog.warn(`Failed to create bucket: ${this.config.bucket}`); return Promise.reject(err); } } - const [, err] = await catchError(() => - Promise.all([ - this.minioClient.setBucketPolicy( - this.config.bucket, - JSON.stringify({ - Version: '2012-10-17', - Statement: [ + if (this.config.retentionDays && this.config.retentionDays > 0) { + const Days = this.config.retentionDays; + const [, err] = await catchError(() => + Promise.all([ + this.client.setBucketPolicy( + this.config.bucket, + JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [`arn:aws:s3:::${this.config.bucket}/*`] + } + ] + }) + ), + this.client.setBucketLifecycle(this.config.bucket, { + Rule: [ { - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [`arn:aws:s3:::${this.config.bucket}/*`] + ID: 'AutoDeleteRule', + Status: 'Enabled', + Expiration: { + Days, + DeleteMarker: false, + DeleteAll: false + } } ] }) - ), - this.minioClient.setBucketLifecycle(this.config.bucket, { - Rule: [ - { - ID: 'AutoDeleteRule', - Status: 'Enabled', - Expiration: { - Days: this.config.retentionDays, - DeleteMarker: false, - DeleteAll: false - } - } - ] - }) - ]) - ); - - if (err) { - addLog.warn(`Failed to set bucket policy: ${this.config.bucket}`); + ]) + ); + if (err) { + addLog.warn(`Failed to set bucket policy: ${this.config.bucket}`); + } } addLog.info(`Bucket initialized, ${this.config.bucket} configured successfully.`); @@ -133,17 +123,60 @@ export class S3Service { return randomBytes(16).toString('hex'); } - private generateAccessUrl(filename: string): string { + private isPublicReadBucket(policy: string): boolean { + const policyJson = JSON.parse(policy); + return policyJson.Statement.some( + (statement: any) => statement.Effect === 'Allow' && statement.Principal === '*' + ); + } + + /** + * Get the file directly. + */ + getFile(objectName: string) { + return this.client.getObject(this.config.bucket, objectName); + } + + /** + * Get public readable URL + */ + async generateExternalUrl(objectName: string, expiry: number = 3600): Promise { + const externalBaseUrl = this.config.externalBaseUrl; + + // 获取桶策略 + const policy = await this.client.getBucketPolicy(this.config.bucket); + const isPublicBucket = this.isPublicReadBucket(policy); + + if (!isPublicBucket) { + const url = await this.client.presignedGetObject(this.config.bucket, objectName, expiry); + // 如果有 externalBaseUrl，需要把域名进行替换 + if (this.config.externalBaseUrl) { + const urlObj = new URL(url); + const externalUrlObj = new URL(this.config.externalBaseUrl); + + // 替换协议和域名，保留路径和查询参数 + urlObj.protocol = externalUrlObj.protocol; + urlObj.hostname = externalUrlObj.hostname; + urlObj.port = externalUrlObj.port; + + return urlObj.toString(); + } + + return url; + } + + if (externalBaseUrl) { + return `${externalBaseUrl}/${this.config.bucket}/${objectName}`; + } + + // Default url const protocol = this.config.useSSL ? 'https' : 'http'; const port = this.config.port && this.config.port !== (this.config.useSSL ? 443 : 80) ? `:${this.config.port}` : ''; - const customEndpoint = process.env.MINIO_CUSTOM_ENDPOINT; - return customEndpoint - ? `${customEndpoint}/${encodeURIComponent(filename)}` - : `${protocol}://${this.config.endpoint}${port}/${this.config.bucket}/${encodeURIComponent(filename)}`; + return `${protocol}://${this.config.endPoint}${port}/${this.config.bucket}/${objectName}`; } async uploadFileAdvanced(input: FileInput): Promise { @@ -199,30 +232,11 @@ export class S3Service { ): Promise => { const inferContentType = (filename: string) => { const ext = path.extname(filename).toLowerCase(); - const mimeMap: Record = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.txt': 'text/plain', - '.json': 'application/json', - '.csv': 'text/csv', - '.zip': 'application/zip', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - '.doc': 'application/msword', - '.xls': 'application/vnd.ms-excel', - '.ppt': 'application/vnd.ms-powerpoint' - }; return mimeMap[ext] || 'application/octet-stream'; }; - if (fileBuffer.length > this.config.maxFileSize) { + if (this.config.maxFileSize && fileBuffer.length > this.config.maxFileSize) { return Promise.reject( `File size ${fileBuffer.length} exceeds limit ${this.config.maxFileSize}` ); @@ -233,18 +247,12 @@ export class S3Service { const uploadTime = new Date(); const contentType = inferContentType(originalFilename); - await this.minioClient.putObject( - this.config.bucket, - objectName, - fileBuffer, - fileBuffer.length, - { - 'Content-Type': contentType, - 'Content-Disposition': `attachment; filename="${encodeURIComponent(originalFilename)}"`, - 'x-amz-meta-original-filename': encodeURIComponent(originalFilename), - 'x-amz-meta-upload-time': uploadTime.toISOString() - } - ); + await this.client.putObject(this.config.bucket, objectName, fileBuffer, fileBuffer.length, { + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${encodeURIComponent(originalFilename)}"`, + 'x-amz-meta-original-filename': encodeURIComponent(originalFilename), + 'x-amz-meta-upload-time': uploadTime.toISOString() + }); const metadata: FileMetadata = { fileId, @@ -252,7 +260,7 @@ export class S3Service { contentType, size: fileBuffer.length, uploadTime, - accessUrl: this.generateAccessUrl(objectName) + accessUrl: await this.generateExternalUrl(objectName) }; return metadata; @@ -270,4 +278,88 @@ export class S3Service { return await uploadFile(buffer, filename); } + + async removeFile(objectName: string) { + await this.client.removeObject(this.config.bucket, objectName); + addLog.info(`MinIO file deleted: ${this.config.bucket}/${objectName}`); + } + + /** + * Get the file's digest, which is called ETag in Minio and in fact it is MD5 + */ + async getDigest(objectName: string): Promise { + // Get the ETag of the object as its digest + const stat = await this.client.statObject(this.config.bucket, objectName); + // Remove quotes around ETag if present + const etag = stat.etag.replace(/^"|"$/g, ''); + return etag; + } + + /** + * Generate a presigned URL for uploading a file to S3 service + */ + generateUploadPresignedURL = async ({ + filepath, + contentType, + metadata, + filename + }: PresignedUrlInputType) => { + const name = this.generateFileId(); + const objectName = `${filepath}/${name}`; + + const client = this.externalClient ?? this.client; + + try { + const policy = client.newPostPolicy(); + policy.setBucket(this.config.bucket); + policy.setKey(objectName); + if (contentType) { + policy.setContentType(contentType); + } + if (this.config.maxFileSize) { + policy.setContentLengthRange(1, this.config.maxFileSize); + } + policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); // 10 mins + + policy.setUserMetaData({ + 'original-filename': encodeURIComponent(filename), + 'upload-time': new Date().toISOString(), + ...metadata + }); + + const res = await client.presignedPostPolicy(policy); + const postURL = (() => { + if (this.config.externalBaseUrl) { + return `${this.config.externalBaseUrl}/${this.config.bucket}`; + } else { + return res.postURL; + } + })(); + return { + postURL, + formData: res.formData, + objectName + }; + } catch (error) { + addLog.error('Failed to generate Upload Presigned URL', error); + return Promise.reject(`Failed to generate Upload Presigned URL: ${getErrText(error)}`); + } + }; + + public async getFiles(prefix: string): Promise { + const objectNames: string[] = []; + const stream = this.client.listObjectsV2(this.config.bucket, prefix, true); + + for await (const obj of stream) { + if (obj.name) { + objectNames.push(obj.name); + } + } + + return objectNames; + } + + public removeFiles(objectNames: string[]) { + return this.client.removeObjects(this.config.bucket, objectNames); + } } diff --git a/src/s3/index.ts b/src/s3/index.ts new file mode 100644 index 00000000..4124eca3 --- /dev/null +++ b/src/s3/index.ts @@ -0,0 +1,31 @@ +import { S3Service } from './controller'; + +export const fileUploadS3Server = (() => { + if (!global._fileUploadS3Server) { + global._fileUploadS3Server = new S3Service({ + maxFileSize: process.env.MAX_FILE_SIZE + ? parseInt(process.env.MAX_FILE_SIZE) + : 20 * 1024 * 1024, // 默认 20MB + retentionDays: process.env.RETENTION_DAYS ? parseInt(process.env.RETENTION_DAYS) : 15, // 默认保留15天 + bucket: process.env.S3_TOOL_BUCKET || 'fastgpt-tool', + externalBaseUrl: process.env.S3_EXTERNAL_BASE_URL + }); + } + return global._fileUploadS3Server; +})(); + +export const pluginFileS3Server = (() => { + if (!global._pluginFileS3Server) { + global._pluginFileS3Server = new S3Service({ + maxFileSize: 50 * 1024 * 1024, // 默认 50MB + bucket: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', + externalBaseUrl: process.env.S3_EXTERNAL_BASE_URL + }); + } + return global._pluginFileS3Server; +})(); + +declare global { + var _fileUploadS3Server: S3Service; + var _pluginFileS3Server: S3Service; +} diff --git a/src/s3/type.ts b/src/s3/type.ts new file mode 100644 index 00000000..23d904f2 --- /dev/null +++ b/src/s3/type.ts @@ -0,0 +1,42 @@ +import { z } from 'zod'; + +export const PresignedUrlInputSchema = z.object({ + filepath: z.string(), + filename: z.string(), + contentType: z.string().optional(), + metadata: z.record(z.string()).optional() +}); + +export type PresignedUrlInputType = z.infer; + +export const FileInputSchema = z + .object({ + url: z.string().url('Invalid URL format').optional(), + path: z.string().min(1, 'File path cannot be empty').optional(), + base64: z.string().min(1, 'Base64 data cannot be empty').optional(), + buffer: z + .union([ + z.instanceof(Buffer, { message: 'Buffer is required' }), + z.instanceof(Uint8Array, { message: 'Uint8Array is required' }) + ]) + .transform((data) => { + if (data instanceof Uint8Array && !(data instanceof Buffer)) { + return Buffer.from(data); + } + return data; + }) + .optional(), + defaultFilename: z.string().optional() + }) + .refine( + (data) => { + const inputMethods = [data.url, data.path, data.base64, data.buffer].filter(Boolean); + return inputMethods.length === 1 && (!(data.base64 || data.buffer) || data.defaultFilename); + }, + { + message: 'Provide exactly one input method. Filename required for base64/buffer inputs.' + } + ); +export type FileInput = z.infer; + +export type GetUploadBufferResponse = { buffer: Buffer; filename: string }; diff --git a/src/utils/catch.ts b/src/utils/catch.ts index ca05d09e..0654bf74 100644 --- a/src/utils/catch.ts +++ b/src/utils/catch.ts @@ -3,7 +3,7 @@ * @param fn * @returns [result, error] */ -export async function catchError(fn: () => T): Promise<[T | null, unknown]> { +export async function catchError(fn: () => T): Promise<[Awaited | null, unknown]> { try { return [await fn(), null]; } catch (e) { diff --git a/src/utils/log.ts b/src/utils/log.ts index 6d2ba5a4..9625e766 100644 --- a/src/utils/log.ts +++ b/src/utils/log.ts @@ -45,11 +45,11 @@ const LOG_LEVEL = (() => { /* add logger */ export const addLog = { - log(level: LogLevelEnum, msg: string, obj: Record = {}) { + log(level: LogLevelEnum, msg: string, obj?: Record) { if (level < LOG_LEVEL) return; const stringifyObj = JSON.stringify(obj); - const isEmpty = Object.keys(obj).length === 0; + const isEmpty = !obj || Object.keys(obj).length === 0; console.log( `${logMap[level].levelLog} ${format(Date.now(), 'yyyy-MM-dd HH:mm:ss')}: ${msg} ${ @@ -57,7 +57,7 @@ export const addLog = { }` ); - if (level === LogLevelEnum.error) console.log(obj); + if (level === LogLevelEnum.error && obj) console.log(obj); const logger = getLogger(); if (logger) { @@ -81,22 +81,28 @@ export const addLog = { this.log(LogLevelEnum.warn, msg, obj); }, error(msg: string, error?: any) { - this.log(LogLevelEnum.error, msg, { - message: error?.message || error, - stack: error?.stack, - ...(error?.config && { - config: { - headers: error.config.headers, - url: error.config.url, - data: error.config.data - } - }), - ...(error?.response && { - response: { - status: error.response.status, - statusText: error.response.statusText - } - }) - }); + this.log( + LogLevelEnum.error, + msg, + error + ? { + message: error?.message || error, + stack: error?.stack, + ...(error?.config && { + config: { + headers: error.config.headers, + url: error.config.url, + data: error.config.data + } + }), + ...(error?.response && { + response: { + status: error.response.status, + statusText: error.response.statusText + } + }) + } + : undefined + ); } }; diff --git a/src/utils/string.ts b/src/utils/string.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/worker/index.ts b/src/worker/index.ts index 693e0474..c0a116b3 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -5,6 +5,7 @@ import { addLog } from '@/utils/log'; import { isProd } from '@/constants'; import type { Worker2MainMessageType } from './type'; import { getErrText } from '@tool/utils/err'; +import { fileUploadS3Server } from '@/s3'; type WorkerQueueItem = { id: string; @@ -199,7 +200,7 @@ export async function dispatchWithNewWorker(data: { console.log(...logData); } else if (type === 'uploadFile') { try { - const result = await global.s3Server.uploadFileAdvanced({ + const result = await fileUploadS3Server.uploadFileAdvanced({ ...data, ...(data.buffer ? { buffer: Buffer.from(data.buffer) } : {}) }); diff --git a/src/worker/type.ts b/src/worker/type.ts index 8d7ecc44..d3b31366 100644 --- a/src/worker/type.ts +++ b/src/worker/type.ts @@ -1,8 +1,8 @@ import z from 'zod'; -import { FileInputSchema } from '@/s3/controller'; import { FileMetadataSchema, type FileMetadata } from '@/s3/config'; import { StreamDataSchema } from '@tool/type/tool'; import { ToolCallbackReturnSchema } from '@tool/type/tool'; +import { FileInputSchema } from '@/s3/type'; declare global { var uploadFileResponseFn: (data: { data?: FileMetadata; error?: string }) => void | undefined; diff --git a/src/worker/worker.ts b/src/worker/worker.ts index c03964d3..0f53f186 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -1,10 +1,9 @@ import { parentPort } from 'worker_threads'; -import path from 'path'; import { LoadToolsByFilename } from '@tool/init'; -import { isProd } from '@/constants'; import { getErrText } from '@tool/utils/err'; import type { Main2WorkerMessageType } from './type'; import { setupProxy } from '@/utils/setupProxy'; + setupProxy(); // rewrite console.log to send to parent @@ -15,15 +14,17 @@ console.log = (...args: any[]) => { }); }; -const basePath = isProd - ? process.env.TOOLS_DIR || path.join(process.cwd(), 'dist', 'tools') - : path.join(process.cwd(), 'modules', 'tool', 'packages'); - parentPort?.on('message', async (params: Main2WorkerMessageType) => { const { type, data } = params; switch (type) { case 'runTool': { - const tools = await LoadToolsByFilename(basePath, data.toolDirName); + // Extract toolSource and filename from toolDirName (format: "toolSource/filename") + const [toolSource, filename] = data.toolDirName.split('/') as [ + 'uploaded' | 'built-in', + string + ]; + + const tools = await LoadToolsByFilename(filename, toolSource); const tool = tools.find((tool) => tool.toolId === data.toolId); diff --git a/tsconfig.json b/tsconfig.json index 22c09923..883254de 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -28,9 +28,9 @@ "baseUrl": "./", "paths": { "@/*": ["./src/*"], - "@tool/*": ["modules/tool/*"], - "@model/*": ["modules/model/*"], - "@workflow/*": ["modules/workflow/*"] + "@tool/*": ["./modules/tool/*"], + "@model/*": ["./modules/model/*"], + "@workflow/*": ["./modules/workflow/*"] } }, "include": ["**/*.ts", "**/*.d.ts"]