feat: 增加同步配置分批上传与失败后续批次继续机制

xream · xream · commit 547e0abdcfad · 2026-05-12T20:02:02.000+08:00
- 将 artifact 同步上传流程改为分批执行，新增 artifactSyncBatchSize 配置项（默认 10，可自定义），避免一次上传量过大
- 提取并复用 Gist 响应日志打印与文件链接解析逻辑（兼容 GitHub/GitLab），减少重复代码
- cron 同步和手动同步都改为调用统一批量上传能力，批次失败时仍继续下一批，提升同步稳定性
- 上传成功/失败数量改为按实际成功上传批次统计，并保留失败明细
- 新增/扩展测试：batch size 归一化、批次上传失败后还能继续处理后续批次
diff --git a/backend/package.json b/backend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "sub-store",
-  "version": "2.22.29",
+  "version": "2.23.0",
   "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and Shadowrocket.",
   "main": "src/main.js",
   "packageManager": "pnpm@11.0.9",
diff --git a/backend/src/products/cron-sync-artifacts.js b/backend/src/products/cron-sync-artifacts.js
@@ -6,8 +6,7 @@ import {
     COLLECTIONS_KEY,
 } from '@/constants';
 import $ from '@/core/app';
-import { produceArtifact } from '@/restful/sync';
-import { syncToGist } from '@/restful/artifacts';
+import { produceArtifact, uploadArtifactBatches } from '@/restful/sync';
 import { findByName } from '@/utils/database';
 import { hasCronArtifactSyncCredentials } from '@/products/cron-sync-artifacts-eligibility';
 
@@ -213,56 +212,20 @@ async function doSync(arg = {}) {
             );
         }
 
-        const resp = await syncToGist(files);
-        const body = JSON.parse(resp.body);
-        delete body.history;
-        delete body.forks;
-        delete body.owner;
-        Object.values(body.files).forEach((file) => {
-            delete file.content;
+        const uploaded = await uploadArtifactBatches({
+            allArtifacts,
+            files,
+            valid,
+            invalid,
         });
-        $.info('上传配置响应:');
-        $.info(JSON.stringify(body, null, 2));
-
-        for (const artifact of allArtifacts) {
-            if (
-                artifact.sync &&
-                artifact.source &&
-                valid.includes(artifact.name)
-            ) {
-                artifact.updated = new Date().getTime();
-                // extract real url from gist
-                let files = body.files;
-                let isGitLab;
-                if (Array.isArray(files)) {
-                    isGitLab = true;
-                    files = Object.fromEntries(
-                        files.map((item) => [item.path, item]),
-                    );
-                }
-                const raw_url =
-                    files[encodeURIComponent(artifact.name)]?.raw_url;
-                const new_url = isGitLab
-                    ? raw_url
-                    : raw_url?.replace(/\/raw\/[^/]*\/(.*)/, '/raw/$1');
-                $.info(
-                    `上传配置完成\n文件列表: ${Object.keys(files).join(
-                        ', ',
-                    )}\n当前文件: ${encodeURIComponent(
-                        artifact.name,
-                    )}\n响应返回的原始链接: ${raw_url}\n处理完的新链接: ${new_url}`,
-                );
-                artifact.url = new_url;
-            }
-        }
 
         $.write(allArtifacts, ARTIFACTS_KEY);
         $.info('上传配置成功');
 
         if (invalid.length > 0) {
             $.notify(
                 '🌍 Sub-Store',
-                `同步配置成功 ${valid.length} 个, 失败 ${invalid.length} 个, 详情请查看日志`,
+                `同步配置成功 ${uploaded.length} 个, 失败 ${invalid.length} 个, 详情请查看日志`,
             );
         } else if (syncSuccessNotify) {
             $.notify('🌍 Sub-Store', '同步配置完成');
diff --git a/backend/src/restful/artifacts.js b/backend/src/restful/artifacts.js
@@ -26,6 +26,17 @@ const ARTIFACT_GIST_PLACEHOLDER_CONTENT = [
     'Sub-Store placeholder',
     'This file keeps the Gist alive when all sync configuration files are deleted.',
 ].join('\n');
+const DEFAULT_ARTIFACT_SYNC_BATCH_SIZE = 10;
+
+function normalizeArtifactSyncBatchSize(value) {
+    const batchSize = Math.floor(Number(value));
+
+    if (!isFinite(batchSize) || batchSize <= 0) {
+        return DEFAULT_ARTIFACT_SYNC_BATCH_SIZE;
+    }
+
+    return batchSize;
+}
 
 export default function register($app) {
     // Initialization
@@ -411,5 +422,5 @@ function formatBytes(size) {
     return `${(size / 1024 / 1024).toFixed(1)} MB`;
 }
 
-export { syncToGist };
+export { syncToGist, normalizeArtifactSyncBatchSize };
 export { createArtifactItem, deleteArtifactItem };
diff --git a/backend/src/restful/settings.js b/backend/src/restful/settings.js
@@ -68,6 +68,7 @@ async function updateSettings(req, res) {
         [
             'defaultTimeout',
             'githubApiTimeout',
+            'artifactSyncBatchSize',
             'cacheThreshold',
             'resourceCacheTtl',
             'headersCacheTtl',
diff --git a/backend/src/restful/sync.js b/backend/src/restful/sync.js
@@ -5,14 +5,18 @@ import {
     RULES_KEY,
     SUBS_KEY,
     FILES_KEY,
+    SETTINGS_KEY,
 } from '@/constants';
 import { failed, success } from '@/restful/response';
 import { InternalServerError, ResourceNotFoundError } from '@/restful/errors';
 import { findByName } from '@/utils/database';
 import download from '@/utils/download';
 import { ProxyUtils } from '@/core/proxy-utils';
 import { RuleUtils } from '@/core/rule-utils';
-import { syncToGist } from '@/restful/artifacts';
+import {
+    normalizeArtifactSyncBatchSize,
+    syncToGist,
+} from '@/restful/artifacts';
 import {
     buildEmptySubscriptionOutput,
     handleIgnoreFailedRemoteSubError,
@@ -701,6 +705,122 @@ async function produceArtifact({
     }
 }
 
+function createArtifactUploadBatches(names, batchSize) {
+    const batches = [];
+    for (let index = 0; index < names.length; index += batchSize) {
+        batches.push(names.slice(index, index + batchSize));
+    }
+    return batches;
+}
+
+function normalizeUploadResponseFiles(files) {
+    if (Array.isArray(files)) {
+        return {
+            isGitLab: true,
+            files: Object.fromEntries(files.map((item) => [item.path, item])),
+        };
+    }
+
+    return {
+        isGitLab: false,
+        files: files || {},
+    };
+}
+
+function logUploadResponse(body) {
+    delete body.history;
+    delete body.forks;
+    delete body.owner;
+    if (body.files) {
+        Object.values(body.files).forEach((file) => {
+            delete file.content;
+        });
+    }
+    $.info('上传配置响应:');
+    $.info(JSON.stringify(body, null, 2));
+}
+
+function resolveArtifactUploadUrl(body, artifactName) {
+    const { files, isGitLab } = normalizeUploadResponseFiles(body.files);
+    const encodedName = encodeURIComponent(artifactName);
+    const raw_url = files[encodedName]?.raw_url;
+    const new_url = isGitLab
+        ? raw_url
+        : raw_url?.replace(/\/raw\/[^/]*\/(.*)/, '/raw/$1');
+    $.info(
+        `上传配置完成\n文件列表: ${Object.keys(files).join(
+            ', ',
+        )}\n当前文件: ${encodedName}\n响应返回的原始链接: ${raw_url}\n处理完的新链接: ${new_url}`,
+    );
+    return new_url;
+}
+
+async function uploadArtifactBatches({ allArtifacts, files, valid, invalid }) {
+    const settings = $.read(SETTINGS_KEY) || {};
+    const batchSize = normalizeArtifactSyncBatchSize(
+        settings.artifactSyncBatchSize,
+    );
+    const batches = createArtifactUploadBatches(valid, batchSize);
+    const uploaded = [];
+
+    $.info(
+        `准备分批上传同步配置: 共 ${valid.length} 个, 每批 ${batchSize} 个, 批次数 ${batches.length}`,
+    );
+
+    for (let index = 0; index < batches.length; index++) {
+        const batchNames = batches[index];
+        const batchFiles = Object.fromEntries(
+            batchNames.map((name) => [
+                encodeURIComponent(name),
+                files[encodeURIComponent(name)],
+            ]),
+        );
+
+        try {
+            $.info(
+                `正在上传第 ${index + 1}/${batches.length} 批同步配置: ${batchNames.join(
+                    ', ',
+                )}`,
+            );
+            const resp = await syncToGist(batchFiles);
+            const body = JSON.parse(resp.body);
+            logUploadResponse(body);
+
+            for (const artifact of allArtifacts) {
+                if (
+                    artifact.sync &&
+                    artifact.source &&
+                    batchNames.includes(artifact.name)
+                ) {
+                    const newUrl = resolveArtifactUploadUrl(
+                        body,
+                        artifact.name,
+                    );
+                    if (newUrl) {
+                        artifact.updated = new Date().getTime();
+                        artifact.url = newUrl;
+                        uploaded.push(artifact.name);
+                    } else {
+                        $.error(
+                            `同步配置 ${artifact.name} 上传成功但响应中未找到文件链接`,
+                        );
+                        invalid.push(artifact.name);
+                    }
+                }
+            }
+        } catch (e) {
+            $.error(
+                `第 ${index + 1}/${batches.length} 批同步配置上传失败: ${batchNames.join(
+                    ', ',
+                )}, 原因: ${e.message ?? e}`,
+            );
+            invalid.push(...batchNames);
+        }
+    }
+
+    return uploaded;
+}
+
 async function syncArtifacts() {
     $.info('开始同步所有远程配置...');
     const allArtifacts = $.read(ARTIFACTS_KEY);
@@ -815,59 +935,22 @@ async function syncArtifacts() {
             );
         }
 
-        const resp = await syncToGist(files);
-        const body = JSON.parse(resp.body);
-
-        delete body.history;
-        delete body.forks;
-        delete body.owner;
-        Object.values(body.files).forEach((file) => {
-            delete file.content;
+        const uploaded = await uploadArtifactBatches({
+            allArtifacts,
+            files,
+            valid,
+            invalid,
         });
-        $.info('上传配置响应:');
-        $.info(JSON.stringify(body, null, 2));
-
-        for (const artifact of allArtifacts) {
-            if (
-                artifact.sync &&
-                artifact.source &&
-                valid.includes(artifact.name)
-            ) {
-                artifact.updated = new Date().getTime();
-                // extract real url from gist
-                let files = body.files;
-                let isGitLab;
-                if (Array.isArray(files)) {
-                    isGitLab = true;
-                    files = Object.fromEntries(
-                        files.map((item) => [item.path, item]),
-                    );
-                }
-                const raw_url =
-                    files[encodeURIComponent(artifact.name)]?.raw_url;
-                const new_url = isGitLab
-                    ? raw_url
-                    : raw_url?.replace(/\/raw\/[^/]*\/(.*)/, '/raw/$1');
-                $.info(
-                    `上传配置完成\n文件列表: ${Object.keys(files).join(
-                        ', ',
-                    )}\n当前文件: ${encodeURIComponent(
-                        artifact.name,
-                    )}\n响应返回的原始链接: ${raw_url}\n处理完的新链接: ${new_url}`,
-                );
-                artifact.url = new_url;
-            }
-        }
 
         $.write(allArtifacts, ARTIFACTS_KEY);
         $.info('上传配置成功');
 
         if (invalid.length > 0) {
             throw new Error(
-                `同步配置成功 ${valid.length} 个, 失败 ${invalid.length} 个, 详情请查看日志`,
+                `同步配置成功 ${uploaded.length} 个, 失败 ${invalid.length} 个, 详情请查看日志`,
             );
         } else {
-            $.info(`同步配置成功 ${valid.length} 个`);
+            $.info(`同步配置成功 ${uploaded.length} 个`);
         }
     } catch (e) {
         $.error(`同步配置失败，原因：${e.message ?? e}`);
@@ -958,32 +1041,8 @@ async function syncArtifact(req, res) {
         artifact.updated = new Date().getTime();
         const body = JSON.parse(resp.body);
 
-        delete body.history;
-        delete body.forks;
-        delete body.owner;
-        Object.values(body.files).forEach((file) => {
-            delete file.content;
-        });
-        $.info('上传配置响应:');
-        $.info(JSON.stringify(body, null, 2));
-
-        let files = body.files;
-        let isGitLab;
-        if (Array.isArray(files)) {
-            isGitLab = true;
-            files = Object.fromEntries(files.map((item) => [item.path, item]));
-        }
-        const raw_url = files[encodeURIComponent(artifact.name)]?.raw_url;
-        const new_url = isGitLab
-            ? raw_url
-            : raw_url?.replace(/\/raw\/[^/]*\/(.*)/, '/raw/$1');
-        $.info(
-            `上传配置完成\n文件列表: ${Object.keys(files).join(
-                ', ',
-            )}\n当前文件: ${encodeURIComponent(
-                artifact.name,
-            )}\n响应返回的原始链接: ${raw_url}\n处理完的新链接: ${new_url}`,
-        );
+        logUploadResponse(body);
+        const new_url = resolveArtifactUploadUrl(body, artifact.name);
         artifact.url = new_url;
         $.write(allArtifacts, ARTIFACTS_KEY);
         success(res, artifact);
@@ -1000,4 +1059,4 @@ async function syncArtifact(req, res) {
     }
 }
 
-export { produceArtifact, syncArtifacts };
+export { produceArtifact, syncArtifacts, uploadArtifactBatches };
diff --git a/backend/src/test/utils/gist.spec.js b/backend/src/test/utils/gist.spec.js

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "sub-store",`
`3`		`- "version": "2.22.29",`
	`3`	`+ "version": "2.23.0",`
`4`	`4`	`"description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and Shadowrocket.",`
`5`	`5`	`"main": "src/main.js",`
`6`	`6`	`"packageManager": "pnpm@11.0.9",`
Original file line number	Diff line number	Diff line change
`@@ -68,6 +68,7 @@ async function updateSettings(req, res) {`
`68`	`68`	`[`
`69`	`69`	`'defaultTimeout',`
`70`	`70`	`'githubApiTimeout',`
	`71`	`+ 'artifactSyncBatchSize',`
`71`	`72`	`'cacheThreshold',`
`72`	`73`	`'resourceCacheTtl',`
`73`	`74`	`'headersCacheTtl',`