Skip to content

Commit 5974d73

Browse files
dreamerbluedreamerblue
committed
fix: upload-cdn script
1 parent 86d0b50 commit 5974d73

1 file changed

Lines changed: 199 additions & 120 deletions

File tree

‎tasks/upload-cdn.js‎

Lines changed: 199 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@
44

55
const fs = require('fs');
66
const pathLib = require('path');
7-
const Batch = require('batch');
7+
const { promisify } = require('util');
88
const COS = require('cos-nodejs-sdk-v5');
9-
const Async = require('cos-nodejs-sdk-v5/sdk/async');
9+
10+
const readdir = promisify(fs.readdir);
11+
const stat = promisify(fs.stat);
1012

1113
const cos = new COS({
1214
SecretId: process.env.COS_SECRET_ID,
@@ -17,143 +19,220 @@ const cos = new COS({
1719
const Bucket = process.env.COS_BUCKET;
1820
const Region = process.env.COS_REGION;
1921

22+
const SLICE_SIZE = 1024 * 1024;
23+
const MAX_FILE_COUNT = 1000000;
24+
const LIST_CONCURRENCY = 16;
25+
const UPLOAD_CONCURRENCY = 3;
26+
const MAX_UPLOAD_RETRIES = 3;
27+
const RETRY_DELAY_MS = 1000;
2028

2129
const publicDir = process.argv[2] || 'onlinejudge3';
22-
console.log('Using public dir:', publicDir);
2330
const cosTargetBase = process.env.COS_TARGET_BASE || ''; // should end with /
24-
const localFolder = `./${publicDir}/`;
31+
const localFolder = pathLib.join('.', publicDir);
2532
const remotePrefix = `${cosTargetBase}${publicDir}/`;
2633

27-
const fastListFolder = function(options, callback) {
28-
const pathJoin = function(dir, name, isDir) {
29-
dir = dir.replace(/\\/g, '/');
30-
const sep = dir.endsWith('/') ? '' : '/';
31-
let p = dir + sep + name;
32-
p = p.replace(/\\/g, '/');
33-
isDir && name && (p += '/');
34-
return p;
35-
};
34+
function delay(ms) {
35+
if (!ms) return Promise.resolve();
36+
return new Promise((resolve) => setTimeout(resolve, ms));
37+
}
3638

37-
const readdir = function stat(dir, cb) {
38-
if (!dir || !cb) throw new Error('stat(dir, cb[, concurrency])');
39-
fs.readdir(dir, function(err, files) {
40-
if (err) return cb(err);
41-
const batch = new Batch();
42-
batch.concurrency(16);
43-
files.forEach(function(file) {
44-
const filePath = pathJoin(dir, file);
45-
batch.push(function(done) {
46-
fs.stat(filePath, done);
47-
});
48-
});
49-
batch.end(function(err, stats) {
50-
if (err) {
51-
console.log('readdir error:', err);
52-
cb(err);
53-
return;
54-
}
55-
stats.forEach(function(stat, i) {
56-
stat.isDir = stat.isDirectory();
57-
stat.path = pathJoin(dir, files[i], stat.isDir);
58-
stat.isDir && (stat.size = 0);
59-
});
60-
cb(err, stats);
61-
});
62-
});
63-
};
39+
function formatError(error) {
40+
if (!error) return 'Unknown error';
41+
if (error.message) return error.message;
42+
try {
43+
return JSON.stringify(error);
44+
} catch (e) {
45+
return String(error);
46+
}
47+
}
6448

65-
const statFormat = function(stat) {
66-
return {
67-
path: stat.path,
68-
size: stat.size,
69-
isDir: stat.isDir,
70-
};
71-
};
49+
async function runWithConcurrency(items, concurrency, worker) {
50+
const results = new Array(items.length);
51+
let nextIndex = 0;
52+
const workerCount = Math.min(concurrency, items.length);
7253

73-
if (typeof options !== 'object') options = { path: options };
74-
const rootPath = options.path;
75-
let list = [];
76-
const _callback = function(err) {
77-
if (err) {
78-
callback(err);
79-
} else if (list.length > 1000000) {
80-
callback(window.lang.t('error.too_much_files'));
81-
} else {
82-
callback(null, list);
54+
async function runNext() {
55+
while (nextIndex < items.length) {
56+
const currentIndex = nextIndex;
57+
nextIndex += 1;
58+
results[currentIndex] = await worker(items[currentIndex], currentIndex);
8359
}
84-
};
85-
const deep = function(dirStat, deepNext) {
86-
list.push(statFormat(dirStat));
87-
readdir(dirStat.path, function(err, files) {
88-
if (err) return deepNext();
89-
const dirList = files.filter((file) => file.isDir);
90-
const fileList = files.filter((file) => !file.isDir);
91-
list = [].concat(list, fileList.map(statFormat));
92-
Async.eachLimit(dirList, 1, deep, deepNext);
60+
}
61+
62+
await Promise.all(Array.from({ length: workerCount }, runNext));
63+
return results;
64+
}
65+
66+
async function listLocalFiles(rootPath) {
67+
const rootStat = await stat(rootPath);
68+
if (!rootStat.isDirectory()) {
69+
throw new Error(`Local path is not a directory: ${rootPath}`);
70+
}
71+
72+
async function walk(dir) {
73+
const names = await readdir(dir);
74+
const entries = await runWithConcurrency(names, LIST_CONCURRENCY, async function(name) {
75+
const filePath = pathLib.join(dir, name);
76+
const fileStat = await stat(filePath);
77+
return {
78+
filePath,
79+
isDirectory: fileStat.isDirectory(),
80+
isFile: fileStat.isFile(),
81+
size: fileStat.size,
82+
};
9383
});
94-
};
95-
fs.stat(rootPath, function(err, stat) {
96-
if (err) return _callback();
97-
stat.isDir = true;
98-
stat.path = pathJoin(rootPath, '', true);
99-
stat.isDir && (stat.size = 0);
100-
deep(stat, _callback);
101-
});
102-
};
103-
104-
fastListFolder(localFolder, function(err, list) {
105-
if (err) return console.error(err);
106-
let files = list.map(function(file) {
107-
let filename = pathLib.relative(localFolder, file.path).replace(/\\/g, '/');
108-
// if (filename && file.isDir && !filename.endsWith('/')) filename += '/';
109-
if (file.isDir) {
110-
return null;
84+
85+
let files = [];
86+
for (const entry of entries) {
87+
if (entry.isDirectory) {
88+
files = files.concat(await walk(entry.filePath));
89+
} else if (entry.isFile) {
90+
files.push({
91+
path: entry.filePath,
92+
size: entry.size,
93+
});
94+
}
11195
}
112-
const Key = remotePrefix + filename;
96+
return files;
97+
}
98+
99+
const files = await walk(rootPath);
100+
if (files.length > MAX_FILE_COUNT) {
101+
throw new Error(`Too many files to upload: ${files.length}`);
102+
}
103+
return files;
104+
}
105+
106+
function buildUploadFiles(localFiles) {
107+
return localFiles.map(function(file) {
108+
const filename = pathLib.relative(localFolder, file.path).replace(/\\/g, '/');
113109
return {
114110
Bucket,
115111
Region,
116-
Key,
112+
Key: remotePrefix + filename,
117113
FilePath: file.path,
114+
Size: file.size,
118115
};
119-
}).filter(Boolean);
120-
// 移动 index.html 到最后上传
116+
});
117+
}
118+
119+
function splitIndexFile(files) {
121120
const indexFile = files.find((file) => file.Key.endsWith('index.html'));
122-
if (indexFile) {
123-
files = files.filter((file) => file.Key !== indexFile.Key);
124-
files.push(indexFile);
121+
if (!indexFile) {
122+
return {
123+
normalFiles: files,
124+
indexFile: null,
125+
};
126+
}
127+
128+
return {
129+
normalFiles: files.filter((file) => file.Key !== indexFile.Key),
130+
indexFile,
131+
};
132+
}
133+
134+
async function uploadFileOnce(file, attempt, totalAttempts) {
135+
let lastProgressLogAt = 0;
136+
await cos.uploadFile({
137+
Bucket: file.Bucket,
138+
Region: file.Region,
139+
Key: file.Key,
140+
FilePath: file.FilePath,
141+
SliceSize: SLICE_SIZE,
142+
onProgress: function(info) {
143+
const now = Date.now();
144+
if (now - lastProgressLogAt < 1000 && info.percent < 1) return;
145+
lastProgressLogAt = now;
146+
147+
const percent = Math.floor(info.percent * 10000) / 100;
148+
const speed = Math.floor((info.speed / 1024 / 1024) * 100) / 100;
149+
console.log(
150+
`${file.Key} progress: ${percent}%; speed: ${speed}Mb/s; attempt: ${attempt}/${totalAttempts}`,
151+
);
152+
},
153+
});
154+
}
155+
156+
async function uploadFileWithRetry(file) {
157+
const totalAttempts = MAX_UPLOAD_RETRIES + 1;
158+
let lastError = null;
159+
160+
for (let attempt = 1; attempt <= totalAttempts; attempt += 1) {
161+
try {
162+
console.log(`${file.Key} upload start; attempt: ${attempt}/${totalAttempts}`);
163+
await uploadFileOnce(file, attempt, totalAttempts);
164+
console.log(`${file.Key} upload success; attempt: ${attempt}/${totalAttempts}`);
165+
return;
166+
} catch (error) {
167+
lastError = error;
168+
console.error(`${file.Key} upload failed; attempt: ${attempt}/${totalAttempts}: ${formatError(error)}`);
169+
170+
if (attempt < totalAttempts) {
171+
console.log(`${file.Key} retrying; retry: ${attempt}/${MAX_UPLOAD_RETRIES}`);
172+
await delay(RETRY_DELAY_MS * attempt);
173+
}
174+
}
175+
}
176+
177+
const error = new Error(`${file.Key} failed after ${totalAttempts} attempts: ${formatError(lastError)}`);
178+
error.cause = lastError;
179+
throw error;
180+
}
181+
182+
async function uploadFiles(files, concurrency) {
183+
if (!files.length) return;
184+
185+
const failures = [];
186+
await runWithConcurrency(files, concurrency, async function(file) {
187+
try {
188+
await uploadFileWithRetry(file);
189+
} catch (error) {
190+
failures.push({ file, error });
191+
}
192+
});
193+
194+
if (failures.length) {
195+
const detail = failures
196+
.map((failure) => `${failure.file.Key}: ${formatError(failure.error)}`)
197+
.join('\n');
198+
const error = new Error(`Upload failed for ${failures.length}/${files.length} files:\n${detail}`);
199+
error.failures = failures;
200+
throw error;
201+
}
202+
}
203+
204+
async function main() {
205+
console.log('Using public dir:', publicDir);
206+
console.log('Using upload concurrency:', UPLOAD_CONCURRENCY);
207+
console.log('Using upload retries:', MAX_UPLOAD_RETRIES);
208+
209+
const localFiles = await listLocalFiles(localFolder);
210+
const files = buildUploadFiles(localFiles);
211+
212+
if (!files.length) {
213+
throw new Error(`No files found to upload in ${localFolder}`);
125214
}
215+
126216
console.log('to upload files:');
127217
files.forEach(function(file) {
128218
console.log(file.FilePath);
129219
});
130-
cos.uploadFiles(
131-
{
132-
files: files,
133-
SliceSize: 1024 * 1024,
134-
onProgress: function(info) {
135-
const percent = Math.floor(info.percent * 10000) / 100;
136-
const speed = Math.floor((info.speed / 1024 / 1024) * 100) / 100;
137-
console.log('progress: ' + percent + '%; speed: ' + speed + 'Mb/s');
138-
},
139-
onFileFinish: function(err, data, options) {
140-
if (err) {
141-
console.log(options.Key + ' upload failed:', err);
142-
} else {
143-
console.log(options.Key + ' upload success');
144-
}
145-
if (err) {
146-
// // 有文件上传失败时不会进入到最终的回调 err，只能在此直接退出
147-
// process.exit(1);
148-
}
149-
},
150-
},
151-
function(err, data) {
152-
if (err) {
153-
console.log('error:', err);
154-
process.exit(1);
155-
}
156-
process.exit(0);
157-
},
158-
);
159-
});
220+
221+
const { normalFiles, indexFile } = splitIndexFile(files);
222+
await uploadFiles(normalFiles, UPLOAD_CONCURRENCY);
223+
224+
// index.html 最后上传，避免入口文件先更新后引用到尚未上传完成的静态资源。
225+
if (indexFile) {
226+
await uploadFiles([indexFile], 1);
227+
}
228+
}
229+
230+
main()
231+
.then(function() {
232+
console.log('Upload CDN success.');
233+
})
234+
.catch(function(error) {
235+
console.error('Upload CDN failed:');
236+
console.error(error && error.stack ? error.stack : error);
237+
process.exit(1);
238+
});

0 commit comments

Comments
 (0)