diff --git a/docs/api.md b/docs/api.md index 208155ad..cd4a450d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -208,6 +208,34 @@ Following example will trigger up to three retries, each after 1s, 3s and 5s res retryDelays: [1000, 3000, 5000] ``` +#### stallDetection + +_Default value:_ `{ enabled: false, stallTimeout: 30000, checkInterval: 5000 }` + +An object controlling the stall detection feature, which can automatically detect when an upload has stopped making progress and trigger a retry. This is useful for recovering from frozen uploads caused by network issues that don't trigger explicit errors. + +The stall detection options are: +- `enabled`: Boolean indicating whether stall detection is active (default: `false`) +- `stallTimeout`: Time in milliseconds without progress before considering the upload stalled (default: `30000`) +- `checkInterval`: How often in milliseconds to check for stalls (default: `5000`) + +**Note:** Stall detection only works with HTTP stacks that support progress events. Currently, this includes: +- `XHRHttpStack` (browser default) - Supported +- `NodeHttpStack` (Node.js default) - Supported +- `FetchHttpStack` - Not supported + +When a stall is detected, the upload will be automatically retried according to your `retryDelays` configuration. If `retryDelays` is `null`, the stall will trigger an error instead. + +Example configuration: + +```js +stallDetection: { + enabled: true, + stallTimeout: 15000, // 15 seconds without progress + checkInterval: 2000 // Check every 2 seconds +} +``` + #### storeFingerprintForResuming _Default value:_ `true` @@ -326,6 +354,7 @@ An object used as the HTTP stack for making network requests. This is an abstrac interface HttpStack { createRequest(method: string, url: string): HttpRequest; getName(): string; + supportsProgressEvents(): boolean; } interface HttpRequest { @@ -367,6 +396,14 @@ interface HttpResponse { ``` +The `supportsProgressEvents()` method should return `true` if the HTTP stack implementation supports progress events during upload, or `false` otherwise. This is used by tus-js-client to determine whether features like stall detection can be enabled. The built-in HTTP stacks have the following support: + +- `XHRHttpStack` (browser default): Returns `true` - XMLHttpRequest supports progress events +- `NodeHttpStack` (Node.js default): Returns `true` - Node.js HTTP module supports progress events +- `FetchHttpStack`: Returns `false` - Fetch API does not support upload progress events + +If you're implementing a custom HTTP stack, you should return `true` only if your implementation can reliably call the progress handler set via `setProgressHandler` during the upload process. + #### urlStorage _Default value:_ Environment-specific implementation diff --git a/lib/StallDetector.ts b/lib/StallDetector.ts new file mode 100644 index 00000000..046c776f --- /dev/null +++ b/lib/StallDetector.ts @@ -0,0 +1,93 @@ +import { log } from './logger.js' +import type { StallDetectionOptions } from './options.js' + +export class StallDetector { + private options: StallDetectionOptions + private onStallDetected: (reason: string) => void + + private intervalId: ReturnType | null = null + private lastProgressTime = 0 + private lastProgressValue = 0 + private isActive = false + + constructor(options: StallDetectionOptions, onStallDetected: (reason: string) => void) { + this.options = options + this.onStallDetected = onStallDetected + } + + /** + * Start monitoring for stalls + */ + start() { + if (this.intervalId) { + return // Already started + } + + this.lastProgressTime = Date.now() + this.lastProgressValue = 0 + this.isActive = true + + log( + `tus: starting stall detection with checkInterval: ${this.options.checkInterval}ms, stallTimeout: ${this.options.stallTimeout}ms`, + ) + + // Setup periodic check + this.intervalId = setInterval(() => { + if (!this.isActive) { + return + } + + const now = Date.now() + if (this._isProgressStalled(now)) { + this._handleStall('no progress') + } + }, this.options.checkInterval) + } + + /** + * Stop monitoring for stalls + */ + stop(): void { + this.isActive = false + if (this.intervalId) { + clearInterval(this.intervalId) + this.intervalId = null + } + } + + /** + * Update progress information + * @param progressValue The current progress value (bytes uploaded) + */ + updateProgress(progressValue: number): void { + // Only update progress time if the value has actually changed + if (progressValue !== this.lastProgressValue) { + this.lastProgressTime = Date.now() + this.lastProgressValue = progressValue + } + } + + /** + * Check if upload has stalled based on progress events + */ + private _isProgressStalled(now: number): boolean { + const timeSinceProgress = now - this.lastProgressTime + const stallTimeout = this.options.stallTimeout + const isStalled = timeSinceProgress > stallTimeout + + if (isStalled) { + log(`tus: no progress for ${timeSinceProgress}ms (limit: ${stallTimeout}ms)`) + } + + return isStalled + } + + /** + * Handle a detected stall + */ + private _handleStall(reason: string): void { + log(`tus: upload stalled: ${reason}`) + this.stop() + this.onStallDetected(reason) + } +} diff --git a/lib/browser/FetchHttpStack.ts b/lib/browser/FetchHttpStack.ts index 0524e296..9c102d57 100644 --- a/lib/browser/FetchHttpStack.ts +++ b/lib/browser/FetchHttpStack.ts @@ -16,6 +16,11 @@ export class FetchHttpStack implements HttpStack { getName() { return 'FetchHttpStack' } + + supportsProgressEvents(): boolean { + // The Fetch API does not support progress events for uploads + return false + } } class FetchRequest implements HttpRequest { diff --git a/lib/browser/XHRHttpStack.ts b/lib/browser/XHRHttpStack.ts index 3e237da0..b95b6843 100644 --- a/lib/browser/XHRHttpStack.ts +++ b/lib/browser/XHRHttpStack.ts @@ -15,6 +15,11 @@ export class XHRHttpStack implements HttpStack { getName() { return 'XHRHttpStack' } + + supportsProgressEvents(): boolean { + // XMLHttpRequest supports progress events via the upload.onprogress event + return true + } } class XHRRequest implements HttpRequest { diff --git a/lib/browser/index.ts b/lib/browser/index.ts index c584040d..5949c333 100644 --- a/lib/browser/index.ts +++ b/lib/browser/index.ts @@ -19,12 +19,32 @@ const defaultOptions = { class Upload extends BaseUpload { constructor(file: UploadInput, options: Partial = {}) { - const allOpts = { ...defaultOptions, ...options } + const allOpts = { + ...defaultOptions, + ...options, + // Deep merge stallDetection options if provided + ...(options.stallDetection && { + stallDetection: { + ...defaultOptions.stallDetection, + ...options.stallDetection, + }, + }), + } super(file, allOpts) } static terminate(url: string, options: Partial = {}) { - const allOpts = { ...defaultOptions, ...options } + const allOpts = { + ...defaultOptions, + ...options, + // Deep merge stallDetection options if provided + ...(options.stallDetection && { + stallDetection: { + ...defaultOptions.stallDetection, + ...options.stallDetection, + }, + }), + } return terminate(url, allOpts) } } @@ -38,4 +58,6 @@ const isSupported = // Note: The exported interface must be the same as in lib/node/index.ts. // Any changes should be reflected in both files. export { Upload, defaultOptions, isSupported, canStoreURLs, enableDebugLog, DetailedError } +export { XHRHttpStack } from './XHRHttpStack.js' +export { FetchHttpStack } from './FetchHttpStack.js' export type * from '../options.js' diff --git a/lib/node/NodeHttpStack.ts b/lib/node/NodeHttpStack.ts index ab018cd2..df3704d5 100644 --- a/lib/node/NodeHttpStack.ts +++ b/lib/node/NodeHttpStack.ts @@ -28,6 +28,11 @@ export class NodeHttpStack implements HttpStack { getName() { return 'NodeHttpStack' } + + supportsProgressEvents(): boolean { + // Node.js HTTP stack supports progress tracking through streams + return true + } } class Request implements HttpRequest { diff --git a/lib/node/index.ts b/lib/node/index.ts index 91516141..2acd8fab 100644 --- a/lib/node/index.ts +++ b/lib/node/index.ts @@ -19,12 +19,32 @@ const defaultOptions = { class Upload extends BaseUpload { constructor(file: UploadInput, options: Partial = {}) { - const allOpts = { ...defaultOptions, ...options } + const allOpts = { + ...defaultOptions, + ...options, + // Deep merge stallDetection options if provided + ...(options.stallDetection && { + stallDetection: { + ...defaultOptions.stallDetection, + ...options.stallDetection, + }, + }), + } super(file, allOpts) } static terminate(url: string, options: Partial = {}) { - const allOpts = { ...defaultOptions, ...options } + const allOpts = { + ...defaultOptions, + ...options, + // Deep merge stallDetection options if provided + ...(options.stallDetection && { + stallDetection: { + ...defaultOptions.stallDetection, + ...options.stallDetection, + }, + }), + } return terminate(url, allOpts) } } @@ -36,4 +56,5 @@ const isSupported = true // Note: The exported interface must be the same as in lib/browser/index.ts. // Any changes should be reflected in both files. export { Upload, defaultOptions, isSupported, canStoreURLs, enableDebugLog, DetailedError } +export { NodeHttpStack } from './NodeHttpStack.js' export type * from '../options.js' diff --git a/lib/options.ts b/lib/options.ts index 011de2f5..3b80e68c 100644 --- a/lib/options.ts +++ b/lib/options.ts @@ -48,6 +48,15 @@ export type UploadInput = // available in React Native | ReactNativeFile +/** + * Options for configuring stall detection behavior + */ +export interface StallDetectionOptions { + enabled: boolean + stallTimeout: number // Time in ms before considering progress stalled + checkInterval: number // How often to check for stalls +} + export interface UploadOptions { endpoint?: string @@ -84,6 +93,8 @@ export interface UploadOptions { httpStack: HttpStack protocol: typeof PROTOCOL_TUS_V1 | typeof PROTOCOL_IETF_DRAFT_03 | typeof PROTOCOL_IETF_DRAFT_05 + + stallDetection?: StallDetectionOptions } export interface OnSuccessPayload { @@ -141,6 +152,10 @@ export type SliceResult = export interface HttpStack { createRequest(method: string, url: string): HttpRequest getName(): string + + // Indicates whether this HTTP stack implementation + // supports progress events during upload. + supportsProgressEvents: () => boolean } export type HttpProgressHandler = (bytesSent: number) => void diff --git a/lib/upload.ts b/lib/upload.ts index 6a0329da..b75cdcfa 100644 --- a/lib/upload.ts +++ b/lib/upload.ts @@ -3,6 +3,7 @@ import { Base64 } from 'js-base64' // provides WHATWG URL? Then we can get rid of @rollup/plugin-commonjs. import URL from 'url-parse' import { DetailedError } from './DetailedError.js' +import { StallDetector } from './StallDetector.js' import { log } from './logger.js' import { type FileSource, @@ -54,6 +55,12 @@ export const defaultOptions = { httpStack: undefined, protocol: PROTOCOL_TUS_V1 as UploadOptions['protocol'], + + stallDetection: { + enabled: false, + stallTimeout: 30000, + checkInterval: 5000, + }, } export class BaseUpload { @@ -97,6 +104,9 @@ export class BaseUpload { // The offset of the remote upload before the latest attempt was started. private _offsetBeforeRetry = 0 + // The reason for the last stall detection, if any + private _stallReason?: string + // An array of BaseUpload instances which are used for uploading the different // parts, if the parallelUploads option is used. private _parallelUploads?: BaseUpload[] @@ -109,6 +119,8 @@ export class BaseUpload { // upload options or HEAD response) private _uploadLengthDeferred: boolean + + constructor(file: UploadInput, options: UploadOptions) { // Warn about removed options from previous versions if ('resume' in options) { @@ -276,7 +288,7 @@ export class BaseUpload { * * @api private */ - private async _startParallelUpload(): Promise { + private async _startParallelUpload(): Promise { const totalSize = this._size let totalProgress = 0 this._parallelUploads = [] @@ -343,6 +355,7 @@ export class BaseUpload { if (totalSize == null) { throw new Error('tus: Expected totalSize to be set') } + this._emitProgress(totalProgress, totalSize) }, // Wait until every partial upload has an upload URL, so we can add @@ -365,6 +378,9 @@ export class BaseUpload { // @ts-expect-error `value` is unknown and not an UploadInput const upload = new BaseUpload(value, options) + + + upload.start() // Store the upload in an array, so we can later abort them if necessary. @@ -391,7 +407,9 @@ export class BaseUpload { let res: HttpResponse try { - res = await this._sendRequest(req) + // Create stall detector for final concatenation POST request + const stallDetector = this._createStallDetector() + res = await this._sendRequest(req, undefined, stallDetector) } catch (err) { if (!(err instanceof Error)) { throw new Error(`tus: value thrown that is not an error: ${err}`) @@ -622,7 +640,9 @@ export class BaseUpload { ) { req.setHeader('Upload-Complete', '?0') } - res = await this._sendRequest(req) + // Create stall detector for POST request + const stallDetector = this._createStallDetector() + res = await this._sendRequest(req, undefined, stallDetector) } } catch (err) { if (!(err instanceof Error)) { @@ -684,7 +704,9 @@ export class BaseUpload { let res: HttpResponse try { - res = await this._sendRequest(req) + // Create stall detector for HEAD request + const stallDetector = this._createStallDetector() + res = await this._sendRequest(req, undefined, stallDetector) } catch (err) { if (!(err instanceof Error)) { throw new Error(`tus: value thrown that is not an error: ${err}`) @@ -704,6 +726,11 @@ export class BaseUpload { throw new DetailedError('tus: upload is currently locked; retry later', undefined, req, res) } + // For 5xx server errors, throw error to trigger retry instead of creating new upload + if (inStatusCategory(status, 500)) { + throw new DetailedError('tus: server error during resume, retrying', undefined, req, res) + } + if (inStatusCategory(status, 400)) { // Remove stored fingerprint and corresponding endpoint, // on client errors since the file can not be found @@ -720,7 +747,7 @@ export class BaseUpload { ) } - // Try to create a new upload + // Try to create a new upload (only for 4xx client errors and 3xx redirects) this.url = null await this._createUpload() } @@ -751,6 +778,8 @@ export class BaseUpload { await this.options.onUploadUrlAvailable() } + + await this._saveUploadInUrlStorage() // Upload has already been completed and we do not need to send additional @@ -810,12 +839,15 @@ export class BaseUpload { throw new Error(`tus: value thrown that is not an error: ${err}`) } - throw new DetailedError( - `tus: failed to upload chunk at offset ${this._offset}`, - err, - req, - undefined, - ) + // Include stall reason in error message if available + const errorMessage = this._stallReason + ? `tus: failed to upload chunk at offset ${this._offset} (stalled: ${this._stallReason})` + : `tus: failed to upload chunk at offset ${this._offset}` + + // Clear the stall reason after using it + this._stallReason = undefined + + throw new DetailedError(errorMessage, err, req, undefined) } if (!inStatusCategory(res.getStatus(), 200)) { @@ -825,6 +857,34 @@ export class BaseUpload { await this._handleUploadResponse(req, res) } + /** + * Create a stall detector if stall detection is enabled and supported. + * + * @api private + */ + private _createStallDetector(): StallDetector | undefined { + if (this.options.stallDetection?.enabled) { + // Only enable stall detection if the HTTP stack supports progress events + if (this.options.httpStack.supportsProgressEvents()) { + return new StallDetector(this.options.stallDetection, (reason: string) => { + // Handle stall by aborting the current request + // The abort will cause the request to fail, which will be caught + // in _performUpload and wrapped in a DetailedError for proper retry handling + if (this._req) { + this._stallReason = reason + this._req.abort() + } + // Don't call _retryOrEmitError here - let the natural error flow handle it + }) + } else { + log( + 'tus: stall detection is enabled but the HTTP stack does not support progress events, it will be disabled for this upload', + ) + } + } + return undefined + } + /** * _addChunktoRequest reads a chunk from the source and sends it using the * supplied request object. It will not handle the response. @@ -835,7 +895,15 @@ export class BaseUpload { const start = this._offset let end = this._offset + this.options.chunkSize + // Create stall detector for this request if stall detection is enabled and supported + // but don't start it yet - we'll start it after onBeforeRequest completes + const stallDetector = this._createStallDetector() + req.setProgressHandler((bytesSent) => { + // Update per-request stall detector if active + if (stallDetector) { + stallDetector.updateProgress(start + bytesSent) + } this._emitProgress(start + bytesSent, this._size) }) @@ -884,7 +952,7 @@ export class BaseUpload { } if (value == null) { - return await this._sendRequest(req) + return await this._sendRequest(req, undefined, stallDetector) } if ( @@ -893,8 +961,9 @@ export class BaseUpload { ) { req.setHeader('Upload-Complete', done ? '?1' : '?0') } + this._emitProgress(this._offset, this._size) - return await this._sendRequest(req, value) + return await this._sendRequest(req, value, stallDetector) } /** @@ -992,8 +1061,12 @@ export class BaseUpload { * * @api private */ - _sendRequest(req: HttpRequest, body?: SliceType): Promise { - return sendRequest(req, body, this.options) + _sendRequest( + req: HttpRequest, + body?: SliceType, + stallDetector?: StallDetector, + ): Promise { + return sendRequest(req, body, this.options, stallDetector) } } @@ -1054,12 +1127,24 @@ async function sendRequest( req: HttpRequest, body: SliceType | undefined, options: UploadOptions, + stallDetector?: StallDetector, ): Promise { if (typeof options.onBeforeRequest === 'function') { await options.onBeforeRequest(req) } - const res = await req.send(body) + if (stallDetector) { + stallDetector.start() + } + + let res: HttpResponse + try { + res = await req.send(body) + } finally { + if (stallDetector) { + stallDetector.stop() + } + } if (typeof options.onAfterResponse === 'function') { await options.onAfterResponse(req, res) diff --git a/test/spec/browser-index.js b/test/spec/browser-index.js index 407b4467..a762c2fd 100644 --- a/test/spec/browser-index.js +++ b/test/spec/browser-index.js @@ -11,3 +11,4 @@ import './test-terminate.js' import './test-web-stream.js' import './test-binary-data.js' import './test-end-to-end.js' +import './test-stall-detection.js' diff --git a/test/spec/helpers/utils.js b/test/spec/helpers/utils.js index bc03d262..75599d48 100644 --- a/test/spec/helpers/utils.js +++ b/test/spec/helpers/utils.js @@ -72,6 +72,11 @@ export class TestHttpStack { this._pendingWaits.push(resolve) }) } + + supportsProgressEvents() { + // Test HTTP stack supports progress events for testing purposes + return true + } } export class TestRequest { diff --git a/test/spec/node-index.js b/test/spec/node-index.js index 4dfb1ffd..ef00eab5 100644 --- a/test/spec/node-index.js +++ b/test/spec/node-index.js @@ -5,3 +5,4 @@ import './test-terminate.js' import './test-web-stream.js' import './test-binary-data.js' import './test-end-to-end.js' +import './test-stall-detection.js' diff --git a/test/spec/test-parallel-uploads.js b/test/spec/test-parallel-uploads.js index 8408ffa9..dc7f6a83 100644 --- a/test/spec/test-parallel-uploads.js +++ b/test/spec/test-parallel-uploads.js @@ -578,5 +578,210 @@ describe('tus', () => { expect(options.onProgress).toHaveBeenCalledWith(5, 11) expect(options.onProgress).toHaveBeenCalledWith(11, 11) }) + + it('should preserve upload URL in partial uploads during retry', async () => { + const testStack = new TestHttpStack() + const file = getBlob('hello') + + const options = { + httpStack: testStack, + parallelUploads: 1, // Use single parallel to focus on one upload + retryDelays: [10], + endpoint: 'https://tus.io/uploads', + onSuccess: waitableFunction(), + headers: { 'Upload-Concat': 'partial' }, // Force partial upload behavior + storeFingerprintForResuming: false, // This is key - partial uploads don't store fingerprints + } + + const upload = new Upload(file, options) + upload.start() + + // Create partial upload + let req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + expect(req.requestHeaders['Upload-Concat']).toBe('partial') + + req.respondWith({ + status: 201, + responseHeaders: { + Location: 'https://tus.io/uploads/upload1', + }, + }) + + // PATCH request fails + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload1') + expect(req.method).toBe('PATCH') + + req.respondWith({ + status: 500, + }) + + // The key test: what happens on retry? + req = await testStack.nextRequest() + + // With the fix: should be HEAD to existing URL (resume) + expect(req.url).toBe('https://tus.io/uploads/upload1') + expect(req.method).toBe('HEAD') + + // Without the fix: would be POST to create new upload + // (because storeFingerprintForResuming: false and uploadUrl: null) + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Length': '5', + 'Upload-Offset': '0', + }, + }) + + // Resume PATCH + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload1') + expect(req.method).toBe('PATCH') + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Offset': '5', + }, + }) + + await options.onSuccess.toBeCalled() + }) + + it('should resume partial uploads on retry instead of creating fresh uploads', async () => { + const testStack = new TestHttpStack() + const file = getBlob('hello world') + + // Track all requests to detect if fresh uploads are created + const allRequests = [] + const originalCreateRequest = testStack.createRequest.bind(testStack) + testStack.createRequest = function(method, url) { + allRequests.push({ method, url }) + return originalCreateRequest(method, url) + } + + const options = { + httpStack: testStack, + parallelUploads: 2, + retryDelays: [10], + endpoint: 'https://tus.io/uploads', + onSuccess: waitableFunction(), + } + + const upload = new Upload(file, options) + upload.start() + + // First partial upload creation + let req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + expect(req.requestHeaders['Upload-Concat']).toBe('partial') + expect(req.requestHeaders['Upload-Length']).toBe('5') + + req.respondWith({ + status: 201, + responseHeaders: { + Location: 'https://tus.io/uploads/upload1', + }, + }) + + // Second partial upload creation + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + expect(req.requestHeaders['Upload-Concat']).toBe('partial') + expect(req.requestHeaders['Upload-Length']).toBe('6') + + req.respondWith({ + status: 201, + responseHeaders: { + Location: 'https://tus.io/uploads/upload2', + }, + }) + + // First PATCH request succeeds + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload1') + expect(req.method).toBe('PATCH') + expect(req.requestHeaders['Upload-Offset']).toBe('0') + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Offset': '5', + }, + }) + + // Second PATCH request fails (network error) + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload2') + expect(req.method).toBe('PATCH') + expect(req.requestHeaders['Upload-Offset']).toBe('0') + + req.respondWith({ + status: 500, + }) + + // CRITICAL TEST: After retry delay, we should NOT see another POST to /uploads + // The next request should be HEAD to the existing upload2 URL + req = await testStack.nextRequest() + + // Verify we're not creating a fresh upload (this is the key test) + expect(req.method).not.toBe('POST') + expect(req.url).toBe('https://tus.io/uploads/upload2') + expect(req.method).toBe('HEAD') + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Length': '6', + 'Upload-Offset': '0', + }, + }) + + // Resume with PATCH from current offset + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads/upload2') + expect(req.method).toBe('PATCH') + expect(req.requestHeaders['Upload-Offset']).toBe('0') + + req.respondWith({ + status: 204, + responseHeaders: { + 'Upload-Offset': '6', + }, + }) + + // Final concatenation + req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + expect(req.requestHeaders['Upload-Concat']).toBe( + 'final;https://tus.io/uploads/upload1 https://tus.io/uploads/upload2', + ) + + req.respondWith({ + status: 201, + responseHeaders: { + Location: 'https://tus.io/uploads/upload3', + }, + }) + + await options.onSuccess.toBeCalled() + + // Final verification: count how many POST requests to /uploads we made + const postToUploads = allRequests.filter(r => + r.method === 'POST' && r.url === 'https://tus.io/uploads' + ) + + // Should only be 3 POSTs: 2 partial + 1 final concatenation + // If the bug exists, we'd see 4 POSTs (an extra one from the retry) + expect(postToUploads.length).toBe(3) + }) + + }) }) diff --git a/test/spec/test-stall-detection.js b/test/spec/test-stall-detection.js new file mode 100644 index 00000000..da7632a6 --- /dev/null +++ b/test/spec/test-stall-detection.js @@ -0,0 +1,397 @@ +import { Upload } from 'tus-js-client' +import { TestHttpStack, getBlob, wait, waitableFunction } from './helpers/utils.js' + +/** + * Helper to get body size for various input types + */ +function getBodySize(body) { + if (body == null) return null + if (body instanceof Blob) return body.size + if (body.length != null) return body.length + return 0 +} + +/** + * Enhanced HTTP stack for testing stall detection scenarios + * Supports both complete stalls and custom progress sequences + */ +class StallTestHttpStack extends TestHttpStack { + constructor() { + super() + this.stallOnNextPatch = false + this.progressSequences = new Map() + this.progressPromises = new Map() + this.nextProgressSequence = null + } + + /** + * Configure the stack to stall on the next PATCH request + */ + simulateStallOnNextPatch() { + this.stallOnNextPatch = true + } + + /** + * Set a custom progress sequence for the next PATCH request + * @param {Array} sequence - Array of {bytes: number, delay: number} objects + */ + setNextProgressSequence(sequence) { + this.nextProgressSequence = sequence + } + + supportsProgressEvents() { + return true + } + + createRequest(method, url) { + const req = super.createRequest(method, url) + + if (method === 'PATCH') { + this._setupPatchRequest(req) + } + + return req + } + + _setupPatchRequest(req) { + const self = this + + // Handle complete stalls + if (this.stallOnNextPatch) { + this.stallOnNextPatch = false + req.send = async function (body) { + this.body = body + if (body) { + this.bodySize = await getBodySize(body) + // Don't call progress handler to simulate a complete stall + } + this._onRequestSend(this) + return this._requestPromise + } + return + } + + // Handle progress sequences + if (this.nextProgressSequence) { + this.progressSequences.set(req, this.nextProgressSequence) + this.nextProgressSequence = null + } + + // Override respondWith to wait for progress events + const originalRespondWith = req.respondWith.bind(req) + req.respondWith = async (resData) => { + const progressPromise = self.progressPromises.get(req) + if (progressPromise) { + await progressPromise + self.progressPromises.delete(req) + } + originalRespondWith(resData) + } + + // Override send to handle progress sequences + req.send = async function (body) { + this.body = body + if (body) { + this.bodySize = await getBodySize(body) + } + + const progressSequence = self.progressSequences.get(req) + if (progressSequence && this._onProgress) { + self._scheduleProgressSequence(req, progressSequence, this._onProgress) + } else if (this._onProgress) { + self._scheduleDefaultProgress(req, this._onProgress, this.bodySize) + } + + this._onRequestSend(this) + return this._requestPromise + } + } + + _scheduleProgressSequence(req, sequence, progressHandler) { + const progressPromise = new Promise((resolve) => { + setTimeout(async () => { + for (const event of sequence) { + await new Promise((resolve) => setTimeout(resolve, event.delay || 0)) + progressHandler(event.bytes) + } + resolve() + }, 10) // Small delay to ensure stall detector is started + }) + this.progressPromises.set(req, progressPromise) + } + + _scheduleDefaultProgress(req, progressHandler, bodySize) { + const progressPromise = new Promise((resolve) => { + setTimeout(() => { + progressHandler(0) + progressHandler(bodySize) + resolve() + }, 10) // Small delay to ensure stall detector is started + }) + this.progressPromises.set(req, progressPromise) + } +} + +/** + * Common test setup helper + */ +function createTestUpload(options = {}) { + const defaultOptions = { + httpStack: new StallTestHttpStack(), + endpoint: 'https://tus.io/uploads', + onError: waitableFunction('onError'), + onSuccess: waitableFunction('onSuccess'), + onProgress: waitableFunction('onProgress'), + } + + const file = options.file || getBlob('hello world') + const uploadOptions = { ...defaultOptions, ...options } + const upload = new Upload(file, uploadOptions) + + return { upload, options: uploadOptions, testStack: uploadOptions.httpStack } +} + +/** + * Helper to handle standard upload creation flow + */ +async function handleUploadCreation(testStack, location = '/uploads/12345') { + const req = await testStack.nextRequest() + expect(req.method).toBe('POST') + req.respondWith({ + status: 201, + responseHeaders: { + Location: location, + }, + }) + return req +} + +describe('tus-stall-detection', () => { + describe('integration tests', () => { + it("should not enable stall detection if HTTP stack doesn't support progress events", async () => { + const { enableDebugLog } = await import('tus-js-client') + enableDebugLog() + + const testStack = new TestHttpStack() + testStack.supportsProgressEvents = () => false + + const { upload } = createTestUpload({ + httpStack: testStack, + stallDetection: { enabled: true }, + }) + + // Capture console output + const originalLog = console.log + let loggedMessage = '' + console.log = (message) => { + loggedMessage += message + } + + upload.start() + + const req = await testStack.nextRequest() + expect(req.url).toBe('https://tus.io/uploads') + expect(req.method).toBe('POST') + req.respondWith({ + status: 201, + responseHeaders: { Location: '/uploads/12345' }, + }) + + await wait(50) + console.log = originalLog + + expect(loggedMessage).toContain( + 'tus: stall detection is enabled but the HTTP stack does not support progress events', + ) + + upload.abort() + }) + + it('should upload a file with stall detection enabled', async () => { + const { upload, options, testStack } = createTestUpload({ + stallDetection: { + enabled: true, + checkInterval: 1000, + stallTimeout: 2000, + }, + }) + + upload.start() + + await handleUploadCreation(testStack) + + const patchReq = await testStack.nextRequest() + expect(patchReq.url).toBe('https://tus.io/uploads/12345') + expect(patchReq.method).toBe('PATCH') + + patchReq.respondWith({ + status: 204, + responseHeaders: { 'Upload-Offset': '11' }, + }) + + await options.onSuccess.toBeCalled() + expect(options.onError.calls.count()).toBe(0) + }) + + it('should detect stalls and emit error when no retries configured', async () => { + const { upload, options, testStack } = createTestUpload({ + stallDetection: { + enabled: true, + checkInterval: 100, + stallTimeout: 200, + }, + retryDelays: null, + }) + + testStack.simulateStallOnNextPatch() + upload.start() + + await handleUploadCreation(testStack) + + const error = await options.onError.toBeCalled() + expect(error.message).toContain('stalled:') + }) + + it('should retry when stall is detected', async () => { + const { upload, options, testStack } = createTestUpload({ + stallDetection: { + enabled: true, + checkInterval: 100, + stallTimeout: 200, + }, + retryDelays: [100], + }) + + testStack.simulateStallOnNextPatch() + upload.start() + + let requestCount = 0 + while (true) { + const req = await testStack.nextRequest() + requestCount++ + + if (req.method === 'POST') { + req.respondWith({ + status: 201, + responseHeaders: { Location: '/uploads/12345' }, + }) + } else if (req.method === 'HEAD') { + req.respondWith({ + status: 200, + responseHeaders: { + 'Upload-Offset': '0', + 'Upload-Length': '11', + }, + }) + } else if (req.method === 'PATCH') { + req.respondWith({ + status: 204, + responseHeaders: { 'Upload-Offset': '11' }, + }) + break + } + + if (requestCount > 10) { + throw new Error('Too many requests') + } + } + + await options.onSuccess.toBeCalled() + expect(options.onError.calls.count()).toBe(0) + expect(requestCount).toBeGreaterThan(1) + }) + + it('should not incorrectly detect stalls during onBeforeRequest delays', async () => { + const { upload, options, testStack } = createTestUpload({ + stallDetection: { + enabled: true, + checkInterval: 100, + stallTimeout: 200, + }, + onBeforeRequest: async (_req) => { + await wait(300) // Longer than stall timeout + }, + }) + + upload.start() + + await handleUploadCreation(testStack) + + const patchReq = await testStack.nextRequest() + expect(patchReq.url).toBe('https://tus.io/uploads/12345') + expect(patchReq.method).toBe('PATCH') + + patchReq.respondWith({ + status: 204, + responseHeaders: { 'Upload-Offset': '11' }, + }) + + await options.onSuccess.toBeCalled() + expect(options.onError.calls.count()).toBe(0) + }) + + it('should detect stalls when progress events stop mid-upload', async () => { + const file = getBlob('hello world'.repeat(100)) + const { upload, options, testStack } = createTestUpload({ + file, + stallDetection: { + enabled: true, + checkInterval: 100, + stallTimeout: 200, + }, + retryDelays: null, + }) + + // Create a progress sequence that stops at 30% of the file + const fileSize = file.size + const progressSequence = [ + { bytes: 0, delay: 10 }, + { bytes: Math.floor(fileSize * 0.1), delay: 50 }, + { bytes: Math.floor(fileSize * 0.2), delay: 50 }, + { bytes: Math.floor(fileSize * 0.3), delay: 50 }, + // No more progress events after 30% + ] + + testStack.setNextProgressSequence(progressSequence) + upload.start() + await handleUploadCreation(testStack) + + const error = await options.onError.toBeCalled() + expect(error.message).toContain('stalled:') + expect(options.onProgress.calls.count()).toBeGreaterThan(0) + }) + + it('should detect stalls when progress value does not change', async () => { + const { upload, options, testStack } = createTestUpload({ + stallDetection: { + enabled: true, + checkInterval: 50, + stallTimeout: 500, + }, + retryDelays: null, + }) + + // Create a progress sequence that gets stuck at 300 bytes + const progressSequence = [ + { bytes: 0, delay: 10 }, + { bytes: 100, delay: 10 }, + { bytes: 200, delay: 10 }, + { bytes: 300, delay: 10 }, + // Repeat the same value to trigger value-based stall detection + ...Array(12).fill({ bytes: 300, delay: 30 }), + ] + + testStack.setNextProgressSequence(progressSequence) + upload.start() + + await handleUploadCreation(testStack) + + const patchReq = await testStack.nextRequest() + expect(patchReq.method).toBe('PATCH') + + const error = await options.onError.toBeCalled() + expect(error.message).toContain('stalled: no progress') + expect(options.onProgress.calls.count()).toBeGreaterThan(0) + }) + }) +})