Skip to content

Commit f295979

Browse files
fix: improve S3 error message for response body decoding failures
When the S3 connector encounters a 'error decoding response body' error, provide a more descriptive error message suggesting the user increase the client_timeout parameter or reduce concurrent dataset loads. Closes spiceai#5980
1 parent c6b06dc commit f295979

1 file changed

Lines changed: 62 additions & 0 deletions

File tree

  • crates/runtime/src/dataconnector

crates/runtime/src/dataconnector/s3.rs

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,20 @@ impl ListingTableConnector for S3 {
363363
connector_component: ConnectorComponent::from(dataset),
364364
source: err.into(),
365365
}
366+
} else if source.to_string().contains("error decoding response body") {
367+
let timeout = self
368+
.params
369+
.get("client_timeout")
370+
.expose()
371+
.ok()
372+
.unwrap_or("default");
373+
DataConnectorError::UnableToConnectInternal {
374+
dataconnector: format!("{self}"),
375+
connector_component: ConnectorComponent::from(dataset),
376+
source: format!(
377+
"S3 request failed while decoding the response body. This may be caused by a network timeout or IO starvation when loading many datasets concurrently. Consider increasing the `client_timeout` parameter (current: {timeout}) or reducing concurrent dataset loads. {source}"
378+
).into(),
379+
}
366380
} else {
367381
DataConnectorError::UnableToConnectInternal {
368382
dataconnector: format!("{self}"),
@@ -422,6 +436,54 @@ mod tests {
422436
.expect("dataset should be built")
423437
}
424438

439+
#[tokio::test]
440+
async fn test_handle_object_store_error_decoding_response_body() {
441+
let params = create_test_parameters(vec![]).await;
442+
let connector = create_test_connector(params);
443+
let dataset = create_test_dataset("s3://spiceai-public-datasets/taxi_small_samples/").await;
444+
445+
let error = object_store::Error::Generic {
446+
store: "S3",
447+
source: "error decoding response body".into(),
448+
};
449+
450+
let result = connector.handle_object_store_error(&dataset, error);
451+
let error_str = result.to_string();
452+
453+
assert!(
454+
error_str.contains("client_timeout"),
455+
"Error message should mention client_timeout, got: {error_str}"
456+
);
457+
assert!(
458+
error_str.contains("IO starvation"),
459+
"Error message should mention IO starvation, got: {error_str}"
460+
);
461+
}
462+
463+
#[tokio::test]
464+
async fn test_handle_object_store_error_decoding_response_body_with_timeout() {
465+
let params = create_test_parameters(vec![(
466+
"client_timeout".to_string(),
467+
"120s".to_string().into(),
468+
)])
469+
.await;
470+
let connector = create_test_connector(params);
471+
let dataset = create_test_dataset("s3://spiceai-public-datasets/taxi_small_samples/").await;
472+
473+
let error = object_store::Error::Generic {
474+
store: "S3",
475+
source: "error decoding response body".into(),
476+
};
477+
478+
let result = connector.handle_object_store_error(&dataset, error);
479+
let error_str = result.to_string();
480+
481+
assert!(
482+
error_str.contains("120s"),
483+
"Error message should show current timeout value, got: {error_str}"
484+
);
485+
}
486+
425487
#[tokio::test]
426488
async fn test_url_style_not_set_omits_fragment() {
427489
let params = create_test_parameters(vec![]).await;

0 commit comments

Comments
 (0)