Skip to content
Merged
2 changes: 1 addition & 1 deletion docs/agent-data-plane/configuration/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ tracking.
| -------------------------------------------- | ----------------------------------------------- | ------- |
| `dogstatsd_experimental_http.enabled` | Enable experimental HTTP/H2C DSD listener | [#1682] |
| `dogstatsd_experimental_http.listen_address` | Bind address for experimental HTTP DSD listener | [#1682] |
| `tls_handshake_timeout` | HTTP TLS handshake timeout | [#178] |

<!-- section:unsupported-not-planned -->
### Not Planned
Expand Down Expand Up @@ -801,6 +800,7 @@ Both commands scrub recognized secret values before writing JSON to standard out
| `syslog_rfc` | Use RFC-style syslog header |
| `syslog_uri` | Syslog destination URI |
| `telemetry.dogstatsd_origin` | Per-origin processed-metrics telemetry |
| `tls_handshake_timeout` | HTTP TLS handshake timeout |
| `use_proxy_for_cloud_metadata` | Proxy cloud metadata endpoints |
| `use_v2_api.series` | Send series via V2 protobuf endpoint |
| `use_v3_api.series.enabled` | Global V3 series mode |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,10 @@ impl DatadogConfigWitness for DatadogTranslator<'_> {
self.config.shared.endpoints.tls.min_tls_version = value;
}

fn consume_tls_handshake_timeout(&mut self, value: Duration) {
self.config.shared.endpoints.tls.handshake_timeout = value;
}

fn consume_multi_region_failover_api_key(&mut self, value: String) {
self.config.domains.multi_region_failover.api_key = non_empty(value);
}
Expand Down
6 changes: 6 additions & 0 deletions lib/agent-data-plane-config/src/shared.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,12 @@ pub struct Tls {

/// Path to which TLS session keys are logged, for debugging.
pub sslkeylogfile: String,

/// Timeout for completing the TLS handshake after a connection is established.
///
/// Defaults to 10 seconds. Bounds only the handshake step, distinct from the overall request timeout. A value
/// of zero disables the handshake-specific deadline, leaving the overall request timeout as the only bound.
pub handshake_timeout: Duration,
Comment thread
jszwedko marked this conversation as resolved.
}

/// Payload compression settings applied before transmission.
Expand Down
1 change: 1 addition & 0 deletions lib/datadog-agent/config-testing/build/registry_gen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ static GOLDEN_ORDER: &[(&str, &[&str])] = &[
"forwarder_storage_max_size_in_bytes",
"forwarder_storage_path",
"forwarder_outdated_file_in_days",
"tls_handshake_timeout",
],
),
(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,17 @@ crate::declare_annotations! {
test_json: None,
pipeline_affinity: PipelineAffinity::CrossCutting,
};
/// `tls_handshake_timeout`-HTTP TLS handshake timeout
TLS_HANDSHAKE_TIMEOUT = SalukiAnnotation {
schema: &schema::TLS_HANDSHAKE_TIMEOUT,
support_level: SupportLevel::Full,
additional_yaml_paths: &[],
env_var_override: None,
used_by: &[structs::FORWARDER_CONFIGURATION],
value_type_override: None,
test_json: None,
pipeline_affinity: PipelineAffinity::CrossCutting,
};
/// `forwarder_apikey_validation_interval`-API key check interval (minutes)
FORWARDER_APIKEY_VALIDATION_INTERVAL = SalukiAnnotation {
schema: &schema::FORWARDER_APIKEY_VALIDATION_INTERVAL,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,17 +60,6 @@ crate::declare_annotations! {
test_json: None,
pipeline_affinity: PipelineAffinity::CrossCutting,
};
/// `tls_handshake_timeout`-HTTP TLS handshake timeout
TLS_HANDSHAKE_TIMEOUT = SalukiAnnotation {
schema: &schema::TLS_HANDSHAKE_TIMEOUT,
support_level: SupportLevel::Incompatible(Severity::Medium),
additional_yaml_paths: &[],
env_var_override: None,
used_by: &[],
value_type_override: None,
test_json: None,
pipeline_affinity: PipelineAffinity::CrossCutting,
};
/// `aggregator_buffer_size`-Channel buffer depth for aggregator queues
AGGREGATOR_BUFFER_SIZE = SalukiAnnotation {
schema: &schema::AGGREGATOR_BUFFER_SIZE,
Expand Down
14 changes: 10 additions & 4 deletions lib/datadog-agent/config/schema/schema_overlay.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2593,12 +2593,18 @@ inventory:
config_registry_filename: dogstatsd.rs

tls_handshake_timeout:
support: none
severity: medium
planned: true
support: full
pipelines: [cross_cutting]
description: "HTTP TLS handshake timeout"
documentation: "Existing request timeout covers the gap."
documentation: |
ADP applies `tls_handshake_timeout` to the TLS handshake step of outbound Datadog intake connections,
independent of `forwarder_timeout`, which bounds the full request. ADP owns the TLS layer directly rather than
delegating the handshake to an opaque connect-and-handshake future, so it can time out the handshake without
also aborting an in-progress TCP connect.
test_support:
used_by: [ForwarderConfiguration]
additional_attributes:
config_registry_filename: forwarder.rs
issue: "#178"

use_dogstatsd:
Expand Down
41 changes: 27 additions & 14 deletions lib/datadog-agent/config/src/classifier/classifier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,9 @@ mod tests {
#[test]
fn incompatible_non_default() {

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The tests in this file needed to be updated now that tls_handshake_timeout is a supported option.

let c = classifier();
let result = c.classify("tls_handshake_timeout", &Value::Number(999.into())).unwrap();
let result = c
.classify("dogstatsd_stats_buffer", &Value::Number(999.into()))
.unwrap();
assert!(matches!(result.support_level, SupportLevel::Incompatible(_)));
assert!(!result.is_default);
}
Expand All @@ -139,8 +141,10 @@ mod tests {
#[test]
fn duration_default_null_is_not_default() {
let c = classifier();
// tls_handshake_timeout has a duration default (10s); a null value can't be normalized.
let result = c.classify("tls_handshake_timeout", &Value::Null).unwrap();
// dogstatsd_packet_buffer_flush_timeout has a duration default (100ms); a null value can't be normalized.
let result = c
.classify("dogstatsd_packet_buffer_flush_timeout", &Value::Null)
.unwrap();
assert!(!result.is_default);
}

Expand All @@ -149,14 +153,17 @@ mod tests {
let c = classifier();
// Neither an empty string nor arbitrary text parses as a duration, so neither matches.
assert!(
!c.classify("tls_handshake_timeout", &Value::String("".into()))
!c.classify("dogstatsd_packet_buffer_flush_timeout", &Value::String("".into()))
.unwrap()
.is_default
);
assert!(
!c.classify("tls_handshake_timeout", &Value::String("something".into()))
.unwrap()
.is_default
!c.classify(
"dogstatsd_packet_buffer_flush_timeout",
&Value::String("something".into())
)
.unwrap()
.is_default
);
}

Expand All @@ -165,15 +172,15 @@ mod tests {
let c = classifier();
// The default is also matched when supplied as a Go duration string rather than nanoseconds.
let result = c
.classify("tls_handshake_timeout", &Value::String("10s".into()))
.classify("dogstatsd_packet_buffer_flush_timeout", &Value::String("100ms".into()))
.unwrap();
assert!(result.is_default);
}

#[test]
fn incompatible_severity_levels() {
let c = classifier();
let result = c.classify("tls_handshake_timeout", &Value::Number(30.into())).unwrap();
let result = c.classify("dogstatsd_stats_buffer", &Value::Number(30.into())).unwrap();
assert!(matches!(
result.support_level,
SupportLevel::Incompatible(Severity::Medium)
Expand All @@ -183,20 +190,26 @@ mod tests {
#[test]
fn duration_default_as_nanoseconds_is_default() {
let c = classifier();
// The Agent transmits tls_handshake_timeout (schema default "10s") as integer nanoseconds.
// The classifier must recognize this as the default and not flag it as an override.
// The Agent transmits dogstatsd_packet_buffer_flush_timeout (schema default "100ms") as integer
// nanoseconds. The classifier must recognize this as the default and not flag it as an override.
let result = c
.classify("tls_handshake_timeout", &Value::Number(10_000_000_000i64.into()))
.classify(
"dogstatsd_packet_buffer_flush_timeout",
&Value::Number(100_000_000i64.into()),
)
.unwrap();
assert!(result.is_default);
}

#[test]
fn duration_non_default_nanoseconds_is_not_default() {
let c = classifier();
// 5s in nanoseconds is not the 10s default.
// 5ms in nanoseconds is not the 100ms default.
let result = c
.classify("tls_handshake_timeout", &Value::Number(5_000_000_000i64.into()))
.classify(
"dogstatsd_packet_buffer_flush_timeout",
&Value::Number(5_000_000i64.into()),
)
.unwrap();
assert!(!result.is_default);
}
Expand Down
7 changes: 0 additions & 7 deletions lib/datadog-agent/config/src/generated/classifier_data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -319,13 +319,6 @@ pub(crate) static CLASSIFIER_ENTRIES: &[ClassifierEntry] = &[
pipeline_affinity: PipelineAffinity::Pipelines(&[Pipeline::DogStatsD]),
default: DefaultValue::Json("[]"),
},
ClassifierEntry {
yaml_path: "tls_handshake_timeout",
aliases: &[],
support_level: SupportLevel::Incompatible(Severity::Medium),
pipeline_affinity: PipelineAffinity::CrossCutting,
default: DefaultValue::DurationNanos(10000000000),
},
ClassifierEntry {
yaml_path: "use_dogstatsd",
aliases: &[],
Expand Down
11 changes: 11 additions & 0 deletions lib/datadog-agent/config/src/generated/datadog_configuration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,13 @@ pub struct DatadogConfiguration {
#[serde(default)]
pub telemetry: Telemetry,

#[serde(
default = "duration_defaults::tls_handshake_timeout",

deserialize_with = "crate::duration_de::deserialize_go_duration"
)]
pub tls_handshake_timeout: std::time::Duration,

#[serde(default)]
pub use_proxy_for_cloud_metadata: bool,

Expand Down Expand Up @@ -561,6 +568,7 @@ impl Default for DatadogConfiguration {
syslog_rfc: Default::default(),
syslog_uri: Default::default(),
telemetry: Default::default(),
tls_handshake_timeout: duration_defaults::tls_handshake_timeout(),
use_proxy_for_cloud_metadata: Default::default(),
use_v2_api: Default::default(),
use_v3_api: Default::default(),
Expand Down Expand Up @@ -1869,4 +1877,7 @@ mod duration_defaults {
pub(super) fn expected_tags_duration() -> std::time::Duration {
std::time::Duration::from_nanos(0)
}
pub(super) fn tls_handshake_timeout() -> std::time::Duration {
std::time::Duration::from_nanos(10000000000)
}
}
5 changes: 5 additions & 0 deletions lib/datadog-agent/config/src/generated/env_keys.rs
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,11 @@ pub static DATADOG_ENV_KEYS: &[EnvKey] = &[
path: &["telemetry", "dogstatsd_origin"],
decode: EnvDecode::Bool,
},
EnvKey {
env_vars: &["DD_TLS_HANDSHAKE_TIMEOUT"],
path: &["tls_handshake_timeout"],
decode: EnvDecode::DurationString,
},
EnvKey {
env_vars: &["DD_USE_PROXY_FOR_CLOUD_METADATA"],
path: &["use_proxy_for_cloud_metadata"],
Expand Down
2 changes: 2 additions & 0 deletions lib/datadog-agent/config/src/generated/witness.rs
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ pub trait DatadogConfigWitness {
fn consume_syslog_rfc(&mut self, value: bool);
fn consume_syslog_uri(&mut self, value: String);
fn consume_telemetry_dogstatsd_origin(&mut self, value: bool);
fn consume_tls_handshake_timeout(&mut self, value: std::time::Duration);
fn consume_use_proxy_for_cloud_metadata(&mut self, value: bool);
fn consume_use_v2_api_series(&mut self, value: bool);
fn consume_use_v3_api_series_enabled(&mut self, value: String);
Expand Down Expand Up @@ -524,6 +525,7 @@ pub fn drive(config: &DatadogConfiguration, consumer: &mut impl DatadogConfigWit
consumer.consume_syslog_rfc(config.syslog_rfc.clone());
consumer.consume_syslog_uri(config.syslog_uri.clone());
consumer.consume_telemetry_dogstatsd_origin(config.telemetry.dogstatsd_origin.clone());
consumer.consume_tls_handshake_timeout(config.tls_handshake_timeout.clone());
consumer.consume_use_proxy_for_cloud_metadata(config.use_proxy_for_cloud_metadata.clone());
consumer.consume_use_v2_api_series(config.use_v2_api.series.clone());
consumer.consume_use_v3_api_series_enabled(config.use_v3_api.series.enabled.clone());
Expand Down
20 changes: 19 additions & 1 deletion lib/saluki-components/src/common/datadog/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::time::Duration;

use agent_data_plane_config::shared::{Endpoints, MetricsEncoding};
use facet::Facet;
use saluki_config::GenericConfiguration;
use saluki_config::{DurationString, GenericConfiguration};
use saluki_error::GenericError;
use saluki_io::net::client::http::{HttpProtocol, TlsMinimumVersion};
use serde::Deserialize;
Expand Down Expand Up @@ -53,6 +53,10 @@ fn default_min_tls_version() -> String {
MIN_TLS_VERSION_TLS12.to_string()
}

fn default_tls_handshake_timeout() -> DurationString {
DurationString::new(Duration::from_secs(10))
}

fn min_tls_version_from_config_value(value: &str) -> TlsMinimumVersion {
let trimmed = value.trim();
match trimmed.to_lowercase().as_str() {
Expand Down Expand Up @@ -350,6 +354,15 @@ pub struct ForwarderConfiguration {
#[facet(opaque)]
parsed_min_tls_version: TlsMinimumVersion,

/// Timeout for completing the TLS handshake after a connection is established, for Datadog intake forwarding.
///
/// Defaults to 10 seconds. This bounds only the TLS handshake step, distinct from `forwarder_timeout`, which
/// bounds the entire request. A value of `0` disables the handshake deadline entirely, matching the core Agent
/// convention for this setting.
#[serde(default = "default_tls_handshake_timeout")]
#[facet(opaque)]
tls_handshake_timeout: DurationString,

/// Whether to signal that the backend should allow arbitrary tag values.
///
/// Defaults to `false`. If set to `true`, the Datadog forwarder adds `Allow-Arbitrary-Tag-Value: true` to every
Expand Down Expand Up @@ -436,6 +449,11 @@ impl ForwarderConfiguration {
Duration::from_secs(self.request_timeout_secs)
}

/// Returns the TLS handshake timeout.
pub const fn tls_handshake_timeout(&self) -> Duration {
self.tls_handshake_timeout.as_duration()
}

/// Returns the maximum number of pending requests for an individual endpoint.
pub const fn endpoint_buffer_size(&self) -> usize {
self.endpoint_buffer_size
Expand Down
1 change: 1 addition & 0 deletions lib/saluki-components/src/common/datadog/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ where
.with_request_timeout(config.request_timeout())
.with_max_idle_conns_per_host(config.max_idle_connections_per_host())
.with_min_tls_version(config.min_tls_version())
.with_tls_handshake_timeout(config.tls_handshake_timeout())

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Apply the handshake timeout after proxy CONNECT

When an HTTPS intake is reached through a configured proxy, hyper_http_proxy::ProxyConnector calls this connector only for the proxy transport and performs the destination TLS handshake itself after CONNECT. Consequently, the configured timeout here never bounds the intake handshake (for an HTTPS proxy it only bounds the proxy-side handshake), so a stalled destination handshake lasts until the overall request timeout instead of tls_handshake_timeout; the timeout must also wrap the proxy connector's post-CONNECT TLS step.

Useful? React with 👍 / 👎.

@jszwedko jszwedko Aug 11, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Claude Sonnet 5] This is a known, pre-existing limitation, not something introduced by this PR — hyper_http_proxy::ProxyConnector performs its own TLS handshake after CONNECT for HTTPS-through-proxy, independent of this connector. tls_handshake_timeout can't bound that step today. Leaving this out of scope for this PR; it's already called out as a known limitation in the PR description.

I'll follow up to add support there.

.with_http_protocol(config.http_protocol())
.with_bytes_sent_counter(telemetry.bytes_sent().clone())
.with_endpoint_telemetry(
Expand Down
48 changes: 48 additions & 0 deletions lib/saluki-io/src/net/client/http/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,17 @@ impl HttpClientBuilder {
self
}

/// Sets the timeout for completing the TLS handshake after a connection is established.
///
/// This bounds only the TLS handshake step, distinct from the connect timeout, which bounds the underlying TCP
/// (or other transport) connection setup that precedes it.
///
/// Defaults to 10 seconds.
pub fn with_tls_handshake_timeout(mut self, timeout: Duration) -> Self {
self.connector_builder = self.connector_builder.with_tls_handshake_timeout(timeout);
self
}

/// Sets the per-request timeout.
///
/// The request timeout applies to each individual request made to the remote host, including each request made when
Expand Down Expand Up @@ -467,6 +478,43 @@ mod tests {
assert_eq!(negotiated_alpn, None);
}

#[tokio::test]
async fn tls_handshake_timeout_fires_against_a_stalled_server() {
initialize_crypto_provider();
let server_cert = SelfSignedCert::localhost();

let listener = TcpListener::bind("127.0.0.1:0").await.expect("should bind listener");
let port = listener.local_addr().expect("should have local address").port();
let server_task = tokio::spawn(async move {
let (stream, _) = listener.accept().await.expect("server should accept a connection");
// Accept the TCP connection but never complete the TLS handshake, so the client's handshake
// deadline is the only thing that can end the connection attempt.
tokio::time::sleep(Duration::from_secs(30)).await;
drop(stream);
});

let mut client = HttpClient::builder()
.with_tls_config(|builder| builder.with_root_cert_store(root_store(&server_cert)))
.with_tls_handshake_timeout(Duration::from_millis(200))
.with_http_protocol(HttpProtocol::Http1)
.build()
.expect("client should build");
let request = Request::get(format!("https://localhost:{port}/"))
.body(Empty::<Bytes>::new())
.expect("request should build");

let error = timeout(Duration::from_secs(5), client.send(request))
.await
.expect("request should not hit the outer test timeout")
.expect_err("handshake should time out before completing");
assert!(
format!("{error:#}").contains("TLS handshake timed out"),
"expected a TLS handshake timeout, got: {error:#}"
);

server_task.abort();
}

fn mutual_tls_configs() -> (ServerConfig, ClientConfig) {
let server_cert = SelfSignedCert::localhost();
let client_cert = SelfSignedCert::new(["saluki-client"]);
Expand Down
Loading