Skip to content

Strip Transfer-Encoding when Content-Length is present in HTTPEncoder #3583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fabianfett merged 4 commits into from
May 6, 2026
Merged

Strip Transfer-Encoding when Content-Length is present in HTTPEncoder #3583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0070404
Strip Transfer-Encoding when Content-Length is present in HTTPEncoder
fabianfett Apr 30, 2026
51971d4
Update thresholds
fabianfett Apr 30, 2026
2b65370
Merge branch 'main' into ff-fix-content-length-and-transfer-encoding
fabianfett May 6, 2026
5983811
The test doesn't crash anymore. Instead the transfer-encoding header …
fabianfett May 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions IntegrationTests/tests_04_performance/Thresholds/6.1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
"1000_copying_circularbuffer_to_array": 1050,
"1000_getHandlers": 8050,
"1000_getHandlers_sync": 37,
"1000_reqs_1_conn": 26350,
"1000_reqs_1_conn": 27400,
Copy link
Copy Markdown
Member Author

@fabianfett fabianfett Apr 30, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks like we are running into a cow now.

"1000_rst_connections": 138050,
"1000_tcpbootstraps": 3050,
"1000_tcpconnections": 146050,
"1000_udp_reqs": 6050,
"1000_udpbootstraps": 2050,
"1000_udpconnections": 75050,
"1_reqs_1000_conn": 375050,
"1_reqs_1000_conn": 377050,
"assume_isolated_scheduling_10000_executions": 88,
"bytebuffer_lots_of_rw": 2050,
"creating_10000_headers": 0,
Expand Down
4 changes: 2 additions & 2 deletions IntegrationTests/tests_04_performance/Thresholds/6.2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
"1000_copying_circularbuffer_to_array": 1050,
"1000_getHandlers": 8050,
"1000_getHandlers_sync": 37,
"1000_reqs_1_conn": 26350,
"1000_reqs_1_conn": 27400,
"1000_rst_connections": 138050,
"1000_tcpbootstraps": 3050,
"1000_tcpconnections": 146050,
"1000_udp_reqs": 6050,
"1000_udpbootstraps": 2050,
"1000_udpconnections": 75050,
"1_reqs_1000_conn": 375050,
"1_reqs_1000_conn": 377050,
"assume_isolated_scheduling_10000_executions": 88,
"bytebuffer_lots_of_rw": 2050,
"creating_10000_headers": 0,
Expand Down
4 changes: 2 additions & 2 deletions IntegrationTests/tests_04_performance/Thresholds/6.3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
"1000_copying_circularbuffer_to_array": 1050,
"1000_getHandlers": 8050,
"1000_getHandlers_sync": 37,
"1000_reqs_1_conn": 26350,
"1000_reqs_1_conn": 27350,
"1000_rst_connections": 138050,
"1000_tcpbootstraps": 3050,
"1000_tcpconnections": 146050,
"1000_udp_reqs": 6050,
"1000_udpbootstraps": 2050,
"1000_udpconnections": 75050,
"1_reqs_1000_conn": 375050,
"1_reqs_1000_conn": 377050,
"assume_isolated_scheduling_10000_executions": 88,
"bytebuffer_lots_of_rw": 2050,
"creating_10000_headers": 0,
Expand Down
106 changes: 53 additions & 53 deletions IntegrationTests/tests_04_performance/Thresholds/nightly-main.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,59 +1,59 @@
{
"10000000_asyncsequenceproducer": 21,
"1000000_asyncwriter": 1000050,
"1000_addHandlers": 46050,
"1000_addHandlers_sync": 39050,
"1000_addRemoveHandlers_handlercontext": 8050,
"1000_addRemoveHandlers_handlername": 8050,
"1000_addRemoveHandlers_handlertype": 8050,
"1000_autoReadGetAndSet": 18050,
"10000000_asyncsequenceproducer": 0,
"1000000_asyncwriter": 0,
"1000_addHandlers": 0,
"1000_addHandlers_sync": 0,
"1000_addRemoveHandlers_handlercontext": 0,
"1000_addRemoveHandlers_handlername": 0,
"1000_addRemoveHandlers_handlertype": 0,
"1000_autoReadGetAndSet": 0,
"1000_autoReadGetAndSet_sync": 0,
"1000_copying_bytebufferview_to_array": 1050,
"1000_copying_circularbuffer_to_array": 1050,
"1000_getHandlers": 8050,
"1000_getHandlers_sync": 37,
"1000_reqs_1_conn": 26350,
"1000_rst_connections": 136050,
"1000_tcpbootstraps": 3050,
"1000_tcpconnections": 144050,
"1000_udp_reqs": 6050,
"1000_udpbootstraps": 2050,
"1000_udpconnections": 74050,
"1_reqs_1000_conn": 372050,
"assume_isolated_scheduling_10000_executions": 88,
"bytebuffer_lots_of_rw": 2050,
"1000_copying_bytebufferview_to_array": 0,
"1000_copying_circularbuffer_to_array": 0,
"1000_getHandlers": 0,
"1000_getHandlers_sync": 0,
"1000_reqs_1_conn": 3050,
"1000_rst_connections": 2050,
"1000_tcpbootstraps": 0,
"1000_tcpconnections": 4050,
"1000_udp_reqs": 1050,
"1000_udpbootstraps": 0,
"1000_udpconnections": 1,
"1_reqs_1000_conn": 10050,
"assume_isolated_scheduling_10000_executions": 0,
"bytebuffer_lots_of_rw": 1,
"creating_10000_headers": 0,
"decode_1000_ws_frames": 2050,
"encode_1000_ws_frames_holding_buffer": 3,
"encode_1000_ws_frames_holding_buffer_with_mask": 2050,
"encode_1000_ws_frames_holding_buffer_with_space": 3,
"encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
"encode_1000_ws_frames_new_buffer": 3050,
"encode_1000_ws_frames_new_buffer_with_mask": 5050,
"encode_1000_ws_frames_new_buffer_with_space": 3050,
"encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
"decode_1000_ws_frames": 0,
"encode_1000_ws_frames_holding_buffer": 1,
"encode_1000_ws_frames_holding_buffer_with_mask": 1050,
"encode_1000_ws_frames_holding_buffer_with_space": 1,
"encode_1000_ws_frames_holding_buffer_with_space_with_mask": 1050,
"encode_1000_ws_frames_new_buffer": 1050,
"encode_1000_ws_frames_new_buffer_with_mask": 2050,
"encode_1000_ws_frames_new_buffer_with_space": 1050,
"encode_1000_ws_frames_new_buffer_with_space_with_mask": 2050,
"execute_hop_10000_tasks": 0,
"flat_schedule_10000_tasks": 100100,
"flat_schedule_assume_isolated_10000_tasks": 80100,
"future_assume_isolated_lots_of_callbacks": 68050,
"future_erase_result": 4050,
"future_lots_of_callbacks": 68050,
"get_100000_headers_canonical_form": 500050,
"get_100000_headers_canonical_form_trimming_whitespace": 500050,
"get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 500050,
"get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 500050,
"flat_schedule_10000_tasks": 0,
"flat_schedule_assume_isolated_10000_tasks": 0,
"future_assume_isolated_lots_of_callbacks": 0,
"future_erase_result": 0,
"future_lots_of_callbacks": 0,
"get_100000_headers_canonical_form": 0,
"get_100000_headers_canonical_form_trimming_whitespace": 0,
"get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 0,
"get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 0,
"modifying_1000_circular_buffer_elements": 0,
"modifying_byte_buffer_view": 6050,
"ping_pong_1000_reqs_1_conn": 302,
"read_10000_chunks_from_file": 110050,
"schedule_10000_tasks": 40100,
"schedule_and_run_10000_tasks": 50050,
"schedule_assume_isolated_10000_tasks": 40100,
"schedule_with_deadline_10000_tasks": 40100,
"schedule_with_deadline_assume_isolated_10000_tasks": 40100,
"scheduling_10000_executions": 88,
"submit_10000_tasks": 20100,
"submit_assume_isolated_10000_tasks": 20100,
"udp_1000_reqs_1_conn": 6200,
"udp_1_reqs_1000_conn": 160050
"modifying_byte_buffer_view": 3050,
"ping_pong_1000_reqs_1_conn": 6,
"read_10000_chunks_from_file": 10050,
"schedule_10000_tasks": 0,
"schedule_and_run_10000_tasks": 0,
"schedule_assume_isolated_10000_tasks": 0,
"schedule_with_deadline_10000_tasks": 0,
"schedule_with_deadline_assume_isolated_10000_tasks": 0,
"scheduling_10000_executions": 0,
"submit_10000_tasks": 0,
"submit_assume_isolated_10000_tasks": 0,
"udp_1000_reqs_1_conn": 1050,
"udp_1_reqs_1000_conn": 7050
}
6 changes: 3 additions & 3 deletions IntegrationTests/tests_04_performance/Thresholds/nightly-next.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"10000000_asyncsequenceproducer": 21,
"10000000_asyncsequenceproducer": 18,
"1000000_asyncwriter": 1000050,
"1000_addHandlers": 46050,
"1000_addHandlers_sync": 39050,
Expand All @@ -12,14 +12,14 @@
"1000_copying_circularbuffer_to_array": 1050,
"1000_getHandlers": 8050,
"1000_getHandlers_sync": 37,
"1000_reqs_1_conn": 26350,
"1000_reqs_1_conn": 27400,
"1000_rst_connections": 138050,
"1000_tcpbootstraps": 3050,
"1000_tcpconnections": 146050,
"1000_udp_reqs": 6050,
"1000_udpbootstraps": 2050,
"1000_udpconnections": 75050,
"1_reqs_1000_conn": 375050,
"1_reqs_1000_conn": 377050,
"assume_isolated_scheduling_10000_executions": 88,
"bytebuffer_lots_of_rw": 2050,
"creating_10000_headers": 0,
Expand Down
1 change: 0 additions & 1 deletion Sources/NIOCrashTester/CrashTestSuites.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ func makeCrashTestSuites() -> [String: Any] {
"EventLoopCrashTests": EventLoopCrashTests(),
"ByteBufferCrashTests": ByteBufferCrashTests(),
"SystemCrashTests": SystemCrashTests(),
"HTTPCrashTests": HTTPCrashTests(),
"StrictCrashTests": StrictCrashTests(),
"LoopBoundTests": LoopBoundTests(),
]
Expand Down
61 changes: 0 additions & 61 deletions Sources/NIOCrashTester/CrashTests+HTTP.swift
View file
Open in desktop

This file was deleted.

13 changes: 2 additions & 11 deletions Sources/NIOHTTP1/HTTPEncoder.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ private func correctlyFrameTransportHeaders(
return .neither
case .yes:
if headers.contains(name: "content-length") {
headers.remove(name: "transfer-encoding")
Copy link
Copy Markdown
Member

@weissi weissi May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why are we trying to fix up illegal HTTP? Originally (as shown by the crash tests) the idea was that this is "programmer error". Kinda defensible but also maybe not super helpful for proxy authors.

But now we're trying to fix up broken HTTP? Why don't we reject using an error?

Copy link
Copy Markdown
Contributor

@Lukasa Lukasa May 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well the crash tests revealed a bigger problem: they only crashed in debug mode. So we definitely couldn't let it go by uninterrupted.

In this particular case, it's cheap enough for us to just fix it. We could have taken the choice to error, that would have been equally valid.

return .contentLength
}
if version.major == 1 && version.minor >= 1 {
Expand All @@ -143,6 +144,7 @@ private func correctlyFrameTransportHeaders(
}
case .unlikely:
if headers.contains(name: "content-length") {
headers.remove(name: "transfer-encoding")
return .contentLength
}
if version.major == 1 && version.minor >= 1 {
Expand Down Expand Up @@ -204,11 +206,6 @@ public final class HTTPRequestEncoder: ChannelOutboundHandler, RemovableChannelH
public func write(context: ChannelHandlerContext, data: NIOAny, promise: EventLoopPromise<Void>?) {
switch HTTPRequestEncoder.unwrapOutboundIn(data) {
case .head(var request):
assert(
!(request.headers.contains(name: "content-length")
&& request.headers[canonicalForm: "transfer-encoding"].contains("chunked"[...])),
"illegal HTTP sent: \(request) contains both a content-length and transfer-encoding:chunked"
)
if self.configuration.automaticallySetFramingHeaders {
self.isChunked =
correctlyFrameTransportHeaders(
Expand Down Expand Up @@ -294,12 +291,6 @@ public final class HTTPResponseEncoder: ChannelOutboundHandler, RemovableChannel
public func write(context: ChannelHandlerContext, data: NIOAny, promise: EventLoopPromise<Void>?) {
switch HTTPResponseEncoder.unwrapOutboundIn(data) {
case .head(var response):
assert(
!(response.headers.contains(name: "content-length")
&& response.headers[canonicalForm: "transfer-encoding"].contains("chunked"[...])),
"illegal HTTP sent: \(response) contains both a content-length and transfer-encoding:chunked"
)

if self.configuration.automaticallySetFramingHeaders {
self.isChunked =
correctlyFrameTransportHeaders(
Expand Down
18 changes: 18 additions & 0 deletions Tests/NIOHTTP1Tests/HTTPRequestEncoderTest.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,24 @@ class HTTPRequestEncoderTests: XCTestCase {
XCTAssertEqual(trailerBuffer, ByteBuffer(string: "0\r\n\r\n"))
}

func testConflictingFramingHeadersStripsTransferEncoding() throws {
let headers = HTTPHeaders([
("transfer-encoding", "chunked"),
("content-length", "3"),
])
let writtenData = try sendRequest(withMethod: .POST, andHeaders: headers)
writtenData.assertContainsOnly("POST /uri HTTP/1.1\r\ncontent-length: 3\r\n\r\n")
}

func testConflictingFramingHeadersStripsTransferEncoding_unlikely() throws {
let headers = HTTPHeaders([
("transfer-encoding", "chunked"),
("content-length", "0"),
])
let writtenData = try sendRequest(withMethod: .GET, andHeaders: headers)
writtenData.assertContainsOnly("GET /uri HTTP/1.1\r\ncontent-length: 0\r\n\r\n")
}

private func assertOutboundContainsOnly(_ channel: EmbeddedChannel, _ expected: String) {
XCTAssertNoThrow(
XCTAssertNotNil(
Expand Down
9 changes: 9 additions & 0 deletions Tests/NIOHTTP1Tests/HTTPResponseEncoderTest.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,4 +357,13 @@ class HTTPResponseEncoderTests: XCTestCase {
}
XCTAssertEqual(trailerBuffer, ByteBuffer(string: "0\r\n\r\n"))
}

func testConflictingFramingHeadersStripsTransferEncoding() throws {
let headers = HTTPHeaders([
("transfer-encoding", "chunked"),
("content-length", "3"),
])
let writtenData = sendResponse(withStatus: .ok, andHeaders: headers)
writtenData.assertContainsOnly("HTTP/1.1 200 OK\r\ncontent-length: 3\r\n\r\n")
}
}
Loading