Skip to content

Increase HTTP header size limit in Netty-based HttpClients #45291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Increase HTTP header size limit in Netty-based HttpClients #45291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4971114
Update Netty to support huge headers, add tests
alzimmermsft May 12, 2025
368eaaa
Merge branch 'main' into AzHttp_IncreaseMaxHeaderSizeDefault
alzimmermsft May 12, 2025
5ba5233
Add sample to azure-core-http-netty and vertx showing how to override…
alzimmermsft May 12, 2025
ae3cc96
Add Content-Length to response
alzimmermsft May 12, 2025
ad0580c
Merge branch 'main' into AzHttp_IncreaseMaxHeaderSizeDefault
alzimmermsft May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 41 additions & 27 deletions sdk/clientcore/core/src/test/java/io/clientcore/core/shared/HttpClientTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,21 @@
import java.util.stream.Stream;

import static io.clientcore.core.implementation.utils.ImplUtils.bomAwareToString;
import static io.clientcore.core.shared.HttpClientTestsServer.BOM_WITH_DIFFERENT_HEADER;
import static io.clientcore.core.shared.HttpClientTestsServer.ECHO_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.HEADER_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.HUGE_HEADER_NAME;
import static io.clientcore.core.shared.HttpClientTestsServer.HUGE_HEADER_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.HUGE_HEADER_VALUE;
import static io.clientcore.core.shared.HttpClientTestsServer.INVALID_HEADER_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.PLAIN_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.RETURN_BYTES;
import static io.clientcore.core.shared.HttpClientTestsServer.SSE_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.UTF_16BE_BOM_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.UTF_16LE_BOM_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.UTF_32BE_BOM_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.UTF_32LE_BOM_RESPONSE;
import static io.clientcore.core.shared.HttpClientTestsServer.UTF_8_BOM_RESPONSE;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
Expand All @@ -69,19 +84,7 @@
*/
@Execution(ExecutionMode.SAME_THREAD)
public abstract class HttpClientTests {
private static final byte[] EXPECTED_RETURN_BYTES = "Hello World!".getBytes(StandardCharsets.UTF_8);
private static final ClientLogger LOGGER = new ClientLogger(HttpClientTests.class);
private static final String PLAIN_RESPONSE = "plainBytesNoHeader";
private static final String HEADER_RESPONSE = "plainBytesWithHeader";
private static final String INVALID_HEADER_RESPONSE = "plainBytesInvalidHeader";
private static final String UTF_8_BOM_RESPONSE = "utf8BomBytes";
private static final String UTF_16BE_BOM_RESPONSE = "utf16BeBomBytes";
private static final String UTF_16LE_BOM_RESPONSE = "utf16LeBomBytes";
private static final String UTF_32BE_BOM_RESPONSE = "utf32BeBomBytes";
private static final String UTF_32LE_BOM_RESPONSE = "utf32LeBomBytes";
private static final String BOM_WITH_DIFFERENT_HEADER = "bomBytesWithDifferentHeader";

protected static final String ECHO_RESPONSE = "echo";

/**
* Get the HTTP client that will be used for each test. This will be called once per test.
Expand Down Expand Up @@ -127,7 +130,7 @@ private String getRequestScheme() {
*/
@Test
public void plainResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_8);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_8);

assertEquals(expected, new String(sendRequest(PLAIN_RESPONSE), StandardCharsets.UTF_8));
}
Expand All @@ -137,7 +140,7 @@ public void plainResponse() {
*/
@Test
public void headerResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_16BE);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_16BE);

assertEquals(expected, new String(sendRequest(HEADER_RESPONSE), StandardCharsets.UTF_16BE));
}
Expand All @@ -147,7 +150,7 @@ public void headerResponse() {
*/
@Test
public void invalidHeaderResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_8);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_8);

assertEquals(expected, new String(sendRequest(INVALID_HEADER_RESPONSE), StandardCharsets.UTF_8));
}
Expand All @@ -157,7 +160,7 @@ public void invalidHeaderResponse() {
*/
@Test
public void utf8BomResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_8);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_8);
byte[] response = sendRequest(UTF_8_BOM_RESPONSE);

assertEquals(expected, bomAwareToString(response, 0, response.length, null));
Expand All @@ -168,7 +171,7 @@ public void utf8BomResponse() {
*/
@Test
public void utf16BeBomResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_16BE);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_16BE);
byte[] response = sendRequest(UTF_16BE_BOM_RESPONSE);

assertEquals(expected, bomAwareToString(response, 0, response.length, null));
Expand All @@ -179,7 +182,7 @@ public void utf16BeBomResponse() {
*/
@Test
public void utf16LeBomResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_16LE);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_16LE);
byte[] response = sendRequest(UTF_16LE_BOM_RESPONSE);

assertEquals(expected, bomAwareToString(response, 0, response.length, null));
Expand All @@ -190,7 +193,7 @@ public void utf16LeBomResponse() {
*/
@Test
public void utf32BeBomResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, Charset.forName("UTF-32BE"));
String expected = new String(RETURN_BYTES, Charset.forName("UTF-32BE"));

assertEquals(expected, new String(sendRequest(UTF_32BE_BOM_RESPONSE), Charset.forName("UTF-32BE")));
}
Expand All @@ -200,7 +203,7 @@ public void utf32BeBomResponse() {
*/
@Test
public void utf32LeBomResponse() {
String expected = new String(EXPECTED_RETURN_BYTES, Charset.forName("UTF-32LE"));
String expected = new String(RETURN_BYTES, Charset.forName("UTF-32LE"));

assertEquals(expected, new String(sendRequest(UTF_32LE_BOM_RESPONSE), Charset.forName("UTF-32LE")));
}
Expand All @@ -210,7 +213,7 @@ public void utf32LeBomResponse() {
*/
@Test
public void bomWithSameHeader() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_8);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_8);
byte[] response = sendRequest(BOM_WITH_DIFFERENT_HEADER);

assertEquals(expected, bomAwareToString(response, 0, response.length, "charset=utf-8"));
Expand All @@ -221,7 +224,7 @@ public void bomWithSameHeader() {
*/
@Test
public void bomWithDifferentHeader() {
String expected = new String(EXPECTED_RETURN_BYTES, StandardCharsets.UTF_8);
String expected = new String(RETURN_BYTES, StandardCharsets.UTF_8);
byte[] response = sendRequest(BOM_WITH_DIFFERENT_HEADER);

assertEquals(expected, bomAwareToString(response, 0, response.length, "charset=utf-16"));
Expand Down Expand Up @@ -711,7 +714,7 @@ private void sendRequestAndConsumeHttpBinJson(HttpRequest request,

@Test
public void canReceiveServerSentEvents() {
String uri = UriBuilder.parse(getRequestUri()).setPath("serversentevent").toString();
String uri = UriBuilder.parse(getRequestUri()).setPath(SSE_RESPONSE).toString();
final int[] i = { 0 };
ServerSentEventListener serverSentEventListener = sse -> {
String expected;
Expand Down Expand Up @@ -747,7 +750,7 @@ public void canReceiveServerSentEvents() {
@Test
public void canRecognizeServerSentEvent() {
List<String> expected = Arrays.asList("YHOO", "+2", "10");
String uri = UriBuilder.parse(getRequestUri()).setPath("serversentevent").toString();
String uri = UriBuilder.parse(getRequestUri()).setPath(SSE_RESPONSE).toString();
HttpHeaders headers = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, ContentType.APPLICATION_OCTET_STREAM);
ServerSentEventListener serverSentEventListener = sse -> assertEquals(expected, sse.getData());

Expand All @@ -764,7 +767,7 @@ public void canRecognizeServerSentEvent() {

@Test
public void onErrorServerSentEvents() throws IOException {
String uri = UriBuilder.parse(getRequestUri()).setPath("serversentevent").toString();
String uri = UriBuilder.parse(getRequestUri()).setPath(SSE_RESPONSE).toString();
final int[] i = { 0 };
ServerSentEventListener serverSentEventListener = new ServerSentEventListener() {
@Override
Expand All @@ -788,7 +791,7 @@ public void onError(Throwable throwable) {

@Test
public void onRetryWithLastEventIdReceiveServerSentEvents() {
String uri = UriBuilder.parse(getRequestUri()).setPath("serversentevent").toString();
String uri = UriBuilder.parse(getRequestUri()).setPath(SSE_RESPONSE).toString();
final int[] i = { 0 };
ServerSentEventListener serverSentEventListener = sse -> {
i[0]++;
Expand Down Expand Up @@ -820,7 +823,7 @@ public void onRetryWithLastEventIdReceiveServerSentEvents() {
public void throwsExceptionForNoListener() {
BinaryData body = BinaryData.fromString("test body");
HttpHeaders headers = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, ContentType.APPLICATION_OCTET_STREAM);
String uri = UriBuilder.parse(getRequestUri()).setPath("serversentevent").toString();
String uri = UriBuilder.parse(getRequestUri()).setPath(SSE_RESPONSE).toString();

assertThrows(RuntimeException.class,
() -> getHttpClient().send(new HttpRequest().setMethod(HttpMethod.PUT)
Expand All @@ -830,6 +833,17 @@ public void throwsExceptionForNoListener() {
.setServerSentEventListener(null)).close());
}

@Test
public void testHugeHeader() {
try (Response<BinaryData> response = getHttpClient()
.send(new HttpRequest().setMethod(HttpMethod.GET).setUri(getRequestUri(HUGE_HEADER_RESPONSE)))) {
String hugeHeaderValue = response.getHeaders().getValue(HUGE_HEADER_NAME);
assertNotNull(hugeHeaderValue, "Huge header value is null.");
assertEquals(HUGE_HEADER_VALUE, hugeHeaderValue, () -> "Huge header value didn't match what was expected. "
+ "Actual length: " + hugeHeaderValue.length() + " Expected length: " + HUGE_HEADER_VALUE.length());
}
}

// Helpers
private static void assertMatchWithHttpOrHttps(String uri1, String uri2) {
final String s1 = "http://" + uri1;
Expand Down
92 changes: 65 additions & 27 deletions sdk/clientcore/core/src/test/java/io/clientcore/core/shared/HttpClientTestsServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package io.clientcore.core.shared;

import io.clientcore.core.http.client.HttpClient;
import io.clientcore.core.http.models.HttpHeaderName;
import io.clientcore.core.implementation.http.ContentType;
import io.clientcore.core.serialization.json.JsonSerializer;
import io.clientcore.core.utils.DateTimeRfc1123;
Expand Down Expand Up @@ -36,26 +37,39 @@
@Execution(ExecutionMode.SAME_THREAD)
public class HttpClientTestsServer {
private static final JsonSerializer SERIALIZER = new JsonSerializer();
private static final String PLAIN_RESPONSE = "/plainBytesNoHeader";
private static final String HEADER_RESPONSE = "/plainBytesWithHeader";
private static final String INVALID_HEADER_RESPONSE = "/plainBytesInvalidHeader";
private static final String UTF_8_BOM_RESPONSE = "/utf8BomBytes";
private static final String UTF_16BE_BOM_RESPONSE = "/utf16BeBomBytes";
private static final String UTF_16LE_BOM_RESPONSE = "/utf16LeBomBytes";
private static final String UTF_32BE_BOM_RESPONSE = "/utf32BeBomBytes";
private static final String UTF_32LE_BOM_RESPONSE = "/utf32LeBomBytes";
private static final String BOM_WITH_SAME_HEADER = "/bomBytesWithSameHeader";
private static final String BOM_WITH_DIFFERENT_HEADER = "/bomBytesWithDifferentHeader";
private static final String ECHO_RESPONSE = "/echo";
static final String PLAIN_RESPONSE = "plainBytesNoHeader";
static final String HEADER_RESPONSE = "plainBytesWithHeader";
static final String INVALID_HEADER_RESPONSE = "plainBytesInvalidHeader";
static final String UTF_8_BOM_RESPONSE = "utf8BomBytes";
static final String UTF_16BE_BOM_RESPONSE = "utf16BeBomBytes";
static final String UTF_16LE_BOM_RESPONSE = "utf16LeBomBytes";
static final String UTF_32BE_BOM_RESPONSE = "utf32BeBomBytes";
static final String UTF_32LE_BOM_RESPONSE = "utf32LeBomBytes";
static final String BOM_WITH_SAME_HEADER = "bomBytesWithSameHeader";
static final String BOM_WITH_DIFFERENT_HEADER = "bomBytesWithDifferentHeader";
static final String ECHO_RESPONSE = "echo";
static final String SSE_RESPONSE = "serversentevent";
static final String HUGE_HEADER_RESPONSE = "hugeHeader";

private static final byte[] UTF_8_BOM = { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF };
private static final byte[] UTF_16BE_BOM = { (byte) 0xFE, (byte) 0xFF };
private static final byte[] UTF_16LE_BOM = { (byte) 0xFF, (byte) 0xFE };
private static final byte[] UTF_32BE_BOM = { (byte) 0x00, (byte) 0x00, (byte) 0xFE, (byte) 0xFF };
private static final byte[] UTF_32LE_BOM = { (byte) 0xFF, (byte) 0xFE, (byte) 0x00, (byte) 0x00 };

private static final byte[] RETURN_BYTES = "Hello World!".getBytes(StandardCharsets.UTF_8);
private static final String SSE_RESPONSE = "/serversentevent";
private static final String HELLO_WORLD = "Hello World!";
static final byte[] RETURN_BYTES = HELLO_WORLD.getBytes(StandardCharsets.UTF_8);
static final HttpHeaderName HUGE_HEADER_NAME = HttpHeaderName.fromString("x-huge-header");
static final String HUGE_HEADER_VALUE;

static {
// Create the huge header value, which is 1024 HELLO_WORLDs (about 12 KB).
StringBuilder sb = new StringBuilder(HELLO_WORLD.length() * 1024);
for (int i = 0; i < 1024; i++) {
sb.append(HELLO_WORLD);
}
HUGE_HEADER_VALUE = sb.toString();
}

public static LocalTestServer getHttpClientTestsServer() {
return new LocalTestServer((req, resp, requestBody) -> {
Expand Down Expand Up @@ -101,47 +115,71 @@ public static LocalTestServer getHttpClientTestsServer() {
resp.setStatus(400);
resp.getOutputStream().write("void exception body thrown".getBytes(StandardCharsets.UTF_8));
resp.flushBuffer();
} else if (get && PLAIN_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, PLAIN_RESPONSE)) {
handleRequest(resp, "application/octet-stream", RETURN_BYTES);
} else if (get && HEADER_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, HEADER_RESPONSE)) {
handleRequest(resp, "charset=UTF-16BE", RETURN_BYTES);
} else if (get && INVALID_HEADER_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, INVALID_HEADER_RESPONSE)) {
handleRequest(resp, "charset=invalid", RETURN_BYTES);
} else if (get && UTF_8_BOM_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, UTF_8_BOM_RESPONSE)) {
handleRequest(resp, "application/octet-stream", addBom(UTF_8_BOM));
} else if (get && UTF_16BE_BOM_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, UTF_16BE_BOM_RESPONSE)) {
handleRequest(resp, "application/octet-stream", addBom(UTF_16BE_BOM));
} else if (get && UTF_16LE_BOM_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, UTF_16LE_BOM_RESPONSE)) {
handleRequest(resp, "application/octet-stream", addBom(UTF_16LE_BOM));
} else if (get && UTF_32BE_BOM_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, UTF_32BE_BOM_RESPONSE)) {
handleRequest(resp, "application/octet-stream", addBom(UTF_32BE_BOM));
} else if (get && UTF_32LE_BOM_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, UTF_32LE_BOM_RESPONSE)) {
handleRequest(resp, "application/octet-stream", addBom(UTF_32LE_BOM));
} else if (get && BOM_WITH_SAME_HEADER.equals(path)) {
} else if (get && pathMatches(path, BOM_WITH_SAME_HEADER)) {
handleRequest(resp, "charset=UTF-8", addBom(UTF_8_BOM));
} else if (get && BOM_WITH_DIFFERENT_HEADER.equals(path)) {
} else if (get && pathMatches(path, BOM_WITH_DIFFERENT_HEADER)) {
handleRequest(resp, "charset=UTF-16", addBom(UTF_8_BOM));
} else if (put && ECHO_RESPONSE.equals(path)) {
} else if (put && pathMatches(path, ECHO_RESPONSE)) {
handleRequest(resp, "application/octet-stream", requestBody);
} else if (get && SSE_RESPONSE.equals(path)) {
} else if (get && pathMatches(path, SSE_RESPONSE)) {
if (req.getHeader("Last-Event-Id") != null) {
sendSSELastEventIdResponse(resp);
} else {
sendSSEResponseWithRetry(resp);
}
} else if (post && SSE_RESPONSE.equals(path)) {
} else if (post && pathMatches(path, SSE_RESPONSE)) {
sendSSEResponseWithDataOnly(resp);
} else if (put && SSE_RESPONSE.equals(path)) {
} else if (put && pathMatches(path, SSE_RESPONSE)) {
resp.addHeader("Content-Type", ContentType.TEXT_EVENT_STREAM);
resp.setStatus(200);
resp.getOutputStream().write(("msg hello world \n\n").getBytes());
resp.flushBuffer();
} else if (get && pathMatches(path, HUGE_HEADER_RESPONSE)) {
resp.addHeader(HUGE_HEADER_NAME.getCaseSensitiveName(), HUGE_HEADER_VALUE);
resp.setContentLength(0);
resp.setStatus(200);
resp.flushBuffer();
} else {
throw new ServletException("Unexpected request " + req.getMethod() + " " + path);
}
}, 100);
}

/**
* Helper method to check if the path matches the test path.
* <p>
* All path constants in this class don't contain a leading slash to allow them to be used in
* {@link HttpClientTests}. This method checks that the path matches the test path by inferring the leading slash
* on the test path.
*
* @param path The path used in the request.
* @param testPathWithoutLeadingSlash The test path without leading slash.
* @return Whether the path matches the test path.
*/
private static boolean pathMatches(String path, String testPathWithoutLeadingSlash) {
// Check that the path starts with a leading slash, that the length of the path is equal to the test path
// length + 1 (for the leading slash), and that the test path matches the path starting from index 1.
return path.charAt(0) == '/'
&& path.length() == testPathWithoutLeadingSlash.length() + 1
&& testPathWithoutLeadingSlash.regionMatches(0, path, 1, testPathWithoutLeadingSlash.length());
}

private static void sendSSEResponseWithDataOnly(Response resp) throws IOException {
resp.addHeader("Content-Type", ContentType.TEXT_EVENT_STREAM);
resp.getOutputStream().write(("data: YHOO\n" + "data: +2\n" + "data: 10\n" + "\n").getBytes());
Expand Down
Loading
Loading