From 739762de3d442b4ed7ba28cfbfdbc8f2a338d553 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Wed, 28 Sep 2022 21:55:48 -0400 Subject: [PATCH 01/14] Added Http2 support --- driver/pom.xml | 5 +++ .../nosql/driver/NoSQLHandleConfig.java | 24 +++++++++++++ .../java/oracle/nosql/driver/http/Client.java | 1 + .../nosql/driver/http/NoSQLHandleImpl.java | 13 +++++++ .../httpclient/Http2SettingsHandler.java | 28 +++++++++++++++ .../nosql/driver/httpclient/HttpClient.java | 13 ++++++- .../HttpClientChannelPoolHandler.java | 35 +++++++++++++++---- .../iam/InstancePrincipalsProvider.java | 3 ++ .../driver/iam/SecurityTokenSupplier.java | 13 ++++--- .../driver/kv/StoreAccessTokenProvider.java | 16 +++++++++ .../driver/httpclient/ConnectionPoolTest.java | 2 ++ 11 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java diff --git a/driver/pom.xml b/driver/pom.xml index 6b68e4c7..7f01a051 100644 --- a/driver/pom.xml +++ b/driver/pom.xml @@ -175,6 +175,11 @@ netty-codec-http ${netty.version} + + io.netty + netty-codec-http2 + ${netty.version} + io.netty netty-handler diff --git a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java index da3cc407..26cc793b 100644 --- a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java +++ b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java @@ -142,6 +142,13 @@ public class NoSQLHandleConfig implements Cloneable { */ private int maxChunkSize = 0; + /** + * Use http2 protocol + * + * Default: false (use http_1_1) + */ + private boolean http2 = false; + /** * A RetryHandler, or null if not configured by the user. */ @@ -553,6 +560,14 @@ public int getDefaultRequestTimeout() { return timeout == 0 ? DEFAULT_TIMEOUT : timeout; } + /** + * + * @return http2 setting + */ + public boolean useHttp2() { + return http2; + } + /** * Returns the configured table request timeout value, in milliseconds. * The table request timeout default can be specified independently to allow @@ -788,6 +803,15 @@ public NoSQLHandleConfig setMaxContentLength(int maxContentLength) { return this; } + /** + * Enables http2 protocol + * @return this + */ + public NoSQLHandleConfig useHttp2(boolean enable) { + this.http2 = enable; + return this; + } + /** * Returns the maximum size, in bytes, of a request operation payload. * On-premise only. This value is ignored for cloud operations. diff --git a/driver/src/main/java/oracle/nosql/driver/http/Client.java b/driver/src/main/java/oracle/nosql/driver/http/Client.java index c0063982..d3e96836 100644 --- a/driver/src/main/java/oracle/nosql/driver/http/Client.java +++ b/driver/src/main/java/oracle/nosql/driver/http/Client.java @@ -244,6 +244,7 @@ public Client(Logger logger, httpConfig.getNumThreads(), httpConfig.getConnectionPoolMinSize(), httpConfig.getConnectionPoolInactivityPeriod(), + httpConfig.useHttp2(), httpConfig.getMaxContentLength(), httpConfig.getMaxChunkSize(), sslCtx, diff --git a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java index b5f33c47..f0242765 100644 --- a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java +++ b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java @@ -12,6 +12,10 @@ import javax.net.ssl.SSLException; +import io.netty.handler.codec.http2.Http2SecurityUtil; +import io.netty.handler.ssl.ApplicationProtocolConfig; +import io.netty.handler.ssl.ApplicationProtocolNames; +import io.netty.handler.ssl.SupportedCipherSuiteFilter; import oracle.nosql.driver.AuthorizationProvider; import oracle.nosql.driver.NoSQLHandle; import oracle.nosql.driver.NoSQLHandleConfig; @@ -124,6 +128,14 @@ private void configSslContext(NoSQLHandleConfig config) { } builder.sessionTimeout(config.getSSLSessionTimeout()); builder.sessionCacheSize(config.getSSLSessionCacheSize()); + if (config.useHttp2()) { + builder.ciphers(Http2SecurityUtil.CIPHERS, SupportedCipherSuiteFilter.INSTANCE); + builder.applicationProtocolConfig( + new ApplicationProtocolConfig(ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2)); + } config.setSslContext(builder.build()); } catch (SSLException se) { throw new IllegalStateException( @@ -137,6 +149,7 @@ private void configAuthProvider(Logger logger, NoSQLHandleConfig config) { if (ap instanceof StoreAccessTokenProvider) { final StoreAccessTokenProvider stProvider = (StoreAccessTokenProvider) ap; + stProvider.useHttp2(config.useHttp2()); if (stProvider.getLogger() == null) { stProvider.setLogger(logger); } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java new file mode 100644 index 00000000..d18b7db1 --- /dev/null +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java @@ -0,0 +1,28 @@ +package oracle.nosql.driver.httpclient; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.codec.http2.Http2Settings; + +import java.util.concurrent.TimeUnit; + +public class Http2SettingsHandler extends SimpleChannelInboundHandler { + private final ChannelPromise promise; + + public Http2SettingsHandler(ChannelPromise promise) { + this.promise = promise; + } + + public void awaitSettings(long timeout, TimeUnit unit) throws Exception { + if (!promise.awaitUninterruptibly(timeout, unit)) { + throw new IllegalStateException("Timed out waiting for settings"); + } + } + + @Override + protected void channelRead0(ChannelHandlerContext channelHandlerContext, Http2Settings http2Settings) throws Exception { + promise.setSuccess(); + channelHandlerContext.pipeline().remove(this); + } +} diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java index a23cc4a6..9e679c74 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java @@ -96,6 +96,7 @@ public class HttpClient { private final String host; private final int port; private final String name; + private final boolean http2; /* * Amount of time to wait for acquiring a channel before timing @@ -130,6 +131,7 @@ public class HttpClient { * * @param host the hostname for the HTTP server * @param port the port for the HTTP server + * @param useHttp2 if set to true, use http2 connections. * @param sslCtx if non-null, SSL context to use for connections. * @param handshakeTimeoutMs if not zero, timeout to use for SSL handshake * @param name A name to use in logging messages for this client. @@ -137,6 +139,7 @@ public class HttpClient { */ public static HttpClient createMinimalClient(String host, int port, + boolean useHttp2, SslContext sslCtx, int handshakeTimeoutMs, String name, @@ -147,6 +150,7 @@ public static HttpClient createMinimalClient(String host, 0, /* pool min */ 0, /* pool inactivity period */ true, /* minimal client */ + useHttp2, DEFAULT_MAX_CONTENT_LENGTH, DEFAULT_MAX_CHUNK_SIZE, sslCtx, handshakeTimeoutMs, name, logger); @@ -184,6 +188,7 @@ public HttpClient(String host, int numThreads, int connectionPoolMinSize, int inactivityPeriodSeconds, + boolean isHttp2, int maxContentLength, int maxChunkSize, SslContext sslCtx, @@ -192,7 +197,7 @@ public HttpClient(String host, Logger logger) { this(host, port, numThreads, connectionPoolMinSize, - inactivityPeriodSeconds, false /* not minimal */, + inactivityPeriodSeconds, false /* not minimal */, isHttp2, maxContentLength, maxChunkSize, sslCtx, handshakeTimeoutMs, name, logger); } @@ -205,6 +210,7 @@ private HttpClient(String host, int connectionPoolMinSize, int inactivityPeriodSeconds, boolean isMinimalClient, + boolean isHttp2, int maxContentLength, int maxChunkSize, SslContext sslCtx, @@ -217,6 +223,7 @@ private HttpClient(String host, this.host = host; this.port = port; this.name = name; + this.http2 = isHttp2; this.maxContentLength = (maxContentLength == 0 ? DEFAULT_MAX_CONTENT_LENGTH : maxContentLength); @@ -292,6 +299,10 @@ String getName() { return name; } + boolean isHttp2() { + return http2; + } + Logger getLogger() { return logger; } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java index 471cfdc7..5a007fad 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java @@ -7,9 +7,11 @@ package oracle.nosql.driver.httpclient; +import static io.netty.handler.logging.LogLevel.DEBUG; import static oracle.nosql.driver.util.LogUtil.logFine; import java.net.InetSocketAddress; + import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; @@ -23,6 +25,7 @@ import io.netty.channel.pool.ChannelPoolHandler; import io.netty.handler.codec.http.HttpClientCodec; import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http2.*; import io.netty.handler.proxy.HttpProxyHandler; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslHandshakeCompletionEvent; @@ -37,6 +40,8 @@ public class HttpClientChannelPoolHandler implements ChannelPoolHandler, ChannelHealthChecker { + private static final Http2FrameLogger logger = new Http2FrameLogger(DEBUG, HttpClientChannelPoolHandler.class); + private static final String CODEC_HANDLER_NAME = "http-codec"; private static final String AGG_HANDLER_NAME = "http-aggregator"; private static final String HTTP_HANDLER_NAME = "http-response-handler"; @@ -81,12 +86,30 @@ public void channelCreated(Channel ch) { p.addLast(sslHandler); p.addLast(new ChannelLoggingHandler(client)); } - p.addLast(CODEC_HANDLER_NAME, new HttpClientCodec - (4096, // initial line - 8192, // header size - client.getMaxChunkSize())); - p.addLast(AGG_HANDLER_NAME, new HttpObjectAggregator( - client.getMaxContentLength())); + if (client.isHttp2()) { + Http2Connection connection = new DefaultHttp2Connection(false); + HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() + .frameListener(new DelegatingDecompressorFrameListener( + connection, + new InboundHttp2ToHttpAdapterBuilder(connection) + .maxContentLength(client.getMaxContentLength()) + .propagateSettings(true) + .build())) + .frameLogger(logger) + .connection(connection) + .build(); + Http2SettingsHandler settingsHandler = new Http2SettingsHandler(ch.newPromise()); + + p.addLast(connectionHandler); + p.addLast(settingsHandler); + } else { // http_1_1 + p.addLast(CODEC_HANDLER_NAME, new HttpClientCodec + (4096, // initial line + 8192, // header size + client.getMaxChunkSize())); + p.addLast(AGG_HANDLER_NAME, new HttpObjectAggregator( + client.getMaxContentLength())); + } p.addLast(HTTP_HANDLER_NAME, new HttpClientHandler(client.getLogger())); diff --git a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java index 07d34c2e..67121269 100644 --- a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java +++ b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java @@ -57,6 +57,7 @@ public class InstancePrincipalsProvider protected final SecurityTokenSupplier tokenSupplier; protected final DefaultSessionKeySupplier sessionKeySupplier; private final Region region; + private boolean useHttp2 = false; public InstancePrincipalsProvider(SecurityTokenSupplier tokenSupplier, SessionKeyPairSupplier keyPairSupplier, @@ -72,6 +73,7 @@ public InstancePrincipalsProvider(SecurityTokenSupplier tokenSupplier, */ public void prepare(NoSQLHandleConfig config) { tokenSupplier.prepare(config); + useHttp2 = config.useHttp2(); } public void close() { @@ -295,6 +297,7 @@ private void autoDetectEndpointUsingMetadataUrl() { try { client = HttpClient.createMinimalClient(METADATA_SERVICE_HOST, 80, + false, null, 0, "InstanceMDClient", diff --git a/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java b/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java index cc0280c2..35ba62e0 100644 --- a/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java +++ b/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java @@ -104,13 +104,16 @@ synchronized void prepare(NoSQLHandleConfig config) { federationClient = buildHttpClient( federationURL, config.getSslContext(), - config.getSSLHandshakeTimeout(), logger); + config.getSSLHandshakeTimeout(), + config.useHttp2(), + logger); } } private static HttpClient buildHttpClient(URI endpoint, SslContext sslCtx, int sslHandshakeTimeout, + boolean useHttp2, Logger logger) { String scheme = endpoint.getScheme(); if (scheme == null) { @@ -119,7 +122,7 @@ private static HttpClient buildHttpClient(URI endpoint, endpoint.toString()); } if (scheme.equalsIgnoreCase("http")) { - return HttpClient.createMinimalClient(endpoint.getHost(), endpoint.getPort(), + return HttpClient.createMinimalClient(endpoint.getHost(), endpoint.getPort(), useHttp2, null, 0, "FederationClient", logger); } @@ -132,7 +135,7 @@ private static HttpClient buildHttpClient(URI endpoint, } } - return HttpClient.createMinimalClient(endpoint.getHost(), 443, + return HttpClient.createMinimalClient(endpoint.getHost(), 443, useHttp2, sslCtx, sslHandshakeTimeout, "FederationClient", logger); } @@ -338,8 +341,8 @@ void validate(long minTokenLifetime) { /** * Checks if two public keys are equal - * @param a one public key - * @param b the other one + * @param actual one public key + * @param expect the other one * @return true if the same */ private boolean isEqualPublicKey(RSAPublicKey actual, diff --git a/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java b/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java index ade9e1b4..a6d7eb61 100644 --- a/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java +++ b/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java @@ -112,6 +112,11 @@ public class StoreAccessTokenProvider implements AuthorizationProvider { */ private boolean autoRenew = true; + /* + * Whether use http2 connection, default use http1.1 + */ + private boolean useHttp2 = false; + /* * Whether this is a secure store token provider. */ @@ -375,6 +380,16 @@ public StoreAccessTokenProvider setLogger(Logger logger) { return this; } + /** + * Sets useHttp2 state + * @param enable set to true to use Http2 connection + * @return this + */ + public StoreAccessTokenProvider useHttp2(boolean enable) { + this.useHttp2 = enable; + return this; + } + public String getEndpoint() { return endpoint; } @@ -483,6 +498,7 @@ private HttpResponse sendRequest(String authHeader, client = HttpClient.createMinimalClient (loginHost, loginPort, + useHttp2, (isSecure && !disableSSLHook) ? sslContext : null, sslHandshakeTimeoutMs, serviceName, diff --git a/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java b/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java index e43ffeb1..9a6cc3e7 100644 --- a/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java +++ b/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java @@ -66,6 +66,7 @@ public void poolTest() throws Exception { 0, // threads poolMinSize, poolInactivityPeriod, + false, 0, // contentLen 0, // chunkSize null, // sslCtx @@ -165,6 +166,7 @@ public void testCloudTimeout() throws Exception { 0, // threads poolMinSize, -1, // poolInactivityPeriod + false, 0, // contentLen 0, // chunkSize buildSslContext(), From 1cff196913cad1be1cb660d330260e3cead51091 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Tue, 4 Oct 2022 13:15:09 -0400 Subject: [PATCH 02/14] remove unused http2 setting --- .../oracle/nosql/driver/iam/InstancePrincipalsProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java index 67121269..6431a13e 100644 --- a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java +++ b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java @@ -57,7 +57,6 @@ public class InstancePrincipalsProvider protected final SecurityTokenSupplier tokenSupplier; protected final DefaultSessionKeySupplier sessionKeySupplier; private final Region region; - private boolean useHttp2 = false; public InstancePrincipalsProvider(SecurityTokenSupplier tokenSupplier, SessionKeyPairSupplier keyPairSupplier, @@ -73,7 +72,6 @@ public InstancePrincipalsProvider(SecurityTokenSupplier tokenSupplier, */ public void prepare(NoSQLHandleConfig config) { tokenSupplier.prepare(config); - useHttp2 = config.useHttp2(); } public void close() { From 5ddaf98cd255e8b06c84d9cf3d2e88b40806254a Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Wed, 12 Oct 2022 11:04:51 -0400 Subject: [PATCH 03/14] Add HttpProtocolNegotiationHandler After SSL handshake and protocol negotiation, the client configure the pipeline based on the negotiation result. --- .../nosql/driver/http/NoSQLHandleImpl.java | 6 + .../httpclient/Http2SettingsHandler.java | 22 +- .../nosql/driver/httpclient/HttpClient.java | 5 + .../HttpClientChannelPoolHandler.java | 42 +--- .../HttpProtocolNegotiationHandler.java | 219 ++++++++++++++++++ .../oracle/nosql/driver/ProxyTestBase.java | 3 + 6 files changed, 259 insertions(+), 38 deletions(-) create mode 100644 driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java diff --git a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java index f0242765..21ca503b 100644 --- a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java +++ b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java @@ -135,6 +135,12 @@ private void configSslContext(NoSQLHandleConfig config) { ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, ApplicationProtocolNames.HTTP_2)); + } else { + builder.applicationProtocolConfig( + new ApplicationProtocolConfig(ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_1_1)); } config.setSslContext(builder.build()); } catch (SSLException se) { diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java index d18b7db1..1beb599f 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java @@ -4,14 +4,19 @@ import io.netty.channel.ChannelPromise; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.handler.codec.http2.Http2Settings; +import io.netty.util.internal.RecyclableArrayList; import java.util.concurrent.TimeUnit; public class Http2SettingsHandler extends SimpleChannelInboundHandler { private final ChannelPromise promise; + RecyclableArrayList bufferedMessages; + ChannelHandlerContext ctx; - public Http2SettingsHandler(ChannelPromise promise) { - this.promise = promise; + public Http2SettingsHandler(ChannelHandlerContext ctx, RecyclableArrayList bufferedMessages) { + this.ctx = ctx; + this.promise = ctx.newPromise(); + this.bufferedMessages = bufferedMessages; } public void awaitSettings(long timeout, TimeUnit unit) throws Exception { @@ -23,6 +28,19 @@ public void awaitSettings(long timeout, TimeUnit unit) throws Exception { @Override protected void channelRead0(ChannelHandlerContext channelHandlerContext, Http2Settings http2Settings) throws Exception { promise.setSuccess(); + fireBufferedMessages(); channelHandlerContext.pipeline().remove(this); } + + private void fireBufferedMessages() { + if (!this.bufferedMessages.isEmpty()) { + for(int i = 0; i < this.bufferedMessages.size(); ++i) { + Pair p = (Pair)this.bufferedMessages.get(i); + this.ctx.channel().write(p.first, p.second); + } + + this.bufferedMessages.clear(); + } + this.bufferedMessages.recycle(); + } } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java index 9e679c74..c0f9999b 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java @@ -31,6 +31,7 @@ import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.SslContext; import io.netty.util.AttributeKey; import io.netty.util.concurrent.Future; @@ -311,6 +312,10 @@ int getHandshakeTimeoutMs() { return handshakeTimeoutMs; } + public String getFallbackProtocol() { + return ApplicationProtocolNames.HTTP_1_1; + } + public int getMaxContentLength() { return maxContentLength; } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java index 5a007fad..a1bf60be 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java @@ -7,7 +7,6 @@ package oracle.nosql.driver.httpclient; -import static io.netty.handler.logging.LogLevel.DEBUG; import static oracle.nosql.driver.util.LogUtil.logFine; import java.net.InetSocketAddress; @@ -23,9 +22,6 @@ import io.netty.channel.EventLoop; import io.netty.channel.pool.ChannelHealthChecker; import io.netty.channel.pool.ChannelPoolHandler; -import io.netty.handler.codec.http.HttpClientCodec; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http2.*; import io.netty.handler.proxy.HttpProxyHandler; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslHandshakeCompletionEvent; @@ -40,12 +36,6 @@ public class HttpClientChannelPoolHandler implements ChannelPoolHandler, ChannelHealthChecker { - private static final Http2FrameLogger logger = new Http2FrameLogger(DEBUG, HttpClientChannelPoolHandler.class); - - private static final String CODEC_HANDLER_NAME = "http-codec"; - private static final String AGG_HANDLER_NAME = "http-aggregator"; - private static final String HTTP_HANDLER_NAME = "http-response-handler"; - private final HttpClient client; /** @@ -85,33 +75,13 @@ public void channelCreated(Channel ch) { p.addLast(sslHandler); p.addLast(new ChannelLoggingHandler(client)); + // Handle ALPN protocol negotiation result, and configure the pipeline accordingly + p.addLast(new HttpProtocolNegotiationHandler( + client.getFallbackProtocol(), new HttpClientHandler(client.getLogger()), client.getMaxChunkSize(), + client.getMaxContentLength(), client.getLogger())); + } else { + // TODO: H2C upgrade } - if (client.isHttp2()) { - Http2Connection connection = new DefaultHttp2Connection(false); - HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() - .frameListener(new DelegatingDecompressorFrameListener( - connection, - new InboundHttp2ToHttpAdapterBuilder(connection) - .maxContentLength(client.getMaxContentLength()) - .propagateSettings(true) - .build())) - .frameLogger(logger) - .connection(connection) - .build(); - Http2SettingsHandler settingsHandler = new Http2SettingsHandler(ch.newPromise()); - - p.addLast(connectionHandler); - p.addLast(settingsHandler); - } else { // http_1_1 - p.addLast(CODEC_HANDLER_NAME, new HttpClientCodec - (4096, // initial line - 8192, // header size - client.getMaxChunkSize())); - p.addLast(AGG_HANDLER_NAME, new HttpObjectAggregator( - client.getMaxContentLength())); - } - p.addLast(HTTP_HANDLER_NAME, - new HttpClientHandler(client.getLogger())); if (client.getProxyHost() != null) { InetSocketAddress sockAddr = diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java new file mode 100644 index 00000000..9c882b78 --- /dev/null +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java @@ -0,0 +1,219 @@ +/*- + * Copyright (c) 2011, 2022 Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * https://oss.oracle.com/licenses/upl/ + */ + +package oracle.nosql.driver.httpclient; + +import static io.netty.handler.logging.LogLevel.DEBUG; +import static oracle.nosql.driver.util.LogUtil.logFine; + +import java.net.SocketAddress; +import java.util.Objects; +import java.util.logging.Logger; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelOutboundHandler; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.HttpClientCodec; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http2.DefaultHttp2Connection; +import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; +import io.netty.handler.codec.http2.Http2Connection; +import io.netty.handler.codec.http2.Http2FrameLogger; +import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandler; +import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; +import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; +import io.netty.handler.ssl.ApplicationProtocolNames; +import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; +import io.netty.util.internal.RecyclableArrayList; + +/** + * Handle TLS protocol negotiation result, either Http1.1 or H2 + * + * The channel initialization process: + * 1. Channel aquired from {@link ConnectionPool} after channel is active. + * 2. SSL negotiation started, pipeline is not ready. + * 3. {@link HttpProtocolNegotiationHandler} holds all {@link HttpMessage} while waiting for the negotiation result. + * 4. Negotiation finished, {@link HttpProtocolNegotiationHandler} changes the pipeline according to the protocol selected. + * 5. {@link HttpProtocolNegotiationHandler} removes itself from the pipeline. Writes any buffered {@link HttpMessage} to the channel. + */ +public class HttpProtocolNegotiationHandler extends ApplicationProtocolNegotiationHandler implements ChannelOutboundHandler { + private static final Http2FrameLogger frameLogger = new Http2FrameLogger(DEBUG, HttpProtocolNegotiationHandler.class); + + private static final String CODEC_HANDLER_NAME = "http-codec"; + private static final String AGG_HANDLER_NAME = "http-aggregator"; + private static final String HTTP_HANDLER_NAME = "http-client-handler"; + + private final Logger logger; + private final RecyclableArrayList bufferedMessages = RecyclableArrayList.newInstance(); + private final HttpClientHandler handler; + private final int maxChunkSize; + private final int maxContentLength; + + public HttpProtocolNegotiationHandler(String fallbackProtocol, HttpClientHandler handler, int maxChunkSize, int maxContentLength, Logger logger) { + super(fallbackProtocol); + + this.logger = logger; + this.handler = handler; + this.maxChunkSize = maxChunkSize; + this.maxContentLength = maxContentLength; + } + + private void writeBufferedMessages(ChannelHandlerContext ctx) { + if (!this.bufferedMessages.isEmpty()) { + for(int i = 0; i < this.bufferedMessages.size(); ++i) { + Pair p = (Pair)this.bufferedMessages.get(i); + ctx.channel().write(p.first, p.second); + } + + this.bufferedMessages.clear(); + } + this.bufferedMessages.recycle(); + } + + private void configureHttp1(ChannelHandlerContext ctx) { + ChannelPipeline p = ctx.pipeline(); + + p.addLast(CODEC_HANDLER_NAME, + new HttpClientCodec(4096, // initial line + 8192, // header size + maxChunkSize)); // chunksize + p.addLast(AGG_HANDLER_NAME, + new HttpObjectAggregator(maxContentLength)); + } + + private void configureHttp2(ChannelHandlerContext ctx) { + ChannelPipeline p = ctx.pipeline(); + + Http2Connection connection = new DefaultHttp2Connection(false); + HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() + .frameListener(new DelegatingDecompressorFrameListener( + connection, + new InboundHttp2ToHttpAdapterBuilder(connection) + .maxContentLength(this.maxContentLength) + .propagateSettings(false) + .build())) + .frameLogger(frameLogger) + .connection(connection) + .build(); + + p.addLast(connectionHandler); + } + + @Override + protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { + if (ApplicationProtocolNames.HTTP_2.equals(protocol)) { + configureHttp2(ctx); + } else if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) { + configureHttp1(ctx); + } else { + throw new IllegalStateException("unknown http protocol: " + protocol); + } + logFine(this.logger, "HTTP protocol selected: " + protocol); + ctx.pipeline().addLast(HTTP_HANDLER_NAME, handler); + } + + /* + * User can write requests right after the channel is active, while protocol + * negotiation is still in progress. At this stage the pipeline is not ready + * to write http request so we must hold them here. + */ + @Override + public void write(ChannelHandlerContext ctx, Object o, ChannelPromise channelPromise) throws Exception { + if (o instanceof HttpMessage) { + Pair p = Pair.of(o, channelPromise); + this.bufferedMessages.add(p); + return; + } + + // let non-http message to pass, so the HTTP2 preface and settings frame can be sent + ctx.write(o, channelPromise); + } + + /* + * Protocol negotiation finish, handler removed, the pipeline is + * ready to handle http messages. Write previousely buffered http messages. + */ + @Override + public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { + super.handlerRemoved(ctx); + this.writeBufferedMessages(ctx); + } + + @Override + public void bind(ChannelHandlerContext ctx, SocketAddress socketAddress, ChannelPromise channelPromise) { + ctx.bind(socketAddress, channelPromise); + } + + @Override + public void connect(ChannelHandlerContext ctx, SocketAddress socketAddress, SocketAddress socketAddress1, ChannelPromise channelPromise) throws Exception { + ctx.connect(socketAddress, socketAddress1, channelPromise); + } + + @Override + public void disconnect(ChannelHandlerContext ctx, ChannelPromise channelPromise) { + ctx.disconnect(channelPromise); + } + + @Override + public void close(ChannelHandlerContext ctx, ChannelPromise channelPromise) throws Exception { + ctx.close(channelPromise); + } + + @Override + public void deregister(ChannelHandlerContext ctx, ChannelPromise channelPromise) { + ctx.deregister(channelPromise); + } + + @Override + public void read(ChannelHandlerContext ctx) throws Exception { + ctx.read(); + } + + @Override + public void flush(ChannelHandlerContext ctx) { + ctx.flush(); + } + + private static class Pair { + + public final A first; + public final B second; + + public Pair(A fst, B snd) { + this.first = fst; + this.second = snd; + } + + public String toString() { + return "Pair[" + first + "," + second + "]"; + } + + public boolean equals(Object other) { + if (other instanceof Pair) { + return Objects.equals(first, ((Pair) other).first) && + Objects.equals(second, ((Pair) other).second); + } + return false; + } + + public int hashCode() { + if (first == null) + return (second == null) ? 0 : second.hashCode() + 1; + else if (second == null) + return first.hashCode() + 2; + else + return first.hashCode() * 17 + second.hashCode(); + } + + public static Pair of(A a, B b) { + return new Pair<>(a, b); + } + } +} + diff --git a/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java b/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java index 0f69b449..b1862874 100644 --- a/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java +++ b/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java @@ -464,6 +464,9 @@ protected NoSQLHandle getHandle(NoSQLHandleConfig config) { logger.setLevel(Level.parse(level)); config.setLogger(logger); + boolean useHttp2 = Boolean.getBoolean("test.http2"); + config.useHttp2(useHttp2); + /* * Open the handle */ From 4e39324eb29267d22eb3242069f4b486c90c91bb Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Wed, 12 Oct 2022 13:15:21 -0400 Subject: [PATCH 04/14] Add http protocols list configuration The default setting prefers H2, but fallback to Http/1.1 --- .../nosql/driver/NoSQLHandleConfig.java | 37 +++++++++------ .../java/oracle/nosql/driver/http/Client.java | 2 +- .../nosql/driver/http/NoSQLHandleImpl.java | 18 +++----- .../httpclient/Http2SettingsHandler.java | 46 ------------------- .../nosql/driver/httpclient/HttpClient.java | 41 ++++++++++------- .../HttpClientChannelPoolHandler.java | 2 +- .../HttpProtocolNegotiationHandler.java | 11 +++-- .../iam/InstancePrincipalsProvider.java | 4 +- .../driver/iam/SecurityTokenSupplier.java | 13 +++--- .../driver/kv/StoreAccessTokenProvider.java | 15 +++--- .../oracle/nosql/driver/ProxyTestBase.java | 11 ++++- .../driver/httpclient/ConnectionPoolTest.java | 6 ++- 12 files changed, 93 insertions(+), 113 deletions(-) delete mode 100644 driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java diff --git a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java index 26cc793b..97faea99 100644 --- a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java +++ b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java @@ -21,6 +21,7 @@ import oracle.nosql.driver.Region.RegionProvider; import oracle.nosql.driver.iam.SignatureProvider; +import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.SslContext; /** @@ -143,11 +144,11 @@ public class NoSQLHandleConfig implements Cloneable { private int maxChunkSize = 0; /** - * Use http2 protocol + * Default http protocols * - * Default: false (use http_1_1) + * Default: prefer H2 but fallback to Http1.1 */ - private boolean http2 = false; + private List httpProtocols = new ArrayList<>(Arrays.asList(ApplicationProtocolNames.HTTP_2, ApplicationProtocolNames.HTTP_1_1)); /** * A RetryHandler, or null if not configured by the user. @@ -562,10 +563,19 @@ public int getDefaultRequestTimeout() { /** * - * @return http2 setting + * @return Http protocol settings + */ + public List getHttpProtocols() { + return httpProtocols; + } + + /** + * Check if "h2" is in the protocols list + * + * @return true if "h2" is in the protocols list */ public boolean useHttp2() { - return http2; + return this.httpProtocols.contains(ApplicationProtocolNames.HTTP_2); } /** @@ -646,6 +656,14 @@ public NoSQLHandleConfig setRequestTimeout(int timeout) { return this; } + public NoSQLHandleConfig setHttpProtocols(String ... protocols) { + this.httpProtocols = new ArrayList<>(2); + for (String p : protocols) { + this.httpProtocols.add(p); + } + return this; + } + /** * Sets the default table request timeout. * The table request timeout can be specified independently @@ -803,15 +821,6 @@ public NoSQLHandleConfig setMaxContentLength(int maxContentLength) { return this; } - /** - * Enables http2 protocol - * @return this - */ - public NoSQLHandleConfig useHttp2(boolean enable) { - this.http2 = enable; - return this; - } - /** * Returns the maximum size, in bytes, of a request operation payload. * On-premise only. This value is ignored for cloud operations. diff --git a/driver/src/main/java/oracle/nosql/driver/http/Client.java b/driver/src/main/java/oracle/nosql/driver/http/Client.java index d3e96836..2c82d43a 100644 --- a/driver/src/main/java/oracle/nosql/driver/http/Client.java +++ b/driver/src/main/java/oracle/nosql/driver/http/Client.java @@ -244,12 +244,12 @@ public Client(Logger logger, httpConfig.getNumThreads(), httpConfig.getConnectionPoolMinSize(), httpConfig.getConnectionPoolInactivityPeriod(), - httpConfig.useHttp2(), httpConfig.getMaxContentLength(), httpConfig.getMaxChunkSize(), sslCtx, config.getSSLHandshakeTimeout(), "NoSQL Driver", + config.getHttpProtocols(), logger); if (httpConfig.getProxyHost() != null) { httpClient.configureProxy(httpConfig); diff --git a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java index 21ca503b..94514ab7 100644 --- a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java +++ b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java @@ -130,18 +130,12 @@ private void configSslContext(NoSQLHandleConfig config) { builder.sessionCacheSize(config.getSSLSessionCacheSize()); if (config.useHttp2()) { builder.ciphers(Http2SecurityUtil.CIPHERS, SupportedCipherSuiteFilter.INSTANCE); - builder.applicationProtocolConfig( - new ApplicationProtocolConfig(ApplicationProtocolConfig.Protocol.ALPN, - ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, - ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, - ApplicationProtocolNames.HTTP_2)); - } else { - builder.applicationProtocolConfig( - new ApplicationProtocolConfig(ApplicationProtocolConfig.Protocol.ALPN, - ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, - ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, - ApplicationProtocolNames.HTTP_1_1)); } + builder.applicationProtocolConfig( + new ApplicationProtocolConfig(ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + config.getHttpProtocols())); config.setSslContext(builder.build()); } catch (SSLException se) { throw new IllegalStateException( @@ -155,7 +149,7 @@ private void configAuthProvider(Logger logger, NoSQLHandleConfig config) { if (ap instanceof StoreAccessTokenProvider) { final StoreAccessTokenProvider stProvider = (StoreAccessTokenProvider) ap; - stProvider.useHttp2(config.useHttp2()); + stProvider.setHttpProtocols(config.getHttpProtocols()); if (stProvider.getLogger() == null) { stProvider.setLogger(logger); } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java deleted file mode 100644 index 1beb599f..00000000 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/Http2SettingsHandler.java +++ /dev/null @@ -1,46 +0,0 @@ -package oracle.nosql.driver.httpclient; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPromise; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.handler.codec.http2.Http2Settings; -import io.netty.util.internal.RecyclableArrayList; - -import java.util.concurrent.TimeUnit; - -public class Http2SettingsHandler extends SimpleChannelInboundHandler { - private final ChannelPromise promise; - RecyclableArrayList bufferedMessages; - ChannelHandlerContext ctx; - - public Http2SettingsHandler(ChannelHandlerContext ctx, RecyclableArrayList bufferedMessages) { - this.ctx = ctx; - this.promise = ctx.newPromise(); - this.bufferedMessages = bufferedMessages; - } - - public void awaitSettings(long timeout, TimeUnit unit) throws Exception { - if (!promise.awaitUninterruptibly(timeout, unit)) { - throw new IllegalStateException("Timed out waiting for settings"); - } - } - - @Override - protected void channelRead0(ChannelHandlerContext channelHandlerContext, Http2Settings http2Settings) throws Exception { - promise.setSuccess(); - fireBufferedMessages(); - channelHandlerContext.pipeline().remove(this); - } - - private void fireBufferedMessages() { - if (!this.bufferedMessages.isEmpty()) { - for(int i = 0; i < this.bufferedMessages.size(); ++i) { - Pair p = (Pair)this.bufferedMessages.get(i); - this.ctx.channel().write(p.first, p.second); - } - - this.bufferedMessages.clear(); - } - this.bufferedMessages.recycle(); - } -} diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java index c0f9999b..e236517f 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java @@ -17,6 +17,9 @@ import static oracle.nosql.driver.util.LogUtil.logWarning; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -97,7 +100,8 @@ public class HttpClient { private final String host; private final int port; private final String name; - private final boolean http2; + private final List httpProtocols; + private final String httpFallbackProtocol; /* * Amount of time to wait for acquiring a channel before timing @@ -132,18 +136,18 @@ public class HttpClient { * * @param host the hostname for the HTTP server * @param port the port for the HTTP server - * @param useHttp2 if set to true, use http2 connections. * @param sslCtx if non-null, SSL context to use for connections. * @param handshakeTimeoutMs if not zero, timeout to use for SSL handshake * @param name A name to use in logging messages for this client. * @param logger A logger to use for logging messages. + * @param httpProtocols A list of preferred http protocols (H2 and Http1.1) */ public static HttpClient createMinimalClient(String host, int port, - boolean useHttp2, SslContext sslCtx, int handshakeTimeoutMs, String name, + List httpProtocols, Logger logger) { return new HttpClient(host, port, @@ -151,10 +155,9 @@ public static HttpClient createMinimalClient(String host, 0, /* pool min */ 0, /* pool inactivity period */ true, /* minimal client */ - useHttp2, DEFAULT_MAX_CONTENT_LENGTH, DEFAULT_MAX_CHUNK_SIZE, - sslCtx, handshakeTimeoutMs, name, logger); + sslCtx, handshakeTimeoutMs, name, httpProtocols, logger); } /** @@ -183,23 +186,24 @@ public static HttpClient createMinimalClient(String host, * @param handshakeTimeoutMs if not zero, timeout to use for SSL handshake * @param name A name to use in logging messages for this client. * @param logger A logger to use for logging messages. + * @param httpProtocols A list of preferred http protocols (H2 and Http1.1) */ public HttpClient(String host, int port, int numThreads, int connectionPoolMinSize, int inactivityPeriodSeconds, - boolean isHttp2, int maxContentLength, int maxChunkSize, SslContext sslCtx, int handshakeTimeoutMs, String name, + List httpProtocols, Logger logger) { this(host, port, numThreads, connectionPoolMinSize, - inactivityPeriodSeconds, false /* not minimal */, isHttp2, - maxContentLength, maxChunkSize, sslCtx, handshakeTimeoutMs, name, logger); + inactivityPeriodSeconds, false /* not minimal */, + maxContentLength, maxChunkSize, sslCtx, handshakeTimeoutMs, name, httpProtocols, logger); } /* @@ -211,12 +215,12 @@ private HttpClient(String host, int connectionPoolMinSize, int inactivityPeriodSeconds, boolean isMinimalClient, - boolean isHttp2, int maxContentLength, int maxChunkSize, SslContext sslCtx, int handshakeTimeoutMs, String name, + List httpProtocols, Logger logger) { this.logger = logger; @@ -224,7 +228,12 @@ private HttpClient(String host, this.host = host; this.port = port; this.name = name; - this.http2 = isHttp2; + + this.httpProtocols = httpProtocols.size() > 0 ? + httpProtocols : + new ArrayList<>(Arrays.asList(ApplicationProtocolNames.HTTP_2, ApplicationProtocolNames.HTTP_1_1)); + + this.httpFallbackProtocol = this.httpProtocols.get(this.httpProtocols.size() - 1); this.maxContentLength = (maxContentLength == 0 ? DEFAULT_MAX_CONTENT_LENGTH : maxContentLength); @@ -300,8 +309,12 @@ String getName() { return name; } - boolean isHttp2() { - return http2; + List getHttpProtocols() { + return httpProtocols; + } + + public String getHttpFallbackProtocol() { + return httpFallbackProtocol; } Logger getLogger() { @@ -312,10 +325,6 @@ int getHandshakeTimeoutMs() { return handshakeTimeoutMs; } - public String getFallbackProtocol() { - return ApplicationProtocolNames.HTTP_1_1; - } - public int getMaxContentLength() { return maxContentLength; } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java index a1bf60be..5c044604 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java @@ -77,7 +77,7 @@ public void channelCreated(Channel ch) { p.addLast(new ChannelLoggingHandler(client)); // Handle ALPN protocol negotiation result, and configure the pipeline accordingly p.addLast(new HttpProtocolNegotiationHandler( - client.getFallbackProtocol(), new HttpClientHandler(client.getLogger()), client.getMaxChunkSize(), + client.getHttpFallbackProtocol(), new HttpClientHandler(client.getLogger()), client.getMaxChunkSize(), client.getMaxContentLength(), client.getLogger())); } else { // TODO: H2C upgrade diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java index 9c882b78..bb1f8ced 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java @@ -36,7 +36,7 @@ * Handle TLS protocol negotiation result, either Http1.1 or H2 * * The channel initialization process: - * 1. Channel aquired from {@link ConnectionPool} after channel is active. + * 1. Channel acquired from {@link ConnectionPool} after channel is active. * 2. SSL negotiation started, pipeline is not ready. * 3. {@link HttpProtocolNegotiationHandler} holds all {@link HttpMessage} while waiting for the negotiation result. * 4. Negotiation finished, {@link HttpProtocolNegotiationHandler} changes the pipeline according to the protocol selected. @@ -121,7 +121,7 @@ protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { /* * User can write requests right after the channel is active, while protocol * negotiation is still in progress. At this stage the pipeline is not ready - * to write http request so we must hold them here. + * to write http requests, so we must hold them here. */ @Override public void write(ChannelHandlerContext ctx, Object o, ChannelPromise channelPromise) throws Exception { @@ -137,7 +137,7 @@ public void write(ChannelHandlerContext ctx, Object o, ChannelPromise channelPro /* * Protocol negotiation finish, handler removed, the pipeline is - * ready to handle http messages. Write previousely buffered http messages. + * ready to handle http messages. Write previously buffered http messages. */ @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { @@ -196,8 +196,9 @@ public String toString() { public boolean equals(Object other) { if (other instanceof Pair) { - return Objects.equals(first, ((Pair) other).first) && - Objects.equals(second, ((Pair) other).second); + Pair pair = (Pair) other; + return Objects.equals(first, pair.first) && + Objects.equals(second, pair.second); } return false; } diff --git a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java index 6431a13e..7fc82425 100644 --- a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java +++ b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java @@ -19,10 +19,12 @@ import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.logging.Logger; +import io.netty.handler.ssl.ApplicationProtocolNames; import oracle.nosql.driver.NoSQLHandleConfig; import oracle.nosql.driver.Region; import oracle.nosql.driver.Region.RegionProvider; @@ -295,10 +297,10 @@ private void autoDetectEndpointUsingMetadataUrl() { try { client = HttpClient.createMinimalClient(METADATA_SERVICE_HOST, 80, - false, null, 0, "InstanceMDClient", + Arrays.asList(ApplicationProtocolNames.HTTP_1_1), logger); HttpResponse response = HttpRequestUtil.doGetRequest diff --git a/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java b/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java index 35ba62e0..8aebc196 100644 --- a/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java +++ b/driver/src/main/java/oracle/nosql/driver/iam/SecurityTokenSupplier.java @@ -21,6 +21,7 @@ import java.util.Base64; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Logger; @@ -105,7 +106,7 @@ synchronized void prepare(NoSQLHandleConfig config) { federationURL, config.getSslContext(), config.getSSLHandshakeTimeout(), - config.useHttp2(), + config.getHttpProtocols(), logger); } } @@ -113,7 +114,7 @@ synchronized void prepare(NoSQLHandleConfig config) { private static HttpClient buildHttpClient(URI endpoint, SslContext sslCtx, int sslHandshakeTimeout, - boolean useHttp2, + List httpProtocols, Logger logger) { String scheme = endpoint.getScheme(); if (scheme == null) { @@ -122,8 +123,8 @@ private static HttpClient buildHttpClient(URI endpoint, endpoint.toString()); } if (scheme.equalsIgnoreCase("http")) { - return HttpClient.createMinimalClient(endpoint.getHost(), endpoint.getPort(), useHttp2, - null, 0, "FederationClient", logger); + return HttpClient.createMinimalClient(endpoint.getHost(), endpoint.getPort(), null, + 0, "FederationClient", httpProtocols, logger); } if (sslCtx == null) { @@ -135,9 +136,9 @@ private static HttpClient buildHttpClient(URI endpoint, } } - return HttpClient.createMinimalClient(endpoint.getHost(), 443, useHttp2, + return HttpClient.createMinimalClient(endpoint.getHost(), 443, sslCtx, sslHandshakeTimeout, - "FederationClient", logger); + "FederationClient", httpProtocols, logger); } private synchronized String refreshAndGetTokenInternal() { diff --git a/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java b/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java index a6d7eb61..19d52c83 100644 --- a/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java +++ b/driver/src/main/java/oracle/nosql/driver/kv/StoreAccessTokenProvider.java @@ -13,6 +13,7 @@ import java.net.URL; import java.util.Arrays; import java.util.Base64; +import java.util.List; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.atomic.AtomicReference; @@ -113,9 +114,9 @@ public class StoreAccessTokenProvider implements AuthorizationProvider { private boolean autoRenew = true; /* - * Whether use http2 connection, default use http1.1 + * list of preferred http protocols */ - private boolean useHttp2 = false; + private List httpProtocols; /* * Whether this is a secure store token provider. @@ -381,12 +382,12 @@ public StoreAccessTokenProvider setLogger(Logger logger) { } /** - * Sets useHttp2 state - * @param enable set to true to use Http2 connection + * Sets Http Protocols + * @param httpProtocols list of preferred http protocols * @return this */ - public StoreAccessTokenProvider useHttp2(boolean enable) { - this.useHttp2 = enable; + public StoreAccessTokenProvider setHttpProtocols(List httpProtocols) { + this.httpProtocols = httpProtocols; return this; } @@ -498,10 +499,10 @@ private HttpResponse sendRequest(String authHeader, client = HttpClient.createMinimalClient (loginHost, loginPort, - useHttp2, (isSecure && !disableSSLHook) ? sslContext : null, sslHandshakeTimeoutMs, serviceName, + httpProtocols, logger); return HttpRequestUtil.doGetRequest( client, diff --git a/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java b/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java index b1862874..88ccbfbc 100644 --- a/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java +++ b/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java @@ -33,6 +33,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import io.netty.handler.ssl.ApplicationProtocolNames; import oracle.nosql.driver.http.Client; import oracle.nosql.driver.http.NoSQLHandleImpl; import oracle.nosql.driver.kv.StoreAccessTokenProvider; @@ -464,8 +465,14 @@ protected NoSQLHandle getHandle(NoSQLHandleConfig config) { logger.setLevel(Level.parse(level)); config.setLogger(logger); - boolean useHttp2 = Boolean.getBoolean("test.http2"); - config.useHttp2(useHttp2); + boolean useHttp1only = Boolean.getBoolean("test.http1only"); + if (useHttp1only) { + config.setHttpProtocols(ApplicationProtocolNames.HTTP_1_1); + } + boolean useHttp2only = Boolean.getBoolean("test.http2only"); + if (useHttp2only) { + config.setHttpProtocols(ApplicationProtocolNames.HTTP_2); + } /* * Open the handle diff --git a/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java b/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java index 9a6cc3e7..28cf35e5 100644 --- a/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java +++ b/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java @@ -9,6 +9,7 @@ import static org.junit.Assert.assertEquals; +import java.util.Arrays; import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; @@ -16,6 +17,7 @@ import javax.net.ssl.SSLException; import java.net.URL; +import io.netty.handler.ssl.ApplicationProtocolNames; import org.junit.Before; import org.junit.Test; @@ -66,12 +68,12 @@ public void poolTest() throws Exception { 0, // threads poolMinSize, poolInactivityPeriod, - false, 0, // contentLen 0, // chunkSize null, // sslCtx 0, // ssl handshake timeout "Pool Test", + Arrays.asList(ApplicationProtocolNames.HTTP_1_1), logger); ConnectionPool pool = client.getConnectionPool(); @@ -166,12 +168,12 @@ public void testCloudTimeout() throws Exception { 0, // threads poolMinSize, -1, // poolInactivityPeriod - false, 0, // contentLen 0, // chunkSize buildSslContext(), 0, // ssl handshake timeout "Pool Cloud Test", + Arrays.asList(ApplicationProtocolNames.HTTP_1_1), logger); ConnectionPool pool = client.getConnectionPool(); From 9f467f453873cbd37451cf4268e48f99f2637d41 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Thu, 13 Oct 2022 22:36:39 -0400 Subject: [PATCH 05/14] Support Clear Text Http protocol negotiation --- driver/pom.xml | 2 +- .../nosql/driver/httpclient/HttpClient.java | 9 + .../HttpClientChannelPoolHandler.java | 78 ++++-- .../HttpProtocolNegotiationHandler.java | 100 +------- .../nosql/driver/httpclient/HttpUtil.java | 237 ++++++++++++++++++ 5 files changed, 311 insertions(+), 115 deletions(-) create mode 100644 driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java diff --git a/driver/pom.xml b/driver/pom.xml index 7f01a051..27788bfd 100644 --- a/driver/pom.xml +++ b/driver/pom.xml @@ -46,7 +46,7 @@ Copyright (c) 2011, 2022 Oracle and/or its affiliates. All rights reserved. http://docs.oracle.com/javase/8/docs/api false - 4.1.77.Final + 4.1.82.Final 2.12.5 1.70 diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java index e236517f..295728da 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java @@ -379,6 +379,15 @@ public int getFreeChannelCount() { return pool.getFreeChannels(); } + /** + * Check if "h2" is in the protocols list + * + * @return true if "h2" is in the protocols list + */ + public boolean useHttp2() { + return this.httpProtocols.contains(ApplicationProtocolNames.HTTP_2); + } + /* available for testing */ ConnectionPool getConnectionPool() { return pool; diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java index 5c044604..6452a583 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java @@ -23,6 +23,7 @@ import io.netty.channel.pool.ChannelHealthChecker; import io.netty.channel.pool.ChannelPoolHandler; import io.netty.handler.proxy.HttpProxyHandler; +import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslHandshakeCompletionEvent; import io.netty.util.concurrent.Future; @@ -48,6 +49,62 @@ public class HttpClientChannelPoolHandler implements ChannelPoolHandler, this.client = client; } + private void configureSSL(Channel ch) { + ChannelPipeline p = ch.pipeline(); + /* Enable hostname verification */ + final SslHandler sslHandler = client.getSslContext().newHandler( + ch.alloc(), client.getHost(), client.getPort()); + final SSLEngine sslEngine = sslHandler.engine(); + final SSLParameters sslParameters = sslEngine.getSSLParameters(); + sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); + sslEngine.setSSLParameters(sslParameters); + sslHandler.setHandshakeTimeoutMillis(client.getHandshakeTimeoutMs()); + + p.addLast(sslHandler); + p.addLast(new ChannelLoggingHandler(client)); + // Handle ALPN protocol negotiation result, and configure the pipeline accordingly + p.addLast(new HttpProtocolNegotiationHandler( + client.getHttpFallbackProtocol(), new HttpClientHandler(client.getLogger()), + client.getMaxChunkSize(), client.getMaxContentLength(), client.getLogger())); + } + + private void configureClearText(Channel ch) { + ChannelPipeline p = ch.pipeline(); + HttpClientHandler handler = new HttpClientHandler(client.getLogger()); + + // Only true when HTTP_2 is the only protocol, as if user set: + // config.setHttpProtocols(ApplicationProtocolNames.HTTP_2); + if (client.useHttp2() && + ApplicationProtocolNames.HTTP_2.equals(client.getHttpFallbackProtocol())) { + // If choose to use H2 and fallback is also H2 + // Then there is no need to upgrade from Http1.1 to H2C + // Directly connects with H2 protocol, so called Http2-prior-knowledge + HttpUtil.configureHttp2(ch.pipeline(), client.getMaxContentLength()); + p.addLast(handler); + return; + } + + // Only true when HTTP_1_1 is the only protocol, as if user set: + // config.setHttpProtocols(ApplicationProtocolNames.HTTP_1_1); + if (!client.useHttp2() && + ApplicationProtocolNames.HTTP_1_1.equals(client.getHttpFallbackProtocol())) { + HttpUtil.configureHttp1(ch.pipeline(), client.getMaxChunkSize(), client.getMaxContentLength()); + p.addLast(handler); + return; + } + + // Only true when both HTTP_2 and HTTP_1_1 are available, the default option: + // config.setHttpProtocols(ApplicationProtocolNames.HTTP_2, + // ApplicationProtocolNames.HTTP_1_1) + if (client.useHttp2() && + ApplicationProtocolNames.HTTP_1_1.equals(client.getHttpFallbackProtocol())) { + HttpUtil.configureH2C(ch.pipeline(), client.getMaxChunkSize(), client.getMaxContentLength()); + p.addLast(handler); + return; + } + throw new IllegalStateException("unknown protocol: " + client.getHttpProtocols()); + } + /** * Initialize a channel with handlers that: * 1 -- handle and HTTP @@ -62,25 +119,10 @@ public void channelCreated(Channel ch) { logFine(client.getLogger(), "HttpClient " + client.getName() + ", channel created: " + ch + ", acquired channel cnt " + client.getAcquiredChannelCount()); - ChannelPipeline p = ch.pipeline(); if (client.getSslContext() != null) { - /* Enable hostname verification */ - final SslHandler sslHandler = client.getSslContext().newHandler( - ch.alloc(), client.getHost(), client.getPort()); - final SSLEngine sslEngine = sslHandler.engine(); - final SSLParameters sslParameters = sslEngine.getSSLParameters(); - sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); - sslEngine.setSSLParameters(sslParameters); - sslHandler.setHandshakeTimeoutMillis(client.getHandshakeTimeoutMs()); - - p.addLast(sslHandler); - p.addLast(new ChannelLoggingHandler(client)); - // Handle ALPN protocol negotiation result, and configure the pipeline accordingly - p.addLast(new HttpProtocolNegotiationHandler( - client.getHttpFallbackProtocol(), new HttpClientHandler(client.getLogger()), client.getMaxChunkSize(), - client.getMaxContentLength(), client.getLogger())); + configureSSL(ch); } else { - // TODO: H2C upgrade + configureClearText(ch); } if (client.getProxyHost() != null) { @@ -94,7 +136,7 @@ public void channelCreated(Channel ch) { client.getProxyUsername(), client.getProxyPassword()); - p.addFirst("proxyServer", proxyHandler); + ch.pipeline().addFirst("proxyServer", proxyHandler); } } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java index bb1f8ced..1b0c1c2e 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java @@ -7,7 +7,6 @@ package oracle.nosql.driver.httpclient; -import static io.netty.handler.logging.LogLevel.DEBUG; import static oracle.nosql.driver.util.LogUtil.logFine; import java.net.SocketAddress; @@ -16,18 +15,8 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandler; -import io.netty.channel.ChannelPipeline; import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http.HttpClientCodec; import io.netty.handler.codec.http.HttpMessage; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http2.DefaultHttp2Connection; -import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; -import io.netty.handler.codec.http2.Http2Connection; -import io.netty.handler.codec.http2.Http2FrameLogger; -import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandler; -import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; -import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; import io.netty.util.internal.RecyclableArrayList; @@ -43,10 +32,6 @@ * 5. {@link HttpProtocolNegotiationHandler} removes itself from the pipeline. Writes any buffered {@link HttpMessage} to the channel. */ public class HttpProtocolNegotiationHandler extends ApplicationProtocolNegotiationHandler implements ChannelOutboundHandler { - private static final Http2FrameLogger frameLogger = new Http2FrameLogger(DEBUG, HttpProtocolNegotiationHandler.class); - - private static final String CODEC_HANDLER_NAME = "http-codec"; - private static final String AGG_HANDLER_NAME = "http-aggregator"; private static final String HTTP_HANDLER_NAME = "http-client-handler"; private final Logger logger; @@ -64,53 +49,12 @@ public HttpProtocolNegotiationHandler(String fallbackProtocol, HttpClientHandler this.maxContentLength = maxContentLength; } - private void writeBufferedMessages(ChannelHandlerContext ctx) { - if (!this.bufferedMessages.isEmpty()) { - for(int i = 0; i < this.bufferedMessages.size(); ++i) { - Pair p = (Pair)this.bufferedMessages.get(i); - ctx.channel().write(p.first, p.second); - } - - this.bufferedMessages.clear(); - } - this.bufferedMessages.recycle(); - } - - private void configureHttp1(ChannelHandlerContext ctx) { - ChannelPipeline p = ctx.pipeline(); - - p.addLast(CODEC_HANDLER_NAME, - new HttpClientCodec(4096, // initial line - 8192, // header size - maxChunkSize)); // chunksize - p.addLast(AGG_HANDLER_NAME, - new HttpObjectAggregator(maxContentLength)); - } - - private void configureHttp2(ChannelHandlerContext ctx) { - ChannelPipeline p = ctx.pipeline(); - - Http2Connection connection = new DefaultHttp2Connection(false); - HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() - .frameListener(new DelegatingDecompressorFrameListener( - connection, - new InboundHttp2ToHttpAdapterBuilder(connection) - .maxContentLength(this.maxContentLength) - .propagateSettings(false) - .build())) - .frameLogger(frameLogger) - .connection(connection) - .build(); - - p.addLast(connectionHandler); - } - @Override protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { if (ApplicationProtocolNames.HTTP_2.equals(protocol)) { - configureHttp2(ctx); + HttpUtil.configureHttp2(ctx.pipeline(), this.maxContentLength); } else if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) { - configureHttp1(ctx); + HttpUtil.configureHttp1(ctx.pipeline(), this.maxChunkSize, this.maxContentLength); } else { throw new IllegalStateException("unknown http protocol: " + protocol); } @@ -126,7 +70,7 @@ protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { @Override public void write(ChannelHandlerContext ctx, Object o, ChannelPromise channelPromise) throws Exception { if (o instanceof HttpMessage) { - Pair p = Pair.of(o, channelPromise); + HttpUtil.Pair p = HttpUtil.Pair.of(o, channelPromise); this.bufferedMessages.add(p); return; } @@ -142,7 +86,7 @@ public void write(ChannelHandlerContext ctx, Object o, ChannelPromise channelPro @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { super.handlerRemoved(ctx); - this.writeBufferedMessages(ctx); + HttpUtil.writeBufferedMessages(ctx, this.bufferedMessages); } @Override @@ -180,41 +124,5 @@ public void flush(ChannelHandlerContext ctx) { ctx.flush(); } - private static class Pair { - - public final A first; - public final B second; - - public Pair(A fst, B snd) { - this.first = fst; - this.second = snd; - } - - public String toString() { - return "Pair[" + first + "," + second + "]"; - } - - public boolean equals(Object other) { - if (other instanceof Pair) { - Pair pair = (Pair) other; - return Objects.equals(first, pair.first) && - Objects.equals(second, pair.second); - } - return false; - } - - public int hashCode() { - if (first == null) - return (second == null) ? 0 : second.hashCode() + 1; - else if (second == null) - return first.hashCode() + 2; - else - return first.hashCode() * 17 + second.hashCode(); - } - - public static Pair of(A a, B b) { - return new Pair<>(a, b); - } - } } diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java new file mode 100644 index 00000000..82d902d3 --- /dev/null +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java @@ -0,0 +1,237 @@ +/*- + * Copyright (c) 2011, 2022 Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * https://oss.oracle.com/licenses/upl/ + */ + +package oracle.nosql.driver.httpclient; + +import static io.netty.handler.logging.LogLevel.DEBUG; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Objects; + +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelOutboundHandler; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.HttpClientCodec; +import io.netty.handler.codec.http.HttpClientUpgradeHandler; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http2.DefaultHttp2Connection; +import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; +import io.netty.handler.codec.http2.Http2ClientUpgradeCodec; +import io.netty.handler.codec.http2.Http2Connection; +import io.netty.handler.codec.http2.Http2FrameLogger; +import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandler; +import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; +import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; +import io.netty.util.internal.RecyclableArrayList; + +public class HttpUtil { + private static final Http2FrameLogger frameLogger = new Http2FrameLogger(DEBUG, HttpProtocolNegotiationHandler.class); + + private static final String CODEC_HANDLER_NAME = "http-codec"; + private static final String AGG_HANDLER_NAME = "http-aggregator"; + + public static void configureHttp1(ChannelPipeline p, int maxChunkSize, int maxContentLength) { + p.addLast(CODEC_HANDLER_NAME, + new HttpClientCodec(4096, // initial line + 8192, // header size + maxChunkSize)); // chunksize + p.addLast(AGG_HANDLER_NAME, + new HttpObjectAggregator(maxContentLength)); + } + + public static void configureHttp2(ChannelPipeline p, int maxContentLength) { + Http2Connection connection = new DefaultHttp2Connection(false); + HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() + .frameListener(new DelegatingDecompressorFrameListener( + connection, + new InboundHttp2ToHttpAdapterBuilder(connection) + .maxContentLength(maxContentLength) + .propagateSettings(false) + .build())) + .frameLogger(frameLogger) + .connection(connection) + .build(); + + p.addLast(connectionHandler); + } + + public static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxContentLength) { + Http2Connection connection = new DefaultHttp2Connection(false); + HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() + .frameListener(new DelegatingDecompressorFrameListener( + connection, + new InboundHttp2ToHttpAdapterBuilder(connection) + .maxContentLength(maxContentLength) + .propagateSettings(false) + .build())) + .frameLogger(frameLogger) + .connection(connection) + .build(); + + HttpClientCodec sourceCodec = new HttpClientCodec(4096, 8192, maxChunkSize); + Http2ClientUpgradeCodec upgradeCodec = new Http2ClientUpgradeCodec(connectionHandler); + HttpClientUpgradeHandler upgradeHandler = new HttpClientUpgradeHandler(sourceCodec, upgradeCodec, maxContentLength); + p.addLast(sourceCodec, + upgradeHandler, + new UpgradeRequestHandler(maxContentLength)); + } + + public static void writeBufferedMessages(ChannelHandlerContext ctx, RecyclableArrayList bufferedMessages) { + if (!bufferedMessages.isEmpty()) { + for(int i = 0; i < bufferedMessages.size(); ++i) { + Pair p = (Pair)bufferedMessages.get(i); + ctx.channel().write(p.first, p.second); + } + + bufferedMessages.clear(); + } + bufferedMessages.recycle(); + } + + /** + * A handler that triggers the cleartext upgrade to HTTP/2 by sending an initial HTTP request. + */ + private static final class UpgradeRequestHandler extends ChannelInboundHandlerAdapter implements ChannelOutboundHandler { + private final int maxContentLength; + private final RecyclableArrayList bufferedMessages = RecyclableArrayList.newInstance(); + + public UpgradeRequestHandler(int maxContentLength) { + this.maxContentLength = maxContentLength; + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + DefaultFullHttpRequest upgradeRequest = + new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/", Unpooled.EMPTY_BUFFER); + + // Set HOST header as the remote peer may require it. + InetSocketAddress remote = (InetSocketAddress) ctx.channel().remoteAddress(); + String hostString = remote.getHostString(); + if (hostString == null) { + hostString = remote.getAddress().getHostAddress(); + } + upgradeRequest.headers().set(HttpHeaderNames.HOST, hostString + ':' + remote.getPort()); + + ctx.writeAndFlush(upgradeRequest); + + ctx.fireChannelActive(); + } + + @Override + public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { + super.handlerRemoved(ctx); + writeBufferedMessages(ctx, this.bufferedMessages); + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof HttpClientUpgradeHandler.UpgradeEvent) { + HttpClientUpgradeHandler.UpgradeEvent reg = (HttpClientUpgradeHandler.UpgradeEvent) evt; + if (reg == HttpClientUpgradeHandler.UpgradeEvent.UPGRADE_REJECTED) { + ctx.pipeline().addAfter(ctx.name(), AGG_HANDLER_NAME, new HttpObjectAggregator(maxContentLength)); + ctx.pipeline().remove(this); + } else if (reg == HttpClientUpgradeHandler.UpgradeEvent.UPGRADE_SUCCESSFUL) { + ctx.pipeline().remove(this); + } + } + ctx.fireUserEventTriggered(evt); + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (msg instanceof HttpMessage) { + Pair p = Pair.of(msg, promise); + this.bufferedMessages.add(p); + return; + } + + // let non-http message to pass, so the HTTP2 preface and settings frame can be sent + ctx.write(msg, promise); + } + + @Override + public void bind(ChannelHandlerContext ctx, SocketAddress localAddress, ChannelPromise promise) throws Exception { + ctx.bind(localAddress, promise); + } + + @Override + public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { + ctx.connect(remoteAddress, localAddress, promise); + } + + @Override + public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + ctx.disconnect(promise); + } + + @Override + public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + ctx.close(promise); + } + + @Override + public void deregister(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + ctx.deregister(promise); + } + + @Override + public void read(ChannelHandlerContext ctx) throws Exception { + ctx.read(); + } + + @Override + public void flush(ChannelHandlerContext ctx) throws Exception { + ctx.flush(); + } + } + + public static class Pair { + + public final A first; + public final B second; + + public Pair(A fst, B snd) { + this.first = fst; + this.second = snd; + } + + public String toString() { + return "Pair[" + first + "," + second + "]"; + } + + public boolean equals(Object other) { + if (other instanceof Pair) { + Pair pair = (Pair) other; + return Objects.equals(first, pair.first) && + Objects.equals(second, pair.second); + } + return false; + } + + public int hashCode() { + if (first == null) + return (second == null) ? 0 : second.hashCode() + 1; + else if (second == null) + return first.hashCode() + 2; + else + return first.hashCode() * 17 + second.hashCode(); + } + + public static Pair of(A a, B b) { + return new Pair<>(a, b); + } + } +} From 1a271f13408c74f9da75c6a92b3a20878a478276 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Fri, 14 Oct 2022 13:48:11 -0400 Subject: [PATCH 06/14] A better way to select fallback http protocol --- .../main/java/oracle/nosql/driver/httpclient/HttpClient.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java index 295728da..f5a42eb2 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java @@ -233,7 +233,10 @@ private HttpClient(String host, httpProtocols : new ArrayList<>(Arrays.asList(ApplicationProtocolNames.HTTP_2, ApplicationProtocolNames.HTTP_1_1)); - this.httpFallbackProtocol = this.httpProtocols.get(this.httpProtocols.size() - 1); + // If Http1.1 is in the httpProtocols list, we prefer use it as the fallback + // Else we use the last protocol in the httpProtocols list. + this.httpFallbackProtocol = this.httpProtocols.contains(ApplicationProtocolNames.HTTP_1_1) ? + ApplicationProtocolNames.HTTP_1_1 : this.httpProtocols.get(this.httpProtocols.size() - 1); this.maxContentLength = (maxContentLength == 0 ? DEFAULT_MAX_CONTENT_LENGTH : maxContentLength); From a1cb224317440a8d9b01a54f1b554fa0e9a66017 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Fri, 14 Oct 2022 14:15:08 -0400 Subject: [PATCH 07/14] fix: writing message before upgrade is finished --- .../HttpProtocolNegotiationHandler.java | 1 - .../nosql/driver/httpclient/HttpUtil.java | 76 ++++++++++--------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java index 1b0c1c2e..f3d81b55 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java @@ -10,7 +10,6 @@ import static oracle.nosql.driver.util.LogUtil.logFine; import java.net.SocketAddress; -import java.util.Objects; import java.util.logging.Logger; import io.netty.channel.ChannelHandlerContext; diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java index 82d902d3..1a8fbc34 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java @@ -9,24 +9,17 @@ import static io.netty.handler.logging.LogLevel.DEBUG; -import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; -import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.ChannelOutboundHandler; import io.netty.channel.ChannelPipeline; import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.HttpClientCodec; import io.netty.handler.codec.http.HttpClientUpgradeHandler; -import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpMessage; -import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http2.DefaultHttp2Connection; import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; import io.netty.handler.codec.http2.Http2ClientUpgradeCodec; @@ -83,7 +76,7 @@ public static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxCont HttpClientCodec sourceCodec = new HttpClientCodec(4096, 8192, maxChunkSize); Http2ClientUpgradeCodec upgradeCodec = new Http2ClientUpgradeCodec(connectionHandler); - HttpClientUpgradeHandler upgradeHandler = new HttpClientUpgradeHandler(sourceCodec, upgradeCodec, maxContentLength); + HttpClientUpgradeHandler upgradeHandler = new UpgradeHandler(sourceCodec, upgradeCodec, maxContentLength); p.addLast(sourceCodec, upgradeHandler, new UpgradeRequestHandler(maxContentLength)); @@ -101,35 +94,31 @@ public static void writeBufferedMessages(ChannelHandlerContext ctx, RecyclableAr bufferedMessages.recycle(); } + private static final class UpgradeHandler extends HttpClientUpgradeHandler { + public UpgradeHandler(SourceCodec sourceCodec, UpgradeCodec upgradeCodec, int maxContentLength) { + super(sourceCodec, upgradeCodec, maxContentLength); + } + + @Override + public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { + super.handlerRemoved(ctx); + ctx.fireUserEventTriggered(UpgradeFinishedEvent.INSTANCE); + } + } + /** * A handler that triggers the cleartext upgrade to HTTP/2 by sending an initial HTTP request. */ - private static final class UpgradeRequestHandler extends ChannelInboundHandlerAdapter implements ChannelOutboundHandler { + private static final class UpgradeRequestHandler extends ChannelDuplexHandler { private final int maxContentLength; private final RecyclableArrayList bufferedMessages = RecyclableArrayList.newInstance(); + private boolean upgradeTried = false; + private boolean upgrading = false; public UpgradeRequestHandler(int maxContentLength) { this.maxContentLength = maxContentLength; } - @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - DefaultFullHttpRequest upgradeRequest = - new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/", Unpooled.EMPTY_BUFFER); - - // Set HOST header as the remote peer may require it. - InetSocketAddress remote = (InetSocketAddress) ctx.channel().remoteAddress(); - String hostString = remote.getHostString(); - if (hostString == null) { - hostString = remote.getAddress().getHostAddress(); - } - upgradeRequest.headers().set(HttpHeaderNames.HOST, hostString + ':' + remote.getPort()); - - ctx.writeAndFlush(upgradeRequest); - - ctx.fireChannelActive(); - } - @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { super.handlerRemoved(ctx); @@ -142,23 +131,33 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc HttpClientUpgradeHandler.UpgradeEvent reg = (HttpClientUpgradeHandler.UpgradeEvent) evt; if (reg == HttpClientUpgradeHandler.UpgradeEvent.UPGRADE_REJECTED) { ctx.pipeline().addAfter(ctx.name(), AGG_HANDLER_NAME, new HttpObjectAggregator(maxContentLength)); - ctx.pipeline().remove(this); - } else if (reg == HttpClientUpgradeHandler.UpgradeEvent.UPGRADE_SUCCESSFUL) { - ctx.pipeline().remove(this); } } + if (evt instanceof UpgradeFinishedEvent) { + // Upgrade Finished (both SUCCESSFUL or REJECTED) + // The HttpClientUpgradeHandler is removed from pipeline + // The pipeline is now configured, and ready + // Remove this handler and flush the buffered messages + ctx.pipeline().remove(this); + } ctx.fireUserEventTriggered(evt); } @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { if (msg instanceof HttpMessage) { - Pair p = Pair.of(msg, promise); - this.bufferedMessages.add(p); - return; + // Only let the very first HttpMessage pass through + // The H2C upgrade handler modifies the first message, + // and tries to negotiate a new protocol with the server + // This process takes one round trip + if (upgrading) { + Pair p = Pair.of(msg, promise); + this.bufferedMessages.add(p); + return; + } + upgrading = true; } - // let non-http message to pass, so the HTTP2 preface and settings frame can be sent ctx.write(msg, promise); } @@ -198,6 +197,13 @@ public void flush(ChannelHandlerContext ctx) throws Exception { } } + public static final class UpgradeFinishedEvent { + private static final UpgradeFinishedEvent INSTANCE = new UpgradeFinishedEvent(); + + private UpgradeFinishedEvent() { + } + } + public static class Pair { public final A first; From eaf3f3ce934da00be320451f53c86118abcc8f19 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Fri, 14 Oct 2022 15:02:44 -0400 Subject: [PATCH 08/14] writeBufferedMessages now takes channle object This make the semantic clearer that we write the buffered messages to channel rather than the context. --- .../HttpProtocolNegotiationHandler.java | 2 +- .../nosql/driver/httpclient/HttpUtil.java | 43 ++++++++----------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java index f3d81b55..275aec3e 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java @@ -85,7 +85,7 @@ public void write(ChannelHandlerContext ctx, Object o, ChannelPromise channelPro @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { super.handlerRemoved(ctx); - HttpUtil.writeBufferedMessages(ctx, this.bufferedMessages); + HttpUtil.writeBufferedMessages(ctx.channel(), this.bufferedMessages); } @Override diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java index 1a8fbc34..97494529 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java @@ -12,6 +12,7 @@ import java.net.SocketAddress; import java.util.Objects; +import io.netty.channel.Channel; import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPipeline; @@ -24,6 +25,7 @@ import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; import io.netty.handler.codec.http2.Http2ClientUpgradeCodec; import io.netty.handler.codec.http2.Http2Connection; +import io.netty.handler.codec.http2.Http2ConnectionHandler; import io.netty.handler.codec.http2.Http2FrameLogger; import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandler; import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; @@ -46,22 +48,19 @@ public static void configureHttp1(ChannelPipeline p, int maxChunkSize, int maxCo } public static void configureHttp2(ChannelPipeline p, int maxContentLength) { - Http2Connection connection = new DefaultHttp2Connection(false); - HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() - .frameListener(new DelegatingDecompressorFrameListener( - connection, - new InboundHttp2ToHttpAdapterBuilder(connection) - .maxContentLength(maxContentLength) - .propagateSettings(false) - .build())) - .frameLogger(frameLogger) - .connection(connection) - .build(); - - p.addLast(connectionHandler); + p.addLast(createHttp2ConnectionHandler(maxContentLength)); } public static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxContentLength) { + HttpClientCodec sourceCodec = new HttpClientCodec(4096, 8192, maxChunkSize); + Http2ClientUpgradeCodec upgradeCodec = new Http2ClientUpgradeCodec(createHttp2ConnectionHandler(maxContentLength)); + HttpClientUpgradeHandler upgradeHandler = new UpgradeHandler(sourceCodec, upgradeCodec, maxContentLength); + p.addLast(sourceCodec, + upgradeHandler, + new UpgradeRequestHandler(maxContentLength)); + } + + private static Http2ConnectionHandler createHttp2ConnectionHandler(int maxContentLength) { Http2Connection connection = new DefaultHttp2Connection(false); HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() .frameListener(new DelegatingDecompressorFrameListener( @@ -73,20 +72,14 @@ public static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxCont .frameLogger(frameLogger) .connection(connection) .build(); - - HttpClientCodec sourceCodec = new HttpClientCodec(4096, 8192, maxChunkSize); - Http2ClientUpgradeCodec upgradeCodec = new Http2ClientUpgradeCodec(connectionHandler); - HttpClientUpgradeHandler upgradeHandler = new UpgradeHandler(sourceCodec, upgradeCodec, maxContentLength); - p.addLast(sourceCodec, - upgradeHandler, - new UpgradeRequestHandler(maxContentLength)); + return connectionHandler; } - public static void writeBufferedMessages(ChannelHandlerContext ctx, RecyclableArrayList bufferedMessages) { + public static void writeBufferedMessages(Channel ch, RecyclableArrayList bufferedMessages) { if (!bufferedMessages.isEmpty()) { for(int i = 0; i < bufferedMessages.size(); ++i) { Pair p = (Pair)bufferedMessages.get(i); - ctx.channel().write(p.first, p.second); + ch.write(p.first, p.second); } bufferedMessages.clear(); @@ -107,12 +100,12 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { } /** - * A handler that triggers the cleartext upgrade to HTTP/2 by sending an initial HTTP request. + * A handler that holds HttpMessages (Except the first one) + * So the upgrade handler can have time to finish clear text protocol upgrade */ private static final class UpgradeRequestHandler extends ChannelDuplexHandler { private final int maxContentLength; private final RecyclableArrayList bufferedMessages = RecyclableArrayList.newInstance(); - private boolean upgradeTried = false; private boolean upgrading = false; public UpgradeRequestHandler(int maxContentLength) { @@ -122,7 +115,7 @@ public UpgradeRequestHandler(int maxContentLength) { @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { super.handlerRemoved(ctx); - writeBufferedMessages(ctx, this.bufferedMessages); + writeBufferedMessages(ctx.channel(), this.bufferedMessages); } @Override From b067963d40e21e219e6ca86af8b3ff05875e6100 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Mon, 17 Oct 2022 13:35:11 -0400 Subject: [PATCH 09/14] Map Netty protocol names to NoSQL HttpConstants --- .../nosql/driver/NoSQLHandleConfig.java | 13 ++--------- .../nosql/driver/http/NoSQLHandleImpl.java | 3 ++- .../nosql/driver/httpclient/HttpClient.java | 17 ++++---------- .../HttpClientChannelPoolHandler.java | 23 ++++++++++--------- .../HttpProtocolNegotiationHandler.java | 6 ++--- .../nosql/driver/httpclient/HttpUtil.java | 14 ++++++----- .../nosql/driver/util/HttpConstants.java | 10 ++++++++ 7 files changed, 41 insertions(+), 45 deletions(-) diff --git a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java index 97faea99..bfabebed 100644 --- a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java +++ b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java @@ -21,7 +21,7 @@ import oracle.nosql.driver.Region.RegionProvider; import oracle.nosql.driver.iam.SignatureProvider; -import io.netty.handler.ssl.ApplicationProtocolNames; +import oracle.nosql.driver.util.HttpConstants; import io.netty.handler.ssl.SslContext; /** @@ -148,7 +148,7 @@ public class NoSQLHandleConfig implements Cloneable { * * Default: prefer H2 but fallback to Http1.1 */ - private List httpProtocols = new ArrayList<>(Arrays.asList(ApplicationProtocolNames.HTTP_2, ApplicationProtocolNames.HTTP_1_1)); + private List httpProtocols = new ArrayList<>(Arrays.asList(HttpConstants.HTTP_2, HttpConstants.HTTP_1_1)); /** * A RetryHandler, or null if not configured by the user. @@ -569,15 +569,6 @@ public List getHttpProtocols() { return httpProtocols; } - /** - * Check if "h2" is in the protocols list - * - * @return true if "h2" is in the protocols list - */ - public boolean useHttp2() { - return this.httpProtocols.contains(ApplicationProtocolNames.HTTP_2); - } - /** * Returns the configured table request timeout value, in milliseconds. * The table request timeout default can be specified independently to allow diff --git a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java index 94514ab7..5f91199f 100644 --- a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java +++ b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java @@ -50,6 +50,7 @@ import oracle.nosql.driver.ops.TableUsageResult; import oracle.nosql.driver.ops.WriteMultipleRequest; import oracle.nosql.driver.ops.WriteMultipleResult; +import oracle.nosql.driver.util.HttpConstants; import oracle.nosql.driver.values.FieldValue; import oracle.nosql.driver.values.JsonUtils; import oracle.nosql.driver.values.MapValue; @@ -128,7 +129,7 @@ private void configSslContext(NoSQLHandleConfig config) { } builder.sessionTimeout(config.getSSLSessionTimeout()); builder.sessionCacheSize(config.getSSLSessionCacheSize()); - if (config.useHttp2()) { + if (config.getHttpProtocols().contains(HttpConstants.HTTP_2)) { builder.ciphers(Http2SecurityUtil.CIPHERS, SupportedCipherSuiteFilter.INSTANCE); } builder.applicationProtocolConfig( diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java index f5a42eb2..88b32ed9 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClient.java @@ -34,7 +34,6 @@ import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.SslContext; import io.netty.util.AttributeKey; import io.netty.util.concurrent.Future; @@ -43,6 +42,7 @@ * from this config needs to be be abstracted to a generic class. */ import oracle.nosql.driver.NoSQLHandleConfig; +import oracle.nosql.driver.util.HttpConstants; /** * Netty HTTP client. Initialization process: @@ -231,12 +231,12 @@ private HttpClient(String host, this.httpProtocols = httpProtocols.size() > 0 ? httpProtocols : - new ArrayList<>(Arrays.asList(ApplicationProtocolNames.HTTP_2, ApplicationProtocolNames.HTTP_1_1)); + new ArrayList<>(Arrays.asList(HttpConstants.HTTP_2, HttpConstants.HTTP_1_1)); // If Http1.1 is in the httpProtocols list, we prefer use it as the fallback // Else we use the last protocol in the httpProtocols list. - this.httpFallbackProtocol = this.httpProtocols.contains(ApplicationProtocolNames.HTTP_1_1) ? - ApplicationProtocolNames.HTTP_1_1 : this.httpProtocols.get(this.httpProtocols.size() - 1); + this.httpFallbackProtocol = this.httpProtocols.contains(HttpConstants.HTTP_1_1) ? + HttpConstants.HTTP_1_1 : this.httpProtocols.get(this.httpProtocols.size() - 1); this.maxContentLength = (maxContentLength == 0 ? DEFAULT_MAX_CONTENT_LENGTH : maxContentLength); @@ -382,15 +382,6 @@ public int getFreeChannelCount() { return pool.getFreeChannels(); } - /** - * Check if "h2" is in the protocols list - * - * @return true if "h2" is in the protocols list - */ - public boolean useHttp2() { - return this.httpProtocols.contains(ApplicationProtocolNames.HTTP_2); - } - /* available for testing */ ConnectionPool getConnectionPool() { return pool; diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java index 6452a583..91474457 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpClientChannelPoolHandler.java @@ -23,10 +23,10 @@ import io.netty.channel.pool.ChannelHealthChecker; import io.netty.channel.pool.ChannelPoolHandler; import io.netty.handler.proxy.HttpProxyHandler; -import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslHandshakeCompletionEvent; import io.netty.util.concurrent.Future; +import oracle.nosql.driver.util.HttpConstants; /** * This is an instance of Netty's ChannelPoolHandler used to initialize @@ -71,11 +71,12 @@ private void configureSSL(Channel ch) { private void configureClearText(Channel ch) { ChannelPipeline p = ch.pipeline(); HttpClientHandler handler = new HttpClientHandler(client.getLogger()); + boolean useHttp2 = client.getHttpProtocols().contains(HttpConstants.HTTP_2); // Only true when HTTP_2 is the only protocol, as if user set: - // config.setHttpProtocols(ApplicationProtocolNames.HTTP_2); - if (client.useHttp2() && - ApplicationProtocolNames.HTTP_2.equals(client.getHttpFallbackProtocol())) { + // config.setHttpProtocols(HttpConstants.HTTP_2); + if (useHttp2 && + HttpConstants.HTTP_2.equals(client.getHttpFallbackProtocol())) { // If choose to use H2 and fallback is also H2 // Then there is no need to upgrade from Http1.1 to H2C // Directly connects with H2 protocol, so called Http2-prior-knowledge @@ -85,19 +86,19 @@ private void configureClearText(Channel ch) { } // Only true when HTTP_1_1 is the only protocol, as if user set: - // config.setHttpProtocols(ApplicationProtocolNames.HTTP_1_1); - if (!client.useHttp2() && - ApplicationProtocolNames.HTTP_1_1.equals(client.getHttpFallbackProtocol())) { + // config.setHttpProtocols(HttpConstants.HTTP_1_1); + if (!useHttp2 && + HttpConstants.HTTP_1_1.equals(client.getHttpFallbackProtocol())) { HttpUtil.configureHttp1(ch.pipeline(), client.getMaxChunkSize(), client.getMaxContentLength()); p.addLast(handler); return; } // Only true when both HTTP_2 and HTTP_1_1 are available, the default option: - // config.setHttpProtocols(ApplicationProtocolNames.HTTP_2, - // ApplicationProtocolNames.HTTP_1_1) - if (client.useHttp2() && - ApplicationProtocolNames.HTTP_1_1.equals(client.getHttpFallbackProtocol())) { + // config.setHttpProtocols(HttpConstants.HTTP_2, + // HttpConstants.HTTP_1_1) + if (useHttp2 && + HttpConstants.HTTP_1_1.equals(client.getHttpFallbackProtocol())) { HttpUtil.configureH2C(ch.pipeline(), client.getMaxChunkSize(), client.getMaxContentLength()); p.addLast(handler); return; diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java index 275aec3e..2f6690e1 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpProtocolNegotiationHandler.java @@ -16,9 +16,9 @@ import io.netty.channel.ChannelOutboundHandler; import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http.HttpMessage; -import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; import io.netty.util.internal.RecyclableArrayList; +import oracle.nosql.driver.util.HttpConstants; /** * Handle TLS protocol negotiation result, either Http1.1 or H2 @@ -50,9 +50,9 @@ public HttpProtocolNegotiationHandler(String fallbackProtocol, HttpClientHandler @Override protected void configurePipeline(ChannelHandlerContext ctx, String protocol) { - if (ApplicationProtocolNames.HTTP_2.equals(protocol)) { + if (HttpConstants.HTTP_2.equals(protocol)) { HttpUtil.configureHttp2(ctx.pipeline(), this.maxContentLength); - } else if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) { + } else if (HttpConstants.HTTP_1_1.equals(protocol)) { HttpUtil.configureHttp1(ctx.pipeline(), this.maxChunkSize, this.maxContentLength); } else { throw new IllegalStateException("unknown http protocol: " + protocol); diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java index 97494529..830508a5 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java @@ -9,18 +9,24 @@ import static io.netty.handler.logging.LogLevel.DEBUG; +import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; +import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPipeline; import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.HttpClientCodec; import io.netty.handler.codec.http.HttpClientUpgradeHandler; +import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http2.DefaultHttp2Connection; import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; import io.netty.handler.codec.http2.Http2ClientUpgradeCodec; @@ -30,6 +36,7 @@ import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandler; import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandlerBuilder; import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapterBuilder; +import io.netty.util.ReferenceCountUtil; import io.netty.util.internal.RecyclableArrayList; public class HttpUtil { @@ -100,8 +107,7 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { } /** - * A handler that holds HttpMessages (Except the first one) - * So the upgrade handler can have time to finish clear text protocol upgrade + * A handler that triggers the cleartext upgrade to HTTP/2 by sending an initial HTTP request. */ private static final class UpgradeRequestHandler extends ChannelDuplexHandler { private final int maxContentLength; @@ -139,10 +145,6 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { if (msg instanceof HttpMessage) { - // Only let the very first HttpMessage pass through - // The H2C upgrade handler modifies the first message, - // and tries to negotiate a new protocol with the server - // This process takes one round trip if (upgrading) { Pair p = Pair.of(msg, promise); this.bufferedMessages.add(p); diff --git a/driver/src/main/java/oracle/nosql/driver/util/HttpConstants.java b/driver/src/main/java/oracle/nosql/driver/util/HttpConstants.java index 906f201d..5f352b04 100644 --- a/driver/src/main/java/oracle/nosql/driver/util/HttpConstants.java +++ b/driver/src/main/java/oracle/nosql/driver/util/HttpConstants.java @@ -14,6 +14,16 @@ */ public class HttpConstants { + /** + * {@code "h2"}: HTTP version 2 + */ + public static final String HTTP_2 = "h2"; + + /** + * {@code "http/1.1"}: HTTP version 1.1 + */ + public static final String HTTP_1_1 = "http/1.1"; + /** * The http header that identifies the client scoped unique request id * associated with each request. The request header is returned by the From 457d288e0c9e8066daa34b6e25f6116803690c66 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Mon, 17 Oct 2022 21:11:44 -0400 Subject: [PATCH 10/14] Add javadoc for the config public API --- .../java/oracle/nosql/driver/NoSQLHandleConfig.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java index bfabebed..9c14c136 100644 --- a/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java +++ b/driver/src/main/java/oracle/nosql/driver/NoSQLHandleConfig.java @@ -562,6 +562,10 @@ public int getDefaultRequestTimeout() { } /** + * Returns the list of Http Protocols. If there is no configured + * protocol, a "default" value of + * List({@link HttpConstants#HTTP_2}, {@link HttpConstants#HTTP_1_1}) + * is used. * * @return Http protocol settings */ @@ -647,6 +651,14 @@ public NoSQLHandleConfig setRequestTimeout(int timeout) { return this; } + /** + * Sets the default http protocol(s). The default is {@link HttpConstants#HTTP_2} + * and fall back to {@link HttpConstants#HTTP_1_1} + * + * @param protocols Protocol list + * + * @return this + */ public NoSQLHandleConfig setHttpProtocols(String ... protocols) { this.httpProtocols = new ArrayList<>(2); for (String p : protocols) { From 740812cc2c08f91a66f52dbf74eb660db009655d Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Mon, 17 Oct 2022 21:21:09 -0400 Subject: [PATCH 11/14] H2C works with both old and new Httpproxy --- .../nosql/driver/httpclient/HttpUtil.java | 108 +++++++++++++----- 1 file changed, 80 insertions(+), 28 deletions(-) diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java index 830508a5..7869724a 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java @@ -8,6 +8,11 @@ package oracle.nosql.driver.httpclient; import static io.netty.handler.logging.LogLevel.DEBUG; +import static io.netty.handler.codec.http.HttpHeaderNames.HOST; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static io.netty.handler.codec.http.HttpMethod.HEAD; +import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; +import static io.netty.handler.codec.http.HttpClientUpgradeHandler.UpgradeEvent; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -20,13 +25,11 @@ import io.netty.channel.ChannelPipeline; import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpClientCodec; import io.netty.handler.codec.http.HttpClientUpgradeHandler; -import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpMessage; -import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http2.DefaultHttp2Connection; import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener; import io.netty.handler.codec.http2.Http2ClientUpgradeCodec; @@ -45,6 +48,10 @@ public class HttpUtil { private static final String CODEC_HANDLER_NAME = "http-codec"; private static final String AGG_HANDLER_NAME = "http-aggregator"; + public static void removeHttpObjectAggregator(ChannelPipeline p) { + p.remove(AGG_HANDLER_NAME); + } + public static void configureHttp1(ChannelPipeline p, int maxChunkSize, int maxContentLength) { p.addLast(CODEC_HANDLER_NAME, new HttpClientCodec(4096, // initial line @@ -62,9 +69,11 @@ public static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxCont HttpClientCodec sourceCodec = new HttpClientCodec(4096, 8192, maxChunkSize); Http2ClientUpgradeCodec upgradeCodec = new Http2ClientUpgradeCodec(createHttp2ConnectionHandler(maxContentLength)); HttpClientUpgradeHandler upgradeHandler = new UpgradeHandler(sourceCodec, upgradeCodec, maxContentLength); - p.addLast(sourceCodec, - upgradeHandler, - new UpgradeRequestHandler(maxContentLength)); + + p.addLast(CODEC_HANDLER_NAME, sourceCodec); + p.addLast(upgradeHandler); + p.addLast(AGG_HANDLER_NAME, new HttpObjectAggregator(maxContentLength)); + p.addLast(new UpgradeRequestHandler()); } private static Http2ConnectionHandler createHttp2ConnectionHandler(int maxContentLength) { @@ -89,6 +98,7 @@ public static void writeBufferedMessages(Channel ch, RecyclableArrayList buffere ch.write(p.first, p.second); } + ch.flush(); bufferedMessages.clear(); } bufferedMessages.recycle(); @@ -107,35 +117,67 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { } /** - * A handler that triggers the cleartext upgrade to HTTP/2 by sending an initial HTTP request. + * A handler that triggers the H2C upgrade to HTTP/2 by sending an initial HTTP1.1 request. + * */ private static final class UpgradeRequestHandler extends ChannelDuplexHandler { - private final int maxContentLength; private final RecyclableArrayList bufferedMessages = RecyclableArrayList.newInstance(); - private boolean upgrading = false; + private UpgradeEvent upgradeResult = null; + + /** + * In channelActive event, we send a probe request "HEAD / Http1.1 upgrade: h2c" to proxy. + */ + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + DefaultFullHttpRequest upgradeRequest = + new DefaultFullHttpRequest(HTTP_1_1, HEAD, "/", Unpooled.EMPTY_BUFFER); + + // Set HOST header as the remote peer may require it. + InetSocketAddress remote = (InetSocketAddress) ctx.channel().remoteAddress(); + String hostString = remote.getHostString(); + if (hostString == null) { + hostString = remote.getAddress().getHostAddress(); + } + upgradeRequest.headers().set(HOST, hostString + ':' + remote.getPort()); - public UpgradeRequestHandler(int maxContentLength) { - this.maxContentLength = maxContentLength; + ctx.writeAndFlush(upgradeRequest); + + ctx.fireChannelActive(); } + /* + * Upgrading, we temporarily hold all user requests. + */ @Override - public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { - super.handlerRemoved(ctx); - writeBufferedMessages(ctx.channel(), this.bufferedMessages); + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (msg instanceof HttpMessage) { + Pair p = Pair.of(msg, promise); + this.bufferedMessages.add(p); + return; + } + + ctx.write(msg, promise); } @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (evt instanceof HttpClientUpgradeHandler.UpgradeEvent) { - HttpClientUpgradeHandler.UpgradeEvent reg = (HttpClientUpgradeHandler.UpgradeEvent) evt; - if (reg == HttpClientUpgradeHandler.UpgradeEvent.UPGRADE_REJECTED) { - ctx.pipeline().addAfter(ctx.name(), AGG_HANDLER_NAME, new HttpObjectAggregator(maxContentLength)); + // HttpClientUpgradeHandler received the first response from proxy + // Based on the response, it triggers UpgradeEvent, either SUCCEFULL or REJECTED. + if (evt instanceof UpgradeEvent) { + // This can also be UpgradeEvent.UPGRADE_ISSUED + // But it will be overwritten when the first response arrive + upgradeResult = (UpgradeEvent) evt; + // If upgrade is SUCCESSFUL, at this point the Http2ConnectionHandler is installed + // We don't need Aggregator anymore, remove it. + if (upgradeResult == UpgradeEvent.UPGRADE_SUCCESSFUL) { + HttpUtil.removeHttpObjectAggregator(ctx.pipeline()); } } - if (evt instanceof UpgradeFinishedEvent) { - // Upgrade Finished (both SUCCESSFUL or REJECTED) + if (evt instanceof UpgradeFinishedEvent && + upgradeResult == UpgradeEvent.UPGRADE_SUCCESSFUL) { // The HttpClientUpgradeHandler is removed from pipeline - // The pipeline is now configured, and ready + // Upgrade is SUCCESSFUL + // The pipeline is now configured for H2 // Remove this handler and flush the buffered messages ctx.pipeline().remove(this); } @@ -143,17 +185,27 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc } @Override - public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { - if (msg instanceof HttpMessage) { - if (upgrading) { - Pair p = Pair.of(msg, promise); - this.bufferedMessages.add(p); + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + // When upgrade is rejected (Old version proxy, does not support Http2) + // the proxy sends a "400 Bad Request" response, drop it here. + // Also remove this handler and flush the buffered messages + if (msg instanceof FullHttpResponse && + upgradeResult == UpgradeEvent.UPGRADE_REJECTED) { + FullHttpResponse rep = (FullHttpResponse) msg; + if (BAD_REQUEST.equals(rep.status())) { + // Just drop the first "400" response, remove this handler + ReferenceCountUtil.release(msg); + ctx.pipeline().remove(this); return; } - upgrading = true; } + ctx.fireChannelRead(msg); + } - ctx.write(msg, promise); + @Override + public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { + super.handlerRemoved(ctx); + writeBufferedMessages(ctx.channel(), this.bufferedMessages); } @Override From 80d2523e54be89080c46501c4711f6160888b03a Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Mon, 17 Oct 2022 22:10:26 -0400 Subject: [PATCH 12/14] fix: wrong class name for the http2 frame logger --- .../src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java index 7869724a..a645eb57 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java @@ -43,7 +43,7 @@ import io.netty.util.internal.RecyclableArrayList; public class HttpUtil { - private static final Http2FrameLogger frameLogger = new Http2FrameLogger(DEBUG, HttpProtocolNegotiationHandler.class); + private static final Http2FrameLogger frameLogger = new Http2FrameLogger(DEBUG, HttpUtil.class); private static final String CODEC_HANDLER_NAME = "http-codec"; private static final String AGG_HANDLER_NAME = "http-aggregator"; From 749501509586d483aaf4de3b7ddbc8ad6835a88b Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Mon, 17 Oct 2022 22:15:29 -0400 Subject: [PATCH 13/14] Replace more instances of ApplicationProtocolNames --- .../main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java | 1 - .../oracle/nosql/driver/iam/InstancePrincipalsProvider.java | 4 ++-- driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java | 6 +++--- .../oracle/nosql/driver/httpclient/ConnectionPoolTest.java | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java index 5f91199f..17a18476 100644 --- a/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java +++ b/driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java @@ -14,7 +14,6 @@ import io.netty.handler.codec.http2.Http2SecurityUtil; import io.netty.handler.ssl.ApplicationProtocolConfig; -import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.SupportedCipherSuiteFilter; import oracle.nosql.driver.AuthorizationProvider; import oracle.nosql.driver.NoSQLHandle; diff --git a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java index 7fc82425..45512160 100644 --- a/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java +++ b/driver/src/main/java/oracle/nosql/driver/iam/InstancePrincipalsProvider.java @@ -24,7 +24,6 @@ import java.util.Set; import java.util.logging.Logger; -import io.netty.handler.ssl.ApplicationProtocolNames; import oracle.nosql.driver.NoSQLHandleConfig; import oracle.nosql.driver.Region; import oracle.nosql.driver.Region.RegionProvider; @@ -34,6 +33,7 @@ import oracle.nosql.driver.iam.SecurityTokenSupplier.SecurityTokenBasedProvider; import oracle.nosql.driver.iam.SessionKeyPairSupplier.DefaultSessionKeySupplier; import oracle.nosql.driver.iam.SessionKeyPairSupplier.JDKKeyPairSupplier; +import oracle.nosql.driver.util.HttpConstants; import oracle.nosql.driver.util.HttpRequestUtil; import oracle.nosql.driver.util.HttpRequestUtil.HttpResponse; @@ -300,7 +300,7 @@ private void autoDetectEndpointUsingMetadataUrl() { null, 0, "InstanceMDClient", - Arrays.asList(ApplicationProtocolNames.HTTP_1_1), + Arrays.asList(HttpConstants.HTTP_1_1), logger); HttpResponse response = HttpRequestUtil.doGetRequest diff --git a/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java b/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java index 88ccbfbc..0e927c48 100644 --- a/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java +++ b/driver/src/test/java/oracle/nosql/driver/ProxyTestBase.java @@ -33,7 +33,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import io.netty.handler.ssl.ApplicationProtocolNames; import oracle.nosql.driver.http.Client; import oracle.nosql.driver.http.NoSQLHandleImpl; import oracle.nosql.driver.kv.StoreAccessTokenProvider; @@ -53,6 +52,7 @@ import oracle.nosql.driver.ops.TableResult; import oracle.nosql.driver.ops.WriteMultipleRequest; import oracle.nosql.driver.ops.WriteMultipleResult; +import oracle.nosql.driver.util.HttpConstants; import oracle.nosql.driver.values.ArrayValue; import oracle.nosql.driver.values.MapValue; @@ -467,11 +467,11 @@ protected NoSQLHandle getHandle(NoSQLHandleConfig config) { boolean useHttp1only = Boolean.getBoolean("test.http1only"); if (useHttp1only) { - config.setHttpProtocols(ApplicationProtocolNames.HTTP_1_1); + config.setHttpProtocols(HttpConstants.HTTP_1_1); } boolean useHttp2only = Boolean.getBoolean("test.http2only"); if (useHttp2only) { - config.setHttpProtocols(ApplicationProtocolNames.HTTP_2); + config.setHttpProtocols(HttpConstants.HTTP_2); } /* diff --git a/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java b/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java index 28cf35e5..ec801974 100644 --- a/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java +++ b/driver/src/test/java/oracle/nosql/driver/httpclient/ConnectionPoolTest.java @@ -17,7 +17,6 @@ import javax.net.ssl.SSLException; import java.net.URL; -import io.netty.handler.ssl.ApplicationProtocolNames; import org.junit.Before; import org.junit.Test; @@ -26,6 +25,7 @@ import io.netty.handler.ssl.SslContextBuilder; import oracle.nosql.driver.NoSQLHandleConfig; +import oracle.nosql.driver.util.HttpConstants; /** * This test is excluded from the test profiles and must be run standalone. @@ -73,7 +73,7 @@ public void poolTest() throws Exception { null, // sslCtx 0, // ssl handshake timeout "Pool Test", - Arrays.asList(ApplicationProtocolNames.HTTP_1_1), + Arrays.asList(HttpConstants.HTTP_1_1), logger); ConnectionPool pool = client.getConnectionPool(); @@ -173,7 +173,7 @@ public void testCloudTimeout() throws Exception { buildSslContext(), 0, // ssl handshake timeout "Pool Cloud Test", - Arrays.asList(ApplicationProtocolNames.HTTP_1_1), + Arrays.asList(HttpConstants.HTTP_1_1), logger); ConnectionPool pool = client.getConnectionPool(); From 788cb1e563de214bcf0f11494fcd1cecb615b328 Mon Sep 17 00:00:00 2001 From: Junyi Liu Date: Tue, 18 Oct 2022 09:53:35 -0400 Subject: [PATCH 14/14] protect method visibility --- .../nosql/driver/httpclient/HttpUtil.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java index a645eb57..7b047243 100644 --- a/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java +++ b/driver/src/main/java/oracle/nosql/driver/httpclient/HttpUtil.java @@ -48,11 +48,26 @@ public class HttpUtil { private static final String CODEC_HANDLER_NAME = "http-codec"; private static final String AGG_HANDLER_NAME = "http-aggregator"; - public static void removeHttpObjectAggregator(ChannelPipeline p) { + private static Http2ConnectionHandler createHttp2ConnectionHandler(int maxContentLength) { + Http2Connection connection = new DefaultHttp2Connection(false); + HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() + .frameListener(new DelegatingDecompressorFrameListener( + connection, + new InboundHttp2ToHttpAdapterBuilder(connection) + .maxContentLength(maxContentLength) + .propagateSettings(false) + .build())) + .frameLogger(frameLogger) + .connection(connection) + .build(); + return connectionHandler; + } + + protected static void removeHttpObjectAggregator(ChannelPipeline p) { p.remove(AGG_HANDLER_NAME); } - public static void configureHttp1(ChannelPipeline p, int maxChunkSize, int maxContentLength) { + protected static void configureHttp1(ChannelPipeline p, int maxChunkSize, int maxContentLength) { p.addLast(CODEC_HANDLER_NAME, new HttpClientCodec(4096, // initial line 8192, // header size @@ -61,11 +76,11 @@ public static void configureHttp1(ChannelPipeline p, int maxChunkSize, int maxCo new HttpObjectAggregator(maxContentLength)); } - public static void configureHttp2(ChannelPipeline p, int maxContentLength) { + protected static void configureHttp2(ChannelPipeline p, int maxContentLength) { p.addLast(createHttp2ConnectionHandler(maxContentLength)); } - public static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxContentLength) { + protected static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxContentLength) { HttpClientCodec sourceCodec = new HttpClientCodec(4096, 8192, maxChunkSize); Http2ClientUpgradeCodec upgradeCodec = new Http2ClientUpgradeCodec(createHttp2ConnectionHandler(maxContentLength)); HttpClientUpgradeHandler upgradeHandler = new UpgradeHandler(sourceCodec, upgradeCodec, maxContentLength); @@ -76,22 +91,7 @@ public static void configureH2C(ChannelPipeline p, int maxChunkSize, int maxCont p.addLast(new UpgradeRequestHandler()); } - private static Http2ConnectionHandler createHttp2ConnectionHandler(int maxContentLength) { - Http2Connection connection = new DefaultHttp2Connection(false); - HttpToHttp2ConnectionHandler connectionHandler = new HttpToHttp2ConnectionHandlerBuilder() - .frameListener(new DelegatingDecompressorFrameListener( - connection, - new InboundHttp2ToHttpAdapterBuilder(connection) - .maxContentLength(maxContentLength) - .propagateSettings(false) - .build())) - .frameLogger(frameLogger) - .connection(connection) - .build(); - return connectionHandler; - } - - public static void writeBufferedMessages(Channel ch, RecyclableArrayList bufferedMessages) { + protected static void writeBufferedMessages(Channel ch, RecyclableArrayList bufferedMessages) { if (!bufferedMessages.isEmpty()) { for(int i = 0; i < bufferedMessages.size(); ++i) { Pair p = (Pair)bufferedMessages.get(i);