Merge branch 'main' into unified-write-data-api

Author: azkrishpy

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ on:
       - 'main'
 
 env:
-  BUILDER_VERSION: v0.9.72
+  BUILDER_VERSION: v0.9.90
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-c-http
@@ -135,6 +135,7 @@ jobs:
 
   # Test downstream repos.
   # This should not be required because we can run into a chicken and egg problem if there is a change that needs some fix in a downstream repo.
+  # bump to ubuntu-22 since ubuntu 18 failed to install the mock server deps with python3.6
   downstream:
     runs-on: ubuntu-24.04 # latest
     steps:
@@ -146,7 +147,7 @@ jobs:
     - name: Build ${{ env.PACKAGE_NAME }}
       run: |
         aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
-        ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build downstream -p ${{ env.PACKAGE_NAME }}
+        ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-ubuntu-22-x64 build downstream -p ${{ env.PACKAGE_NAME }}
 
   windows:
     runs-on: windows-2025 # latest
@@ -203,7 +204,7 @@ jobs:
         python .\aws-c-http\build\deps\aws-c-common\scripts\appverifier_ctest.py --build_directory .\aws-c-http\build\aws-c-http
 
   macos:
-    runs-on: macos-14
+    runs-on: macos-15
     strategy:
       fail-fast: false
       matrix:
@@ -220,7 +221,7 @@ jobs:
         ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_NETWORK_FRAMEWORK=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }}
 
   macos-x64:
-    runs-on: macos-14-large
+    runs-on: macos-15-large
     steps:
     - uses: aws-actions/configure-aws-credentials@v4
       with:
@@ -247,7 +248,7 @@ jobs:
         python3 builder.pyz build -p aws-c-http --cmake-extra=-DENABLE_LOCALHOST_INTEGRATION_TESTS=ON --config Debug
 
   localhost-test-macos:
-    runs-on: macos-14
+    runs-on: macos-15
     strategy:
       fail-fast: false
       matrix:

include/aws/http/http2_stream_manager.h

@@ -154,9 +154,17 @@ struct aws_http2_stream_manager_options {
     size_t max_concurrent_streams_per_connection;
     /**
      * Required.
-     * The max number of connections will be open at same time. If all the connections are full, manager will wait until
-     * available to vender more streams */
+     * The max number of connections that will be open at the same time. If all the connections are full, the manager
+     * will wait until a connection is available to vend more streams.
+     */
     size_t max_connections;
+    /**
+     * Optional.
+     * The max number of concurrent streams that can be active across all connections at the same time.
+     * 0 means no limit (default). When this limit is reached, the stream manager will wait for
+     * existing streams to complete before creating new ones, even if connections have available capacity.
+     */
+    size_t max_concurrent_streams;
 };
 
 struct aws_http2_stream_manager_acquire_stream_options {

include/aws/http/private/h2_stream.h

@@ -80,6 +80,7 @@ struct aws_h2_stream {
      * to keep flow continues.
      */
     int32_t window_size_threshold_to_send_update;
+    aws_http2_on_remote_end_stream_fn *on_h2_remote_end_stream;
 
     /* Only the event-loop thread may touch this data */
     struct {

include/aws/http/private/http2_stream_manager_impl.h

@@ -43,6 +43,12 @@ struct aws_h2_sm_connection {
     } thread_data;
 
     enum aws_h2_sm_connection_state_type state;
+
+    /**
+     * Node for tracking in the all_held_connections list,
+     * NOTE: lock required to alter the state of the list.
+     */
+    struct aws_linked_list_node node;
 };
 
 /* Live from the user request to acquire a stream to the stream completed. */
@@ -114,6 +120,14 @@ struct aws_http2_stream_manager {
      */
     size_t max_concurrent_streams_per_connection;
 
+    /**
+     * Optional. 0 means no limit (default).
+     * The max number of concurrent streams that can be active across all connections at the same time.
+     * When this limit is reached, the stream manager will wait for existing streams to complete
+     * before creating new ones, even if connections have available capacity.
+     */
+    size_t max_concurrent_streams;
+
     /**
      * Task to invoke pending acquisition callbacks asynchronously if stream manager is shutting.
      */
@@ -129,26 +143,36 @@ struct aws_http2_stream_manager {
         enum aws_h2_sm_state_type state;
 
         /**
-         * A set of all connections that meet all requirement to use. Note: there will be connections not in this set,
-         * but hold by the stream manager, which can be tracked by the streams created on it. Set of `struct
-         * aws_h2_sm_connection *`
+         * A set of all connections that meet all requirement to use. Doesn't own the connection.
+         *
+         * Note: there will be connections not in this set, but hold by the stream manager, which can be tracked by the
+         * all_held_connections. Set of `struct aws_h2_sm_connection *`
          */
         struct aws_random_access_set ideal_available_set;
         /**
-         * A set of all available connections that exceed the soft limits set by users. Note: there will be connections
-         * not in this set, but hold by the stream manager, which can be tracked by the streams created. Set of `struct
-         * aws_h2_sm_connection *`
+         * A set of all available connections that exceed the soft limits set by users. Doesn't own the connection.
+         *
+         * Note: there will be connections not in this set, but hold by the stream manager, which can be tracked by the
+         * all_held_connections. Set of `struct aws_h2_sm_connection *`
          */
         struct aws_random_access_set nonideal_available_set;
-        /* We don't mantain set for connections that is full or "dead" (Cannot make any new streams). We have streams
-         * opening from the connection tracking them */
+        /* We don't mantain set for connections that is full or "dead" (Cannot make any new streams). We have
+         * all_held_connections tracking them */
 
         /**
          * The set of all incomplete stream acquisition requests (haven't decide what connection to make the request
          * to), list of `struct aws_h2_sm_pending_stream_acquisition*`
          */
         struct aws_linked_list pending_stream_acquisitions;
 
+        /**
+         * List of all aws_h2_sm_connection that holding the HTTP connection from connection manager.
+         * This list tracks all aws_h2_sm_connection from getting the connection from connection manager until release
+         * it back.
+         * list of `struct aws_h2_sm_connection*`
+         */
+        struct aws_linked_list all_held_connections;
+
         /**
          * The number of connections acquired from connection manager and not released yet.
          */

include/aws/http/request_response.h

@@ -189,6 +189,24 @@ typedef int(
  */
 typedef int(aws_http_on_incoming_request_done_fn)(struct aws_http_stream *stream, void *user_data);
 
+/**
+ * Invoked when the remote peer sends END_STREAM on an HTTP/2 stream.
+ * This is always invoked on the HTTP connection's event-loop thread.
+ *
+ * **HTTP/2 ONLY** - This callback is only supported for HTTP/2 request.
+ *
+ * This callback is invoked when the remote peer finishes sending by setting the END_STREAM
+ * flag on the final HEADERS or DATA frame. This indicates that no more data will be received from the
+ * remote peer for this stream.
+ *
+ * Note: If the server sends RST_STREAM instead of END_STREAM, `on_remote_end_stream` will NOT fire,
+ * but `on_complete` will fire with an error code.
+ *
+ * @param stream The HTTP/2 stream
+ * @param user_data User data provided in aws_http_make_request_options
+ */
+typedef void(aws_http2_on_remote_end_stream_fn)(struct aws_http_stream *stream, void *user_data);
+
 /**
  * Invoked when a request/response stream is complete, whether successful or unsuccessful
  * This is always invoked on the HTTP connection's event-loop thread.
@@ -290,6 +308,16 @@ struct aws_http_make_request_options {
      */
     aws_http_on_stream_metrics_fn *on_metrics;
 
+    /**
+     * Invoked when the remote peer sends END_STREAM on an HTTP/2 stream (HTTP/2 ONLY).
+     * Optional.
+     * See `aws_http2_on_remote_end_stream_fn`.
+     *
+     * This callback fires when the remote peer sends END_STREAM, which happens BEFORE `on_complete`.
+     * Ignored for HTTP/1.x connections.
+     */
+    aws_http2_on_remote_end_stream_fn *on_h2_remote_end_stream;
+
     /**
      * Invoked when request/response stream is complete, whether successful or unsuccessful
      * Optional.

source/h2_stream.c

@@ -277,6 +277,7 @@ struct aws_h2_stream *aws_h2_stream_new_request(
     AWS_PRECONDITION(options);
 
     struct aws_h2_stream *stream = aws_mem_calloc(client_connection->alloc, 1, sizeof(struct aws_h2_stream));
+    stream->on_h2_remote_end_stream = options->on_h2_remote_end_stream;
 
     /* Initialize base stream */
     stream->base.vtable = &s_h2_stream_vtable;
@@ -1274,6 +1275,10 @@ struct aws_h2err aws_h2_stream_on_decoder_end_stream(struct aws_h2_stream *strea
         }
     }
 
+    if (stream->on_h2_remote_end_stream) {
+        stream->on_h2_remote_end_stream(&stream->base, stream->base.user_data);
+    }
+
     if (stream->thread_data.state == AWS_H2_STREAM_STATE_HALF_CLOSED_LOCAL) {
         /* Both sides have sent END_STREAM */
         stream->thread_data.state = AWS_H2_STREAM_STATE_CLOSED;