memory limit from env var

TingDaoK · TingDaoK · commit 454c2f02f40c · 2025-11-19T14:41:12.000-08:00
diff --git a/README.md b/README.md
@@ -3,17 +3,48 @@
 The AWS-C-S3 library is an asynchronous AWS S3 client focused on maximizing throughput and network utilization.
 
 ### Key features:
-- **Automatic Request Splitting**: Improves throughput by automatically splitting the request into part-sized chunks and performing parallel uploads/downloads of these chunks over multiple connections. There's a cap on the throughput of single S3 connection, the only way to go faster is multiple parallel connections.
-- **Automatic Retries**: Increases resilience by retrying individual failed chunks of a file transfer, eliminating the need to restart transfers from scratch after an intermittent error.
-- **DNS Load Balancing**: DNS resolver continuously harvests Amazon S3 IP addresses. When load is spread across the S3 fleet, overall throughput more reliable than if all connections are going to a single IP.
-- **Advanced Network Management**: The client incorporates automatic request parallelization, effective timeouts and retries, and efficient connection reuse. This approach helps to maximize throughput and network utilization, and to avoid network overloads.
-- **Thread Pools and Async I/O**: Avoids bottlenecks associated with single-thread processing.
-- **Parallel Reads**: When uploading a large file from disk, reads from multiple parts of the file in parallel. This is faster than reading the file sequentially from beginning to end.
+
+* **Automatic Request Splitting**: Improves throughput by automatically splitting the request into part-sized chunks and performing parallel uploads/downloads of these chunks over multiple connections. There's a cap on the throughput of single S3 connection, the only way to go faster is multiple parallel connections.
+* **Automatic Retries**: Increases resilience by retrying individual failed chunks of a file transfer, eliminating the need to restart transfers from scratch after an intermittent error.
+* **DNS Load Balancing**: DNS resolver continuously harvests Amazon S3 IP addresses. When load is spread across the S3 fleet, overall throughput more reliable than if all connections are going to a single IP.
+* **Advanced Network Management**: The client incorporates automatic request parallelization, effective timeouts and retries, and efficient connection reuse. This approach helps to maximize throughput and network utilization, and to avoid network overloads.
+* **Thread Pools and Async I/O**: Avoids bottlenecks associated with single-thread processing.
+* **Parallel Reads**: When uploading a large file from disk, reads from multiple parts of the file in parallel. This is faster than reading the file sequentially from beginning to end.
 
 ### Documentation
 
-- [GetObject](docs/GetObject.md): A visual representation of the GetObject request flow.
-- [Memory Aware Requests Execution](docs/memory_aware_request_execution.md): An in-depth guide on optimizing memory usage during request executions.
+* [GetObject](docs/GetObject.md): A visual representation of the GetObject request flow.
+* [Memory Aware Requests Execution](docs/memory_aware_request_execution.md): An in-depth guide on optimizing memory usage during request executions.
+
+### Configuration
+
+#### Memory Limit
+
+The S3 client uses a buffer pool to manage memory for concurrent transfers. You can control the memory limit in two ways:
+
+1. **Via Configuration** (Recommended): Set `memory_limit_in_bytes` in `aws_s3_client_config`:
+
+```c
+   struct aws_s3_client_config config = {
+       .memory_limit_in_bytes = GB_TO_BYTES(4), // 4 GiB limit
+       // ... other configuration
+   };
+   ```
+
+2. **Via Environment Variable**: Set the `AWS_CRT_S3_MEMORY_LIMIT_IN_GIB` environment variable:
+
+```bash
+   export AWS_CRT_S3_MEMORY_LIMIT_IN_GIB=4  # 4 GiB limit
+   ```
+
+**Priority**: The configuration value takes precedence over the environment variable. If `memory_limit_in_bytes` is set to a non-zero value in the config, the environment variable is ignored.
+
+**Default Behavior**: If neither is set (config is 0 and environment variable is not set), the client sets a default memory limit based on the target throughput.
+
+**Notes**:
+* The environment variable value must be a valid positive integer representing gigabytes (GiB).
+* The value is converted from GiB to bytes internally (1 GiB = 1024³ bytes).
+* Invalid values or overflow conditions will cause client creation to fail with `AWS_ERROR_INVALID_ARGUMENT`.
 
 ## License
 
@@ -86,14 +117,19 @@ cmake --build aws-c-s3/build --target install
 After installing all the dependencies, and building aws-c-s3, you can run the sample directly from the s3 build directory.
 
 To download:
+
 ```
 aws-c-s3/build/samples/s3/s3 cp s3://<bucket-name>/<object-name> <download-path> --region <region>
 ```
+
 To upload:
+
 ```
 aws-c-s3/build/samples/s3/s3 cp <upload-path> s3://<bucket-name>/<object-name> --region <region>
 ```
+
 To list objects:
+
 ```
 aws-c-s3/build/samples/s3/s3 ls s3://<bucket-name> --region <region>
 ```
diff --git a/source/s3_client.c b/source/s3_client.c
@@ -21,6 +21,7 @@
 #include <aws/common/atomics.h>
 #include <aws/common/clock.h>
 #include <aws/common/device_random.h>
+#include <aws/common/environment.h>
 #include <aws/common/file.h>
 #include <aws/common/json.h>
 #include <aws/common/math.h>
@@ -334,7 +335,7 @@ struct aws_s3_client *aws_s3_client_new(
         if (memory_limit_from_env_var) {
             uint64_t mem_limit_in_gib = 0;
             if (aws_byte_cursor_utf8_parse_u64(
-                    aws_byte_cursor_from_string(memory_limit_from_env_var), mem_limit_in_gib)) {
+                    aws_byte_cursor_from_string(memory_limit_from_env_var), &mem_limit_in_gib)) {
                 aws_string_destroy(memory_limit_from_env_var);
                 AWS_LOGF_ERROR(
                     AWS_LS_S3_CLIENT,
@@ -344,18 +345,20 @@ struct aws_s3_client *aws_s3_client_new(
                 aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
                 return NULL;
             }
+            aws_string_destroy(memory_limit_from_env_var);
             uint64_t mem_limit_in_bytes = 0;
             /* Covert mem_limit_in_gib to bytes */
-            if (aws_mul_u64_checked(mem_limit_in_gib, 1024, mem_limit_in_bytes) ||
-                aws_mul_u64_checked(mem_limit_in_bytes, 1024, mem_limit_in_bytes) ||
-                aws_mul_u64_checked(mem_limit_in_bytes, 1024, mem_limit_in_bytes)) {
+            if (aws_mul_u64_checked(mem_limit_in_gib, 1024, &mem_limit_in_bytes) ||
+                aws_mul_u64_checked(mem_limit_in_bytes, 1024, &mem_limit_in_bytes) ||
+                aws_mul_u64_checked(mem_limit_in_bytes, 1024, &mem_limit_in_bytes)) {
                 AWS_LOGF_ERROR(
                     AWS_LS_S3_CLIENT,
                     "Cannot create client from client_config; envrionment variable: %s, overflow detected.",
                     s_memory_limit_env_var);
                 aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
                 return NULL;
             }
+
             mem_limit_configured = mem_limit_in_bytes;
         }
     } else {
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -448,6 +448,12 @@ add_net_test_case(client_update_upload_part_timeout)
 add_net_test_case(client_meta_request_override_part_size)
 add_net_test_case(client_meta_request_override_multipart_upload_threshold)
 
+# Memory limit environment variable tests
+add_net_test_case(s3_client_memory_limit_from_env_var_valid)
+add_net_test_case(s3_client_memory_limit_config_takes_precedence)
+add_net_test_case(s3_client_memory_limit_from_env_var_invalid)
+add_net_test_case(s3_client_memory_limit_from_env_var_overflow)
+
 add_net_test_case(test_s3_default_get_without_content_length)
 
 set(TEST_BINARY_NAME ${PROJECT_NAME}-tests)
diff --git a/tests/s3_client_memory_limit_env_var_test.c b/tests/s3_client_memory_limit_env_var_test.c
@@ -0,0 +1,181 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#include "s3_tester.h"
+#include <aws/common/environment.h>
+#include <aws/s3/private/s3_default_buffer_pool.h>
+#include <aws/s3/private/s3_util.h>
+#include <aws/testing/aws_test_harness.h>
+
+#define TEST_CASE(NAME)                                                                                                \
+    AWS_TEST_CASE(NAME, s_test_##NAME);                                                                                \
+    static int s_test_##NAME(struct aws_allocator *allocator, void *ctx)
+
+static const char *s_memory_limit_env_var = "AWS_CRT_S3_MEMORY_LIMIT_IN_GIB";
+
+/* Copied from s3_default_buffer_pool.c */
+static const size_t s_buffer_pool_reserved_mem = MB_TO_BYTES(128);
+
+/**
+ * Test that memory limit can be set via environment variable when config value is 0
+ */
+TEST_CASE(s3_client_memory_limit_from_env_var_valid) {
+    (void)ctx;
+    struct aws_s3_tester tester;
+    ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
+
+    /* Set environment variable to 4 GiB */
+    struct aws_string *env_var_name = aws_string_new_from_c_str(allocator, s_memory_limit_env_var);
+    struct aws_string *env_var_value = aws_string_new_from_c_str(allocator, "1");
+    ASSERT_SUCCESS(aws_set_environment_value(env_var_name, env_var_value));
+
+    struct aws_s3_client_config client_config = {
+        .part_size = MB_TO_BYTES(8),
+        .throughput_target_gbps = 10.0,
+        .memory_limit_in_bytes = 0, /* Will read from environment variable */
+    };
+
+    ASSERT_SUCCESS(aws_s3_tester_bind_client(
+        &tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION | AWS_S3_TESTER_BIND_CLIENT_SIGNING));
+
+    struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
+    ASSERT_TRUE(client != NULL);
+
+    /* Verify that buffer pool was configured with 4 GiB limit */
+    size_t expected_memory_limit = GB_TO_BYTES(1) - s_buffer_pool_reserved_mem;
+    ASSERT_TRUE(client->buffer_pool != NULL);
+    struct aws_s3_default_buffer_pool_usage_stats stats = aws_s3_default_buffer_pool_get_usage(client->buffer_pool);
+    ASSERT_UINT_EQUALS(stats.mem_limit, expected_memory_limit);
+
+    aws_s3_client_release(client);
+
+    /* Clean up environment variable */
+    aws_string_destroy(env_var_name);
+    aws_string_destroy(env_var_value);
+
+    aws_s3_tester_clean_up(&tester);
+    return AWS_OP_SUCCESS;
+}
+
+/**
+ * Test that config value takes precedence over environment variable
+ */
+TEST_CASE(s3_client_memory_limit_config_takes_precedence) {
+    (void)ctx;
+    struct aws_s3_tester tester;
+    ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
+
+    /* Set environment variable to 4 GiB */
+    struct aws_string *env_var_name = aws_string_new_from_c_str(allocator, s_memory_limit_env_var);
+    struct aws_string *env_var_value = aws_string_new_from_c_str(allocator, "1");
+    ASSERT_SUCCESS(aws_set_environment_value(env_var_name, env_var_value));
+    aws_string_destroy(env_var_name);
+    aws_string_destroy(env_var_value);
+
+    struct aws_s3_client_config client_config = {
+        .part_size = MB_TO_BYTES(8),
+        .throughput_target_gbps = 10.0,
+        .memory_limit_in_bytes = GB_TO_BYTES(2), /* Config value should take precedence */
+    };
+    ASSERT_SUCCESS(aws_s3_tester_bind_client(
+        &tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION | AWS_S3_TESTER_BIND_CLIENT_SIGNING));
+
+    struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
+    ASSERT_TRUE(client != NULL);
+
+    /* The 2 GiB from config should be used, not the 1 GiB from env var */
+    ASSERT_TRUE(client->buffer_pool != NULL);
+    size_t expected_memory_limit = GB_TO_BYTES(2) - s_buffer_pool_reserved_mem;
+    ASSERT_TRUE(client->buffer_pool != NULL);
+    struct aws_s3_default_buffer_pool_usage_stats stats = aws_s3_default_buffer_pool_get_usage(client->buffer_pool);
+    ASSERT_UINT_EQUALS(stats.mem_limit, expected_memory_limit);
+
+    aws_s3_client_release(client);
+
+    /* Clean up environment variable */
+    env_var_name = aws_string_new_from_c_str(allocator, s_memory_limit_env_var);
+    ASSERT_SUCCESS(aws_unset_environment_value(env_var_name));
+    aws_string_destroy(env_var_name);
+
+    aws_s3_tester_clean_up(&tester);
+    return AWS_OP_SUCCESS;
+}
+
+/**
+ * Test that invalid environment variable value causes client creation to fail
+ */
+TEST_CASE(s3_client_memory_limit_from_env_var_invalid) {
+    (void)ctx;
+    struct aws_s3_tester tester;
+    ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
+
+    /* Set environment variable to invalid value */
+    struct aws_string *env_var_name = aws_string_new_from_c_str(allocator, s_memory_limit_env_var);
+    struct aws_string *env_var_value = aws_string_new_from_c_str(allocator, "invalid");
+    ASSERT_SUCCESS(aws_set_environment_value(env_var_name, env_var_value));
+    aws_string_destroy(env_var_name);
+    aws_string_destroy(env_var_value);
+
+    struct aws_s3_client_config client_config = {
+        .part_size = MB_TO_BYTES(8),
+        .throughput_target_gbps = 10.0,
+        .memory_limit_in_bytes = 0, /* Will try to read from environment variable */
+    };
+    ASSERT_SUCCESS(aws_s3_tester_bind_client(
+        &tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION | AWS_S3_TESTER_BIND_CLIENT_SIGNING));
+
+    /* Client creation should fail due to invalid env var value */
+    struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
+    ASSERT_TRUE(client == NULL);
+    ASSERT_INT_EQUALS(AWS_ERROR_INVALID_ARGUMENT, aws_last_error());
+    /* Client failed to set up. */
+    tester.bound_to_client = false;
+    /* Clean up environment variable */
+    env_var_name = aws_string_new_from_c_str(allocator, s_memory_limit_env_var);
+    ASSERT_SUCCESS(aws_unset_environment_value(env_var_name));
+    aws_string_destroy(env_var_name);
+
+    aws_s3_tester_clean_up(&tester);
+    return AWS_OP_SUCCESS;
+}
+
+/**
+ * Test that environment variable with value causing overflow is handled properly
+ */
+TEST_CASE(s3_client_memory_limit_from_env_var_overflow) {
+    (void)ctx;
+    struct aws_s3_tester tester;
+    ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
+
+    /* Set environment variable to a very large value that would overflow when converted to bytes */
+    struct aws_string *env_var_name = aws_string_new_from_c_str(allocator, s_memory_limit_env_var);
+    struct aws_string *env_var_value = aws_string_new_from_c_str(allocator, "18446744073709551615"); /* UINT64_MAX */
+    ASSERT_SUCCESS(aws_set_environment_value(env_var_name, env_var_value));
+    aws_string_destroy(env_var_name);
+    aws_string_destroy(env_var_value);
+
+    struct aws_s3_client_config client_config = {
+        .part_size = MB_TO_BYTES(8),
+        .throughput_target_gbps = 10.0,
+        .memory_limit_in_bytes = 0, /* Will try to read from environment variable */
+    };
+
+    ASSERT_SUCCESS(aws_s3_tester_bind_client(
+        &tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION | AWS_S3_TESTER_BIND_CLIENT_SIGNING));
+
+    /* Client creation should fail due to overflow during GiB to bytes conversion */
+    struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
+    ASSERT_TRUE(client == NULL);
+    ASSERT_INT_EQUALS(AWS_ERROR_INVALID_ARGUMENT, aws_last_error());
+    /* Client failed to set up. */
+    tester.bound_to_client = false;
+
+    /* Clean up environment variable */
+    env_var_name = aws_string_new_from_c_str(allocator, s_memory_limit_env_var);
+    ASSERT_SUCCESS(aws_unset_environment_value(env_var_name));
+    aws_string_destroy(env_var_name);
+    aws_s3_tester_clean_up(&tester);
+    return AWS_OP_SUCCESS;
+}
