s3_util.c

/**
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

#include "aws/s3/private/s3_util.h"
#include "aws/s3/private/s3_client_impl.h"
#include "aws/s3/private/s3_meta_request_impl.h"
#include "aws/s3/private/s3_platform_info.h"
#include "aws/s3/private/s3_request.h"
#include <aws/auth/credentials.h>
#include <aws/common/clock.h>
#include <aws/common/encoding.h>
#include <aws/common/string.h>
#include <aws/common/xml_parser.h>
#include <aws/http/request_response.h>
#include <aws/s3/s3_client.h>
#include <inttypes.h>

#ifdef _MSC_VER
/* sscanf warning (not currently scanning for strings) */
#    pragma warning(disable : 4996)
#endif

const struct aws_byte_cursor g_s3_client_version = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL(AWS_S3_CLIENT_VERSION);
const struct aws_byte_cursor g_s3_service_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("s3");
const struct aws_byte_cursor g_s3express_service_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("s3express");
const struct aws_byte_cursor g_host_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Host");
const struct aws_byte_cursor g_range_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Range");
const struct aws_byte_cursor g_if_match_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("If-Match");
const struct aws_byte_cursor g_request_id_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-request-id");
const struct aws_byte_cursor g_amz_id_2_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-id-2");
const struct aws_byte_cursor g_etag_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ETag");
const struct aws_byte_cursor g_content_range_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-Range");
const struct aws_byte_cursor g_content_type_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-Type");
const struct aws_byte_cursor g_content_encoding_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-Encoding");
const struct aws_byte_cursor g_content_encoding_header_aws_chunked =
    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("aws-chunked");
const struct aws_byte_cursor g_content_length_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-Length");
const struct aws_byte_cursor g_decoded_content_length_header_name =
    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-decoded-content-length");
const struct aws_byte_cursor g_content_md5_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-MD5");
const struct aws_byte_cursor g_trailer_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-trailer");
const struct aws_byte_cursor g_request_validation_mode = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-mode");
const struct aws_byte_cursor g_enabled = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("enabled");

const struct aws_byte_cursor g_checksum_algorithm_header_name =
    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-algorithm");
const struct aws_byte_cursor g_sdk_checksum_algorithm_header_name =
    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-sdk-checksum-algorithm");
const struct aws_byte_cursor g_accept_ranges_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("accept-ranges");
const struct aws_byte_cursor g_acl_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-acl");
const struct aws_byte_cursor g_mp_parts_count_header_name =
    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-mp-parts-count");
const struct aws_byte_cursor g_post_method = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("POST");
const struct aws_byte_cursor g_head_method = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("HEAD");
const struct aws_byte_cursor g_delete_method = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("DELETE");

const struct aws_byte_cursor g_user_agent_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("User-Agent");
const struct aws_byte_cursor g_user_agent_header_product_name =
    AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("CRTS3NativeClient");
const struct aws_byte_cursor g_user_agent_header_platform = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("platform");
const struct aws_byte_cursor g_user_agent_header_unknown = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("unknown");

const uint32_t g_s3_max_num_upload_parts = 10000;
const size_t g_s3_min_upload_part_size = MB_TO_BYTES(5);

const double g_default_throughput_target_gbps = 10.0;

/**
 * Streaming buffer size selection based on experimental results on EBS:
 *
 * - Too small buffer sizes (e.g., 16KiB) impact disk read performance,
 *   achieving only 6.73 Gbps throughput from EBS.
 * - Too large buffer sizes cause network connections to starve more easily
 *   when disk reads cannot provide data fast enough.
 * - 1MiB buffer size provides optimal balance: sufficient disk read throughput
 *   while maintaining reasonable retry rates due to connection starvation.
 */
const size_t g_streaming_buffer_size = MB_TO_BYTES(1);

/**
 * The streaming approach reduces memory consumption without introducing unexpected errors
 * or performance degradation.
 *
 * We start streaming for objects larger than 1TiB, with plans to lower this threshold in future iterations.
 *
 * The 1TiB threshold was chosen to minimize the blast radius of this behavioral change
 * while still providing meaningful memory usage improvements for large objects.
 */
const uint64_t g_streaming_object_size_threshold = TB_TO_BYTES(1);

/**
 * TODO: update this default part size 17/16 MiB based on S3 best practice.
 * Default part size is 8 MiB to reach the best performance from the experiments we had.
 * Default max part size is 5GiB as the current server limit.
 **/
const uint64_t g_default_part_size_fallback = MB_TO_BYTES(8);
#define G_DEFAULT_MAX_PART_SIZE 5368709120ULL
const uint64_t g_default_max_part_size = G_DEFAULT_MAX_PART_SIZE;

/* TODO: Use a reasonable alignment with the update of the buffer pool */
const uint64_t g_s3_optimal_range_size_alignment = 1;
/**
 * The most parts in memory will be:
 * - All downloaded parts to deliver them in the right order (download)
 * - All parts read into memory for preparing for the HTTP level (upload)
 * - All transferring parts in HTTP
 * So, it gives us at most 3 × (the max range size) × (concurrency) in memory.
 */
static const uint32_t s_optimal_range_size_memory_divisor = 3;

/**
 * TODO: THIS IS A TEMP WORKAROUND, not the long term solution.
 * As in Nov 2025, S3Express recommended to have no more than 75 connections to one single part.
 *
 * However, client don't know the part boundaries unless with an extra call. Thus, these are the workarounds to avoid
 * the issue.
 * 1. If the Part Size we set is larger than the possible size to hit the limitation, we are safe to make as many
 * connections as we want.
 * 2. If the object size is less than the threshold, we keep our previous behavior, as it's less likely to hit the
 * server side limitation.
 */
#define G_S3EXPRESS_CONNECTION_LIMITATION 75
const uint32_t g_s3express_connection_limitation = G_S3EXPRESS_CONNECTION_LIMITATION;
const uint64_t g_s3express_connection_limitation_part_size_threshold =
    G_DEFAULT_MAX_PART_SIZE / G_S3EXPRESS_CONNECTION_LIMITATION;
#undef G_S3EXPRESS_CONNECTION_LIMITATION
#undef G_DEFAULT_MAX_PART_SIZE
const uint64_t g_s3express_connection_limitation_object_size_threshold = TB_TO_BYTES(4);

void copy_http_headers(const struct aws_http_headers *src, struct aws_http_headers *dest) {
    AWS_PRECONDITION(src);
    AWS_PRECONDITION(dest);

    size_t headers_count = aws_http_headers_count(src);

    for (size_t header_index = 0; header_index < headers_count; ++header_index) {
        struct aws_http_header header;

        aws_http_headers_get_index(src, header_index, &header);
        aws_http_headers_set(dest, header.name, header.value);
    }
}
/* user_data for XML traversal */
struct xml_get_body_at_path_traversal {
    struct aws_allocator *allocator;
    const char **path_name_array;
    size_t path_name_count;
    size_t path_name_i;
    struct aws_byte_cursor *out_body;
    bool found_node;
};

static int s_xml_get_body_at_path_on_node(struct aws_xml_node *node, void *user_data) {
    struct xml_get_body_at_path_traversal *traversal = user_data;

    /* if we already found what we're looking for, just finish parsing */
    if (traversal->found_node) {
        return AWS_OP_SUCCESS;
    }

    /* check if this node is on the path */
    struct aws_byte_cursor node_name = aws_xml_node_get_name(node);
    const char *expected_name = traversal->path_name_array[traversal->path_name_i];
    if (aws_byte_cursor_eq_c_str_ignore_case(&node_name, expected_name)) {

        bool is_final_node_on_path = traversal->path_name_i + 1 == traversal->path_name_count;
        if (is_final_node_on_path) {
            /* retrieve the body */
            if (aws_xml_node_as_body(node, traversal->out_body) != AWS_OP_SUCCESS) {
                return AWS_OP_ERR;
            }
            traversal->found_node = true;
            return AWS_OP_SUCCESS;
        } else {
            /* node is on path, but it's not the final node, so traverse its children */
            traversal->path_name_i++;
            if (aws_xml_node_traverse(node, s_xml_get_body_at_path_on_node, traversal) != AWS_OP_SUCCESS) {
                return AWS_OP_ERR;
            }
            traversal->path_name_i--;
            return AWS_OP_SUCCESS;
        }
    } else {
        /* this node is not on the path, continue parsing siblings */
        return AWS_OP_SUCCESS;
    }
}

int aws_xml_get_body_at_path(
    struct aws_allocator *allocator,
    struct aws_byte_cursor xml_doc,
    const char **path_name_array,
    struct aws_byte_cursor *out_body) {

    struct xml_get_body_at_path_traversal traversal = {
        .allocator = allocator,
        .path_name_array = path_name_array,
        .path_name_count = 0,
        .out_body = out_body,
    };

    /* find path_name_count */
    while (path_name_array[traversal.path_name_count] != NULL) {
        traversal.path_name_count++;
        AWS_ASSERT(traversal.path_name_count < 4); /* sanity check, increase cap if necessary */
    }
    AWS_ASSERT(traversal.path_name_count > 0);

    /* parse XML */
    struct aws_xml_parser_options parse_options = {
        .doc = xml_doc,
        .on_root_encountered = s_xml_get_body_at_path_on_node,
        .user_data = &traversal,
    };
    if (aws_xml_parse(allocator, &parse_options)) {
        goto error;
    }

    if (!traversal.found_node) {
        aws_raise_error(AWS_ERROR_STRING_MATCH_NOT_FOUND);
        goto error;
    }

    return AWS_OP_SUCCESS;
error:
    AWS_ZERO_STRUCT(*out_body);
    return AWS_OP_ERR;
}

struct aws_cached_signing_config_aws *aws_cached_signing_config_new(
    struct aws_s3_client *client,
    const struct aws_signing_config_aws *signing_config) {
    AWS_PRECONDITION(client);
    AWS_PRECONDITION(signing_config);

    struct aws_allocator *allocator = client->allocator;

    struct aws_cached_signing_config_aws *cached_signing_config =
        aws_mem_calloc(allocator, 1, sizeof(struct aws_cached_signing_config_aws));

    cached_signing_config->allocator = allocator;

    cached_signing_config->config.config_type =
        signing_config->config_type ? signing_config->config_type : AWS_SIGNING_CONFIG_AWS;

    AWS_ASSERT(aws_byte_cursor_is_valid(&signing_config->region));
    if (signing_config->region.len > 0) {
        cached_signing_config->region = aws_string_new_from_cursor(allocator, &signing_config->region);
    } else {
        /* Fall back to client region. */
        cached_signing_config->region = aws_string_new_from_string(allocator, client->region);
    }
    cached_signing_config->config.region = aws_byte_cursor_from_string(cached_signing_config->region);

    if (signing_config->service.len > 0) {
        cached_signing_config->service = aws_string_new_from_cursor(allocator, &signing_config->service);
        cached_signing_config->config.service = aws_byte_cursor_from_string(cached_signing_config->service);
    } else {
        cached_signing_config->config.service = g_s3_service_name;
    }

    cached_signing_config->config.date = signing_config->date;

    AWS_ASSERT(aws_byte_cursor_is_valid(&signing_config->signed_body_value));

    if (signing_config->signed_body_value.len > 0) {
        cached_signing_config->signed_body_value =
            aws_string_new_from_cursor(allocator, &signing_config->signed_body_value);
        cached_signing_config->config.signed_body_value =
            aws_byte_cursor_from_string(cached_signing_config->signed_body_value);
    } else {
        cached_signing_config->config.signed_body_value = g_aws_signed_body_value_unsigned_payload;
    }

    if (signing_config->credentials != NULL) {
        aws_credentials_acquire(signing_config->credentials);
        cached_signing_config->config.credentials = signing_config->credentials;
    }

    if (signing_config->credentials_provider != NULL) {
        aws_credentials_provider_acquire(signing_config->credentials_provider);
        cached_signing_config->config.credentials_provider = signing_config->credentials_provider;
    }

    /* Configs default to Zero. */
    cached_signing_config->config.algorithm = signing_config->algorithm;
    cached_signing_config->config.signature_type = signing_config->signature_type;
    /* TODO: you don't have a way to override this config as the other option is zero. But, you cannot really use the
     * other value, as it is always required. */
    cached_signing_config->config.signed_body_header = AWS_SBHT_X_AMZ_CONTENT_SHA256;
    cached_signing_config->config.should_sign_header = signing_config->should_sign_header;
    /* It's the user's responsibility to keep the user data around */
    cached_signing_config->config.should_sign_header_ud = signing_config->should_sign_header_ud;

    cached_signing_config->config.flags = signing_config->flags;
    cached_signing_config->config.expiration_in_seconds = signing_config->expiration_in_seconds;

    return cached_signing_config;
}

void aws_cached_signing_config_destroy(struct aws_cached_signing_config_aws *cached_signing_config) {
    if (cached_signing_config == NULL) {
        return;
    }

    aws_credentials_release(cached_signing_config->config.credentials);
    aws_credentials_provider_release(cached_signing_config->config.credentials_provider);

    aws_string_destroy(cached_signing_config->service);
    aws_string_destroy(cached_signing_config->region);
    aws_string_destroy(cached_signing_config->signed_body_value);

    aws_mem_release(cached_signing_config->allocator, cached_signing_config);
}

void aws_s3_init_default_signing_config(
    struct aws_signing_config_aws *signing_config,
    const struct aws_byte_cursor region,
    struct aws_credentials_provider *credentials_provider) {
    AWS_PRECONDITION(signing_config);
    AWS_PRECONDITION(credentials_provider);

    AWS_ZERO_STRUCT(*signing_config);

    signing_config->config_type = AWS_SIGNING_CONFIG_AWS;
    signing_config->algorithm = AWS_SIGNING_ALGORITHM_V4;
    signing_config->credentials_provider = credentials_provider;
    signing_config->region = region;
    signing_config->service = g_s3_service_name;
    signing_config->signed_body_header = AWS_SBHT_X_AMZ_CONTENT_SHA256;
    signing_config->signed_body_value = g_aws_signed_body_value_unsigned_payload;
}

struct aws_string *aws_strip_quotes(struct aws_allocator *allocator, struct aws_byte_cursor in_cur) {

    if (in_cur.len >= 2 && in_cur.ptr[0] == '"' && in_cur.ptr[in_cur.len - 1] == '"') {
        aws_byte_cursor_advance(&in_cur, 1);
        --in_cur.len;
    }

    return aws_string_new_from_cursor(allocator, &in_cur);
}

int aws_last_error_or_unknown(void) {
    int error = aws_last_error();
    AWS_ASSERT(error != AWS_ERROR_SUCCESS); /* Someone forgot to call aws_raise_error() */
    if (error == AWS_ERROR_SUCCESS) {
        return AWS_ERROR_UNKNOWN;
    }

    return error;
}

void aws_s3_add_user_agent_header(struct aws_allocator *allocator, struct aws_http_message *message) {
    AWS_PRECONDITION(allocator);
    AWS_PRECONDITION(message);

    const struct aws_byte_cursor space_delimiter = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL(" ");
    const struct aws_byte_cursor forward_slash = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("/");
    struct aws_byte_cursor platform_cursor = aws_s3_get_current_platform_ec2_intance_type(true /* cached_only */);
    if (!platform_cursor.len) {
        platform_cursor = g_user_agent_header_unknown;
    }
    const size_t user_agent_length = g_user_agent_header_product_name.len + forward_slash.len +
                                     g_s3_client_version.len + space_delimiter.len + g_user_agent_header_platform.len +
                                     forward_slash.len + platform_cursor.len;

    struct aws_http_headers *headers = aws_http_message_get_headers(message);
    AWS_ASSERT(headers != NULL);

    struct aws_byte_cursor current_user_agent_header;
    AWS_ZERO_STRUCT(current_user_agent_header);

    struct aws_byte_buf user_agent_buffer;
    AWS_ZERO_STRUCT(user_agent_buffer);

    if (aws_http_headers_get(headers, g_user_agent_header_name, &current_user_agent_header) == AWS_OP_SUCCESS) {
        /* If the header was found, then create a buffer with the total size we'll need, and append the current user
         * agent header with a trailing space. */
        aws_byte_buf_init(
            &user_agent_buffer, allocator, current_user_agent_header.len + space_delimiter.len + user_agent_length);

        aws_byte_buf_append_dynamic(&user_agent_buffer, &current_user_agent_header);

        aws_byte_buf_append_dynamic(&user_agent_buffer, &space_delimiter);

    } else {
        AWS_ASSERT(aws_last_error() == AWS_ERROR_HTTP_HEADER_NOT_FOUND);

        /* If the header was not found, then create a buffer with just the size of the user agent string that is about
         * to be appended to the buffer. */
        aws_byte_buf_init(&user_agent_buffer, allocator, user_agent_length);
    }

    /* Append the client's user-agent string. */
    {
        aws_byte_buf_append_dynamic(&user_agent_buffer, &g_user_agent_header_product_name);
        aws_byte_buf_append_dynamic(&user_agent_buffer, &forward_slash);
        aws_byte_buf_append_dynamic(&user_agent_buffer, &g_s3_client_version);
        aws_byte_buf_append_dynamic(&user_agent_buffer, &space_delimiter);
        aws_byte_buf_append_dynamic(&user_agent_buffer, &g_user_agent_header_platform);
        aws_byte_buf_append_dynamic(&user_agent_buffer, &forward_slash);
        aws_byte_buf_append_dynamic(&user_agent_buffer, &platform_cursor);
    }

    /* Apply the updated header. */
    aws_http_headers_set(headers, g_user_agent_header_name, aws_byte_cursor_from_buf(&user_agent_buffer));

    /* Clean up the scratch buffer. */
    aws_byte_buf_clean_up(&user_agent_buffer);
}

int aws_s3_parse_content_range_cursor(
    struct aws_byte_cursor content_range_cursor,
    uint64_t *out_range_start,
    uint64_t *out_range_end,
    uint64_t *out_object_size) {

    /* Expected Format of header is: "bytes StartByte-EndByte/TotalObjectSize" */

    /* Check if it starts with "bytes " */
    struct aws_byte_cursor bytes_prefix = aws_byte_cursor_from_c_str("bytes ");
    if (!aws_byte_cursor_starts_with(&content_range_cursor, &bytes_prefix)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER);
    }

    /* Skip past "bytes " */
    aws_byte_cursor_advance(&content_range_cursor, bytes_prefix.len);

    /* Parse range start */
    struct aws_byte_cursor range_start_cursor;
    AWS_ZERO_STRUCT(range_start_cursor);
    if (!aws_byte_cursor_next_split(&content_range_cursor, '-', &range_start_cursor)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER);
    }

    uint64_t range_start = 0;
    if (aws_byte_cursor_utf8_parse_u64(range_start_cursor, &range_start)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER);
    }
    /* Move the cursor to pass the `-` */
    aws_byte_cursor_advance(&content_range_cursor, range_start_cursor.len + 1);
    /* Parse range end */
    struct aws_byte_cursor range_end_cursor;
    AWS_ZERO_STRUCT(range_end_cursor);
    if (!aws_byte_cursor_next_split(&content_range_cursor, '/', &range_end_cursor)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER);
    }

    uint64_t range_end = 0;
    if (aws_byte_cursor_utf8_parse_u64(range_end_cursor, &range_end)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER);
    }

    /* Move the cursor to pass the `/` */
    aws_byte_cursor_advance(&content_range_cursor, range_end_cursor.len + 1);
    /* Parse object size (remaining part) */
    uint64_t object_size = 0;
    if (aws_byte_cursor_utf8_parse_u64(content_range_cursor, &object_size)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER);
    }

    /* Set output values */
    if (out_range_start != NULL) {
        *out_range_start = range_start;
    }

    if (out_range_end != NULL) {
        *out_range_end = range_end;
    }

    if (out_object_size != NULL) {
        *out_object_size = object_size;
    }

    return AWS_OP_SUCCESS;
}

int aws_s3_parse_content_range_response_header(
    struct aws_http_headers *response_headers,
    uint64_t *out_range_start,
    uint64_t *out_range_end,
    uint64_t *out_object_size) {
    AWS_ERROR_PRECONDITION(response_headers);
    struct aws_byte_cursor content_range_header_value;

    if (aws_http_headers_get(response_headers, g_content_range_header_name, &content_range_header_value)) {
        aws_raise_error(AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER);
        return AWS_OP_ERR;
    }
    return aws_s3_parse_content_range_cursor(
        content_range_header_value, out_range_start, out_range_end, out_object_size);
}

int aws_s3_parse_content_length_response_header(
    struct aws_allocator *allocator,
    struct aws_http_headers *response_headers,
    uint64_t *out_content_length) {
    AWS_PRECONDITION(allocator);
    AWS_PRECONDITION(response_headers);
    AWS_PRECONDITION(out_content_length);

    struct aws_byte_cursor content_length_header_value;

    if (aws_http_headers_get(response_headers, g_content_length_header_name, &content_length_header_value)) {
        aws_raise_error(AWS_ERROR_S3_MISSING_CONTENT_LENGTH_HEADER);
        return AWS_OP_ERR;
    }

    struct aws_string *content_length_header_value_str =
        aws_string_new_from_cursor(allocator, &content_length_header_value);

    int result = AWS_OP_ERR;

    if (sscanf((const char *)content_length_header_value_str->bytes, "%" PRIu64, out_content_length) == 1) {
        result = AWS_OP_SUCCESS;
    } else {
        aws_raise_error(AWS_ERROR_S3_INVALID_CONTENT_LENGTH_HEADER);
    }

    aws_string_destroy(content_length_header_value_str);
    return result;
}

int aws_s3_parse_request_range_header(
    struct aws_http_headers *request_headers,
    bool *out_has_start_range,
    bool *out_has_end_range,
    uint64_t *out_start_range,
    uint64_t *out_end_range) {

    AWS_PRECONDITION(request_headers);
    AWS_PRECONDITION(out_has_start_range);
    AWS_PRECONDITION(out_has_end_range);
    AWS_PRECONDITION(out_start_range);
    AWS_PRECONDITION(out_end_range);

    bool has_start_range = false;
    bool has_end_range = false;
    uint64_t start_range = 0;
    uint64_t end_range = 0;

    struct aws_byte_cursor range_header_value;

    if (aws_http_headers_get(request_headers, g_range_header_name, &range_header_value)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
    }

    struct aws_byte_cursor range_header_start = aws_byte_cursor_from_c_str("bytes=");

    /* verify bytes= */
    if (!aws_byte_cursor_starts_with(&range_header_value, &range_header_start)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
    }

    aws_byte_cursor_advance(&range_header_value, range_header_start.len);
    struct aws_byte_cursor substr = {0};
    /* parse start range */
    if (!aws_byte_cursor_next_split(&range_header_value, '-', &substr)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
    }
    if (substr.len > 0) {
        if (aws_byte_cursor_utf8_parse_u64(substr, &start_range)) {
            return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
        }
        has_start_range = true;
    }

    /* parse end range */
    if (!aws_byte_cursor_next_split(&range_header_value, '-', &substr)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
    }
    if (substr.len > 0) {
        if (aws_byte_cursor_utf8_parse_u64(substr, &end_range)) {
            return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
        }
        has_end_range = true;
    }

    /* verify that there is nothing extra */
    if (aws_byte_cursor_next_split(&range_header_value, '-', &substr)) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
    }

    /* verify that start-range <= end-range */
    if (has_end_range && start_range > end_range) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
    }

    /* verify that start-range or end-range is present */
    if (!has_start_range && !has_end_range) {
        return aws_raise_error(AWS_ERROR_S3_INVALID_RANGE_HEADER);
    }

    *out_has_start_range = has_start_range;
    *out_has_end_range = has_end_range;
    *out_start_range = start_range;
    *out_end_range = end_range;
    return AWS_OP_SUCCESS;
}

uint32_t aws_s3_calculate_auto_ranged_get_num_parts(
    size_t part_size,
    uint64_t first_part_size,
    uint64_t object_range_start,
    uint64_t object_range_end) {
    uint32_t num_parts = 1;

    if (first_part_size == 0) {
        return num_parts;
    }
    uint64_t second_part_start = object_range_start + first_part_size;

    /* If the range has room for a second part, calculate the additional amount of parts. */
    if (second_part_start <= object_range_end) {
        uint64_t aligned_range_remainder = object_range_end + 1 - second_part_start; /* range-end is inclusive */
        num_parts += (uint32_t)(aligned_range_remainder / (uint64_t)part_size);

        if ((aligned_range_remainder % part_size) > 0) {
            ++num_parts;
        }
    }

    return num_parts;
}

void aws_s3_calculate_auto_ranged_get_part_range(
    uint64_t object_range_start,
    uint64_t object_range_end,
    size_t part_size,
    uint64_t first_part_size,
    uint32_t part_number,
    uint64_t *out_part_range_start,
    uint64_t *out_part_range_end) {
    AWS_PRECONDITION(out_part_range_start);
    AWS_PRECONDITION(out_part_range_end);

    AWS_ASSERT(part_number > 0);

    const uint32_t part_index = part_number - 1;

    /* Part index is assumed to be in a valid range. */
    AWS_ASSERT(
        part_index <
        aws_s3_calculate_auto_ranged_get_num_parts(part_size, first_part_size, object_range_start, object_range_end));

    uint64_t part_size_uint64 = (uint64_t)part_size;

    if (part_index == 0) {
        /* If this is the first part, then use the first part size. */
        *out_part_range_start = object_range_start;
        *out_part_range_end = *out_part_range_start + first_part_size - 1;
    } else {
        /* Else, find the next part by adding the object range + total number of whole parts before this one + initial
         * part size*/
        *out_part_range_start = object_range_start + ((uint64_t)(part_index - 1)) * part_size_uint64 + first_part_size;
        *out_part_range_end = *out_part_range_start + part_size_uint64 - 1; /* range-end is inclusive */
    }

    /* Cap the part's range end using the object's range end. */
    if (*out_part_range_end > object_range_end) {
        *out_part_range_end = object_range_end;
    }
}

int aws_s3_calculate_optimal_mpu_part_size_and_num_parts(
    uint64_t content_length,
    size_t client_part_size,
    uint64_t client_max_part_size,
    size_t *out_part_size,
    uint32_t *out_num_parts) {

    AWS_FATAL_ASSERT(out_part_size);
    AWS_FATAL_ASSERT(out_num_parts);

    if (content_length == 0) {
        *out_part_size = 0;
        *out_num_parts = 0;
        return AWS_OP_SUCCESS;
    }

    uint64_t part_size_uint64 = content_length / (uint64_t)g_s3_max_num_upload_parts;

    if ((content_length % g_s3_max_num_upload_parts) > 0) {
        ++part_size_uint64;
    }

    if (part_size_uint64 > SIZE_MAX) {
        AWS_LOGF_ERROR(
            AWS_LS_S3_META_REQUEST,
            "Could not create meta request; required part size of %" PRIu64 " bytes is too large for platform.",
            part_size_uint64);

        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
    }

    size_t part_size = (size_t)part_size_uint64;

    if (part_size > client_max_part_size) {
        AWS_LOGF_ERROR(
            AWS_LS_S3_META_REQUEST,
            "Could not create meta request; required part size for request is %" PRIu64
            ", but current maximum part size is %" PRIu64,
            (uint64_t)part_size,
            (uint64_t)client_max_part_size);
        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
    }

    if (part_size < client_part_size) {
        part_size = client_part_size;
    }

    if (content_length < part_size) {
        /* When the content length is smaller than part size and larger than the threshold, we set one part
         * with the whole length */
        part_size = (size_t)content_length;
    }

    uint32_t num_parts = (uint32_t)(content_length / part_size);
    if ((content_length % part_size) > 0) {
        ++num_parts;
    }
    AWS_ASSERT(num_parts <= g_s3_max_num_upload_parts);

    *out_part_size = part_size;
    *out_num_parts = num_parts;
    return AWS_OP_SUCCESS;
}

int aws_s3_crt_error_code_from_recoverable_server_error_code_string(struct aws_byte_cursor error_code_string) {
    if (aws_byte_cursor_eq_c_str_ignore_case(&error_code_string, "SlowDown")) {
        return AWS_ERROR_S3_SLOW_DOWN;
    }
    if (aws_byte_cursor_eq_c_str_ignore_case(&error_code_string, "InternalError") ||
        aws_byte_cursor_eq_c_str_ignore_case(&error_code_string, "InternalErrors")) {
        return AWS_ERROR_S3_INTERNAL_ERROR;
    }
    if (aws_byte_cursor_eq_c_str_ignore_case(&error_code_string, "RequestTimeTooSkewed")) {
        return AWS_ERROR_S3_REQUEST_TIME_TOO_SKEWED;
    }

    if (aws_byte_cursor_eq_c_str_ignore_case(&error_code_string, "RequestTimeout")) {
        return AWS_ERROR_S3_REQUEST_TIMEOUT;
    }

    return AWS_ERROR_UNKNOWN;
}

void aws_s3_request_finish_up_metrics_synced(struct aws_s3_request *request, struct aws_s3_meta_request *meta_request) {
    AWS_PRECONDITION(meta_request);
    AWS_PRECONDITION(request);
    ASSERT_SYNCED_DATA_LOCK_HELD(meta_request);

    if (request->send_data.metrics != NULL) {
        /* Request is done, complete the metrics for the request now. */
        struct aws_s3_request_metrics *metrics = request->send_data.metrics;
        aws_high_res_clock_get_ticks((uint64_t *)&metrics->time_metrics.end_timestamp_ns);
        metrics->time_metrics.total_duration_ns =
            metrics->time_metrics.end_timestamp_ns - metrics->time_metrics.start_timestamp_ns;

        if (meta_request->telemetry_callback != NULL) {
            struct aws_s3_meta_request_event event = {.type = AWS_S3_META_REQUEST_EVENT_TELEMETRY};
            event.u.telemetry.metrics = aws_s3_request_metrics_acquire(metrics);
            aws_s3_meta_request_add_event_for_delivery_synced(meta_request, &event);
        }
        request->send_data.metrics = aws_s3_request_metrics_release(metrics);
    }
}

int aws_s3_check_headers_for_checksum(
    struct aws_s3_meta_request *meta_request,
    const struct aws_http_headers *headers,
    struct aws_s3_checksum **out_checksum,
    struct aws_byte_buf *out_checksum_buffer,
    bool meta_request_level) {
    AWS_PRECONDITION(meta_request);
    AWS_PRECONDITION(out_checksum);
    AWS_PRECONDITION(out_checksum_buffer);

    if (!headers || aws_http_headers_count(headers) == 0) {
        *out_checksum = NULL;
        return AWS_OP_SUCCESS;
    }
    if (meta_request_level && aws_http_headers_has(headers, g_mp_parts_count_header_name)) {
        /* g_mp_parts_count_header_name indicates it's a object was uploaded as a
         * multipart upload. So, the checksum should not be applied to the meta request level.
         * But we we want to check it for the request level. */
        *out_checksum = NULL;
        return AWS_OP_SUCCESS;
    }

    for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) {
        enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i];
        if (!aws_s3_meta_request_checksum_config_has_algorithm(meta_request, algorithm)) {
            /* If user doesn't select this algorithm, skip */
            continue;
        }
        const struct aws_byte_cursor algorithm_header_name =
            aws_get_http_header_name_from_checksum_algorithm(algorithm);
        struct aws_byte_cursor checksum_value;
        if (aws_http_headers_get(headers, algorithm_header_name, &checksum_value) == AWS_OP_SUCCESS) {
            /* Found the checksum header, keep the header value and initialize the running checksum */
            size_t encoded_len = 0;
            aws_base64_compute_encoded_len(aws_get_digest_size_from_checksum_algorithm(algorithm), &encoded_len);
            if (checksum_value.len == encoded_len) {
                aws_byte_buf_init_copy_from_cursor(out_checksum_buffer, meta_request->allocator, checksum_value);
                *out_checksum = aws_checksum_new(meta_request->allocator, algorithm);
                if (!*out_checksum) {
                    AWS_LOGF_ERROR(
                        AWS_LS_S3_META_REQUEST,
                        "Could not create checksum for algorithm: %d, due to error code %d (%s)",
                        algorithm,
                        aws_last_error_or_unknown(),
                        aws_error_str(aws_last_error_or_unknown()));
                    return AWS_OP_ERR;
                }
                return AWS_OP_SUCCESS;
            }
            break;
        }
    }
    *out_checksum = NULL;
    return AWS_OP_SUCCESS;
}

int aws_s3_calculate_client_optimal_range_size(
    uint64_t memory_limit_in_bytes,
    uint32_t max_connections,
    uint64_t *out_client_optimal_range_size) {

    AWS_ERROR_PRECONDITION(out_client_optimal_range_size);

    /* Validate input parameters */
    if (memory_limit_in_bytes == 0 || max_connections == 0) {
        AWS_LOGF_ERROR(
            AWS_LS_S3_GENERAL,
            "Invalid parameters for client optimal range size calculation: memory_limit=%" PRIu64
            ", max_connections=%" PRIu32,
            memory_limit_in_bytes,
            max_connections);
        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
    }

    /* Calculate memory-constrained range size using the formula:
     * MemoryLimit / concurrency / s_optimal_range_size_memory_divisor
     *
     * The division by s_optimal_range_size_memory_divisor accounts for:
     * - All downloaded parts to deliver them in the right order (download)
     * - All parts read into memory for preparing for the HTTP level (upload)
     * - All transferring parts in HTTP
     */
    uint64_t memory_constrained_size = memory_limit_in_bytes / max_connections / s_optimal_range_size_memory_divisor;

    /* Apply minimum constraint first */
    uint64_t optimal_size = memory_constrained_size;
    if (optimal_size < g_default_part_size_fallback) {
        optimal_size = g_default_part_size_fallback;
    }
    /* Apply maximum constraint */
    if (optimal_size > g_default_max_part_size) {
        optimal_size = g_default_max_part_size;
    }

    *out_client_optimal_range_size = optimal_size;

    AWS_LOGF_DEBUG(
        AWS_LS_S3_GENERAL,
        "Calculated client optimal range size: memory_limit=%" PRIu64 ", max_connections=%" PRIu32
        ", client_optimal_range_size=%" PRIu64,
        memory_limit_in_bytes,
        max_connections,
        optimal_size);

    return AWS_OP_SUCCESS;
}

int aws_s3_calculate_request_optimal_range_size(
    uint64_t client_optimal_range_size,
    uint64_t estimated_object_stored_part_size,
    bool is_express,
    uint64_t *out_request_optimal_range_size) {

    AWS_ERROR_PRECONDITION(out_request_optimal_range_size);

    /* Validate input parameters */
    if (client_optimal_range_size == 0) {
        AWS_LOGF_ERROR(
            AWS_LS_S3_GENERAL,
            "Invalid client_optimal_range_size for request optimal range size calculation: %" PRIu64,
            client_optimal_range_size);
        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
    }

    /* Apply the minimum constraint from the formula:
     * min(client_optimal_range_size, estimated_object_stored_part_size)
     * If estimated_object_stored_part_size is 0 (unknown), use client_optimal_range_size
     */
    uint64_t optimal_size = client_optimal_range_size;
    if (estimated_object_stored_part_size > 0 && estimated_object_stored_part_size < client_optimal_range_size) {
        optimal_size = estimated_object_stored_part_size;
    }
    /* Apply minimum constraint first to avoid excessive alignment */
    if (optimal_size < g_default_part_size_fallback) {
        optimal_size = g_default_part_size_fallback;
    }
    /* Apply a reasonable upper bound to this. The goal to increase the part size is to have less connection to hit one
     * single part from server so that we are not bottleneck by the server throughput on one part */
    if (is_express) {
        /* As in 2025, each part in S3 express can provide over 100 Gbps throughput.
         *
         * Each S3express part can provide a much high throughput than we currently asking for (More than 100Gbps per
         * part throughput, which means it in theory can handle 100 concurrent connections to 1 part). So, 128MiB
         * with 5GiB max part size gives 40 connections to hit one part. Have larger range(less connections to hit one
         * part from server) won't help the goal mentioned above. */
        optimal_size = aws_min_u64(optimal_size, MB_TO_BYTES(128));
    } else {
        /* As in 2025, each part in S3 general bucket can provide around 10 Gbps throughput.
         * Use 2GiB to below the INT32_MAX. Given the 5GiB max part size */
        optimal_size = aws_min_u64(optimal_size, GB_TO_BYTES(2));
    }

    *out_request_optimal_range_size = optimal_size;

    AWS_LOGF_TRACE(
        AWS_LS_S3_GENERAL,
        "Calculated request optimal range size: client_optimal_range_size=%" PRIu64 ", estimated_part_size=%" PRIu64
        ", request_optimal_range_size=%" PRIu64,
        client_optimal_range_size,
        estimated_object_stored_part_size,
        optimal_size);

    return AWS_OP_SUCCESS;
}

static bool s_is_quote_or_space_char(uint8_t c) {
    return c == '"' || c == ' ';
}

int aws_s3_extract_parts_from_etag(struct aws_byte_cursor etag_header_value, uint32_t *out_num_parts) {

    AWS_ERROR_PRECONDITION(out_num_parts);
    /* Strip quotes if present (ETags often come wrapped in quotes) */
    struct aws_byte_cursor etag_cursor = aws_byte_cursor_trim_pred(&etag_header_value, s_is_quote_or_space_char);

    /* Handle empty or invalid ETag */
    if (etag_cursor.len == 0) {
        AWS_LOGF_ERROR(AWS_LS_S3_GENERAL, "Empty ETag header value");
        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
    }

    /* Use aws_byte_cursor_next_split to iterate through dash-separated parts */
    struct aws_byte_cursor substr = {0};
    struct aws_byte_cursor remaining_cursor = etag_cursor;
    int split_count = 0;
    struct aws_byte_cursor parts_count = {0};

    /* Count splits and extract parts */
    while (aws_byte_cursor_next_split(&remaining_cursor, '-', &substr)) {
        split_count++;
        if (split_count == 2) {
            /**
             * The ETag should follow the pattern <hash>-<parts_count>, so the second part is the parts count.
             * The S3 ETag will not have `-` in the hash value, as it's a HEX string.
             **/
            parts_count = substr;
        }
        if (split_count > 2) {
            /* More than 2 parts - invalid format */
            AWS_LOGF_ERROR(
                AWS_LS_S3_GENERAL,
                "Invalid ETag format - multiple dashes found: " PRInSTR,
                AWS_BYTE_CURSOR_PRI(etag_header_value));
            return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
        }
    }

    if (split_count == 1) {
        /* No dash found - this is a single-part upload */
        *out_num_parts = 1;
        AWS_LOGF_TRACE(
            AWS_LS_S3_GENERAL, "Single-part ETag detected (no dash): " PRInSTR, AWS_BYTE_CURSOR_PRI(etag_cursor));
        return AWS_OP_SUCCESS;
    } else if (split_count == 2) {
        /* Exactly one dash - multipart upload format */
        /* Parse the number after the dash */
        uint64_t num_parts_u64;
        if (aws_byte_cursor_utf8_parse_u64(parts_count, &num_parts_u64)) {
            AWS_LOGF_ERROR(
                AWS_LS_S3_GENERAL,
                "Invalid ETag format - could not parse number of parts: " PRInSTR,
                AWS_BYTE_CURSOR_PRI(parts_count));
            return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
        }

        /* Validate the number is within reasonable bounds */
        if (num_parts_u64 == 0 || num_parts_u64 > g_s3_max_num_upload_parts) {
            AWS_LOGF_ERROR(
                AWS_LS_S3_GENERAL,
                "Invalid number of parts in ETag: %" PRIu64 " (must be 1-%" PRIu32 ")",
                num_parts_u64,
                g_s3_max_num_upload_parts);
            return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
        }

        *out_num_parts = (uint32_t)num_parts_u64;
    } else {
        /* Should not reach here due to early exit above, but handle just in case */
        AWS_LOGF_ERROR(AWS_LS_S3_GENERAL, "Unexpected split count: %d", split_count);
        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
    }

    AWS_LOGF_TRACE(
        AWS_LS_S3_GENERAL,
        "Extracted %" PRIu32 " parts from ETag: " PRInSTR,
        *out_num_parts,
        AWS_BYTE_CURSOR_PRI(etag_header_value));

    return AWS_OP_SUCCESS;
}

static const struct aws_byte_cursor s_checksum_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-");

bool aws_s3_is_checksum_value_header_name(struct aws_byte_cursor header_name) {
    return aws_byte_cursor_starts_with_ignore_case(&header_name, &s_checksum_prefix) &&
           !aws_byte_cursor_eq_c_str_ignore_case(&header_name, "x-amz-checksum-type") &&
           !aws_byte_cursor_eq_c_str_ignore_case(&header_name, "x-amz-checksum-mode") &&
           !aws_byte_cursor_eq_c_str_ignore_case(&header_name, "x-amz-checksum-algorithm");
}