Skip to content

S3Express CreateSession Allowlist Headers #492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Jan 27, 2025
Merged

S3Express CreateSession Allowlist Headers #492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
f8ffac1
piping the headers
waahm7 Jan 22, 2025
8076511
More fixes
waahm7 Jan 22, 2025
dfc4a8d
null safety
waahm7 Jan 22, 2025
b3f48c7
lint
waahm7 Jan 22, 2025
7d6b98a
Add test and update mock server
waahm7 Jan 23, 2025
ff8577b
update readme
waahm7 Jan 23, 2025
bea7dc4
Fix tests
waahm7 Jan 23, 2025
30e9a18
Add hash key test
waahm7 Jan 24, 2025
2d74e6e
Add support for session mode
waahm7 Jan 24, 2025
73c5fc2
Cleanup
waahm7 Jan 24, 2025
e74da12
Add experimental warning
waahm7 Jan 24, 2025
c7975d2
lint
waahm7 Jan 24, 2025
0b07914
add type hint
waahm7 Jan 24, 2025
54cecca
fix codecov action
waahm7 Jan 24, 2025
2ccc3b2
init and cleanup
waahm7 Jan 24, 2025
a9507a5
try the new host override
waahm7 Jan 24, 2025
decf5f0
fix server
waahm7 Jan 24, 2025
2d93aef
try setting port
waahm7 Jan 24, 2025
94fec48
authority instead of host
waahm7 Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ concurrency:
cancel-in-progress: true

env:
BUILDER_VERSION: v0.9.72
BUILDER_VERSION: v0.9.74
BUILDER_SOURCE: releases
BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
PACKAGE_NAME: aws-c-s3
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/codecov.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:


env:
BUILDER_VERSION: v0.9.72
BUILDER_VERSION: v0.9.74
BUILDER_SOURCE: releases
BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
PACKAGE_NAME: aws-c-s3
Expand All @@ -30,4 +30,4 @@ jobs:
run: |
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
chmod a+x builder
./builder build -p ${{ env.PACKAGE_NAME }} --compiler=gcc-12 --cmake-extra=-DASSERT_LOCK_HELD=ON --coverage --coverage-exclude=source/s3_copy_object.c
./builder build -p ${{ env.PACKAGE_NAME }} --compiler=gcc --cmake-extra=-DASSERT_LOCK_HELD=ON --coverage --coverage-exclude=source/s3_copy_object.c
6 changes: 6 additions & 0 deletions include/aws/s3/private/s3_request_messages.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,12 @@ extern const size_t g_s3_complete_multipart_upload_excluded_headers_count;
AWS_S3_API
extern const struct aws_byte_cursor g_s3_abort_multipart_upload_excluded_headers[];

AWS_S3_API
extern const size_t g_s3_create_session_allowed_headers_count;

AWS_S3_API
extern const struct aws_byte_cursor g_s3_create_session_allowed_headers[];

AWS_S3_API
extern const size_t g_s3_abort_multipart_upload_excluded_headers_count;

Expand Down
13 changes: 10 additions & 3 deletions include/aws/s3/private/s3express_credentials_provider_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ struct aws_s3express_session {
/* The region and host of the session */
struct aws_string *region;
struct aws_string *host;

struct aws_http_headers *headers;
bool inactive;

/* Only used for mock tests */
Expand Down Expand Up @@ -105,14 +107,19 @@ struct aws_s3express_credentials_provider *aws_s3express_credentials_provider_ne
const struct aws_s3express_credentials_provider_default_options *options);

/**
* Encode the hash key to be [host_value][hash_of_credentials]
* hash_of_credentials is the sha256 of [access_key][secret_access_key]
* Encodes the hash key in the format: [host_value][hash_of_credentials_and_headers]
*
* The hash_of_credentials_and_headers is calculated as follows:
* 1. Concatenate: [access_key][secret_access_key][headers]
* where headers = ",header_name1:header_value1,header_name2:header_value2..."
* 2. Generates SHA256 hash of the concatenated string
*/
AWS_S3_API
struct aws_string *aws_encode_s3express_hash_key_new(
struct aws_allocator *allocator,
const struct aws_credentials *original_credentials,
struct aws_byte_cursor host_value);
struct aws_byte_cursor host_value,
struct aws_http_headers *headers);

AWS_EXTERN_C_END
#endif /* AWS_S3EXPRESS_CREDENTIALS_PROVIDER_IMPL_H */
4 changes: 4 additions & 0 deletions include/aws/s3/s3_client.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,10 @@ struct aws_s3_client_config {
* If set, client will invoke the factory to get the provider to use, when needed.
*
* If not set, client will create a default S3 Express provider under the hood.
*
* NOTE: THE FOLLOWING BEHAVIOR IS EXPERIMENTAL AND UNSTABLE
* Default S3 Express provider will pass the headers allowed in `g_s3_create_session_allowed_headers` to the
* CreateSession call.
*/
aws_s3express_provider_factory_fn *s3express_provider_override_factory;
void *factory_user_data;
Expand Down
2 changes: 2 additions & 0 deletions include/aws/s3/s3express_credentials_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ struct aws_credentials_properties_s3express {
* If empty, the region of the S3 client will be used.
*/
struct aws_byte_cursor region;

struct aws_http_headers *headers;
};

struct aws_s3express_credentials_provider_vtable {
Expand Down
1 change: 1 addition & 0 deletions source/s3_client.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1030,6 +1030,7 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
}

endpoint_host_name = aws_string_new_from_cursor(client->allocator, aws_uri_host_name(&host_uri));
port = aws_uri_port(&host_uri);
aws_uri_clean_up(&host_uri);
}

Expand Down
1 change: 1 addition & 0 deletions source/s3_meta_request.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,7 @@ void aws_s3_meta_request_sign_request_default_impl(
context->user_data = user_data;
context->properties.host = aws_byte_cursor_from_string(meta_request->s3express_session_host);
context->properties.region = signing_config.region;
context->properties.headers = aws_http_message_get_headers(meta_request->initial_request_message);

if (signing_config.credentials) {
context->original_credentials = signing_config.credentials;
Expand Down
17 changes: 17 additions & 0 deletions source/s3_request_messages.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const struct aws_byte_cursor g_s3_create_multipart_upload_excluded_headers[] = {
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sse headers passed to create, upload and complete? seems kinda counter intuitive

Copy link
Contributor Author

@waahm7 waahm7 Jan 24, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, sse headers can be the same as in CreateMPU and Upload Part. From CreateMultipart Docs:
"""
In the Zonal endpoint API calls (except CopyObject and UploadPartCopy) using the REST API, the encryption request headers must match the encryption settings that are specified in the CreateSession request. You can't override the values of the encryption settings (x-amz-server-side-encryption, x-amz-server-side-encryption-aws-kms-key-id, x-amz-server-side-encryption-context, and x-amz-server-side-encryption-bucket-key-enabled) that are specified in the CreateSession request. You don't need to explicitly specify these encryption settings values in Zonal endpoint API calls, and Amazon S3 will use the encryption settings values from the CreateSession request to protect new objects in the directory bucket.
"""

We can exclude them but it feels like needless complexity trying to differentiate between UploadPart for S3 and UploadPart for S3Express after CreateSession.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They are always excluded in the completeMPU.

};

const size_t g_s3_create_multipart_upload_excluded_headers_count =
Expand Down Expand Up @@ -62,6 +63,7 @@ const struct aws_byte_cursor g_s3_upload_part_excluded_headers[] = {
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
};

const size_t g_s3_upload_part_excluded_headers_count = AWS_ARRAY_SIZE(g_s3_upload_part_excluded_headers);
Expand Down Expand Up @@ -96,6 +98,7 @@ const struct aws_byte_cursor g_s3_complete_multipart_upload_excluded_headers[] =
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-mp-object-size"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
};

const size_t g_s3_complete_multipart_upload_excluded_headers_count =
Expand Down Expand Up @@ -131,6 +134,7 @@ const struct aws_byte_cursor g_s3_complete_multipart_upload_with_checksum_exclud
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-sdk-checksum-algorithm"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-mp-object-size"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
};

const struct aws_byte_cursor g_s3_list_parts_excluded_headers[] = {
Expand Down Expand Up @@ -162,6 +166,7 @@ const struct aws_byte_cursor g_s3_list_parts_excluded_headers[] = {
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-legal-hold"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
};

const size_t g_s3_list_parts_excluded_headers_count = AWS_ARRAY_SIZE(g_s3_list_parts_excluded_headers);
Expand Down Expand Up @@ -192,6 +197,7 @@ const struct aws_byte_cursor g_s3_list_parts_with_checksum_excluded_headers[] =
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-legal-hold"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
};

const size_t g_s3_list_parts_with_checksum_excluded_headers_count =
Expand Down Expand Up @@ -227,8 +233,19 @@ const struct aws_byte_cursor g_s3_abort_multipart_upload_excluded_headers[] = {
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
};

const struct aws_byte_cursor g_s3_create_session_allowed_headers[] = {
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-create-session-mode"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-server-side-encryption"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-server-side-encryption-aws-kms-key-id"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-server-side-encryption-context"),
AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-server-side-encryption-bucket-key-enabled"),
};

const size_t g_s3_create_session_allowed_headers_count = AWS_ARRAY_SIZE(g_s3_create_session_allowed_headers);

static const struct aws_byte_cursor s_x_amz_meta_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-meta-");

static const struct aws_byte_cursor s_checksum_type_header =
Expand Down
Loading