-
-
Notifications
You must be signed in to change notification settings - Fork 665
implement sigv4 signing for s3 downloads #21956
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
566cc79
c81b212
c3d701c
ac45865
44fbac0
acef24b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
|
@@ -15,6 +15,7 @@ | |||
DownloadS3SchemeURL, | ||||
) | ||||
from pants.backend.url_handlers.s3.register import rules as s3_rules | ||||
from pants.backend.url_handlers.s3.subsystem import S3AuthSigning | ||||
from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest | ||||
from pants.engine.fs import Digest, FileDigest, NativeDownloadFile, Snapshot | ||||
from pants.engine.rules import QueryRule | ||||
|
@@ -44,11 +45,15 @@ def rule_runner() -> RuleRunner: | |||
) | ||||
|
||||
|
||||
class NoCredentialsError(Exception): | ||||
pass | ||||
|
||||
|
||||
@pytest.fixture | ||||
def monkeypatch_botocore(monkeypatch): | ||||
def do_patching(expected_url): | ||||
def do_patching(expected_auth_url): | ||||
botocore = SimpleNamespace() | ||||
botocore.exceptions = SimpleNamespace(NoCredentialsError=Exception) | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use of Exception here was causing some tests to pass (through this catch
|
||||
botocore.exceptions = SimpleNamespace(NoCredentialsError=NoCredentialsError) | ||||
|
||||
class FakeSession: | ||||
def __init__(self): | ||||
|
@@ -58,6 +63,9 @@ def __init__(self): | |||
def set_config_variable(self, key, value): | ||||
self.config_vars.update({key: value}) | ||||
|
||||
def get_config_variable(self, key): | ||||
return self.config_vars.get(key) or "us-east-1" | ||||
|
||||
def get_credentials(self): | ||||
if self.creds: | ||||
return self.creds | ||||
|
@@ -91,14 +99,24 @@ def load_credentials(self): | |||
Credentials=FakeCredentials.create, | ||||
) | ||||
|
||||
def fake_auth_ctor(creds): | ||||
def fake_auth_ctor(creds, service_name, region_name): | ||||
assert service_name == "s3" | ||||
assert region_name in ["us-east-1", "us-west-2"] | ||||
|
||||
def add_auth(request): | ||||
assert request.url == expected_auth_url | ||||
request.headers["AUTH"] = "TOKEN" | ||||
|
||||
return SimpleNamespace(add_auth=add_auth) | ||||
|
||||
def fake_hmac_v1_auth_ctor(creds): | ||||
def add_auth(request): | ||||
request.url == expected_url | ||||
assert request.url == expected_auth_url | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. missing assert |
||||
request.headers["AUTH"] = "TOKEN" | ||||
|
||||
return SimpleNamespace(add_auth=add_auth) | ||||
|
||||
botocore.auth = SimpleNamespace(HmacV1Auth=fake_auth_ctor) | ||||
botocore.auth = SimpleNamespace(SigV4Auth=fake_auth_ctor, HmacV1Auth=fake_hmac_v1_auth_ctor) | ||||
|
||||
monkeypatch.setitem(sys.modules, "botocore", botocore) | ||||
|
||||
|
@@ -121,69 +139,149 @@ def new_init(self, **kwargs): | |||
|
||||
|
||||
@pytest.mark.parametrize( | ||||
"request_url, expected_auth_url, expected_native_url, req_type", | ||||
"request_url, expected_auth_url, expected_native_url, req_type, auth_type", | ||||
[ | ||||
( | ||||
"s3://bucket/keypart1/keypart2/file.txt", | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. under hmacv1 the url thats signed is the s3 path style (which can be different that what the request is actually made with) |
||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3SchemeURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"s3://bucket/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. for sigv4 is the same url we make the request with (virtual host style) |
||||
DownloadS3SchemeURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
( | ||||
"s3://bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3SchemeURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"s3://bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3SchemeURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
# Path-style | ||||
( | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
( | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
( | ||||
"https://s3.us-west-2.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://s3.us-west-2.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://s3.us-west-2.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
( | ||||
"https://s3.us-west-2.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://s3.us-west-2.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://s3.us-west-2.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityPathStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
# Virtual-hosted-style | ||||
( | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
( | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
( | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://s3.amazonaws.com/bucket/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
"https://bucket.s3.amazonaws.com/keypart1/keypart2/file.txt?versionId=ABC123", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
( | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://s3.us-west-2.amazonaws.com/bucket/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.HMACV1, | ||||
), | ||||
( | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
"https://bucket.s3.us-west-2.amazonaws.com/keypart1/keypart2/file.txt", | ||||
DownloadS3AuthorityVirtualHostedStyleURL, | ||||
S3AuthSigning.SIGV4, | ||||
), | ||||
], | ||||
) | ||||
|
@@ -194,6 +292,7 @@ def test_download_s3( | |||
expected_auth_url: str, | ||||
expected_native_url: str, | ||||
req_type: type, | ||||
auth_type: S3AuthSigning, | ||||
replace_url, | ||||
) -> None: | ||||
class S3HTTPHandler(BaseHTTPRequestHandler): | ||||
|
@@ -213,6 +312,12 @@ def send_headers(self): | |||
self.send_header("Content-Length", f"{len(self.response_text)}") | ||||
self.end_headers() | ||||
|
||||
rule_runner.set_options( | ||||
args=[ | ||||
f"--s3-url-handler-auth-signing={auth_type.value}", | ||||
], | ||||
) | ||||
|
||||
monkeypatch_botocore(expected_auth_url) | ||||
with http_server(S3HTTPHandler) as port: | ||||
replace_url(expected_native_url, port) | ||||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
renamed to match the parameter in the tests cases below