Skip to content

(PTFE-3070) Add multi part upload for artifact #214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
matthiasL-scality wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

(PTFE-3070) Add multi part upload for artifact #214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
71cca9d
(PTFE-3070) Add multi part upload for artifact
matthiasL-scality Mar 18, 2026
4683280
Redirect upload to s3
matthiasL-scality Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 235 additions & 0 deletions conf/nginx.conf.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,6 +764,241 @@ http {
proxy_pass __S3_ENDPOINT_PROTO__$aws_tgt_bucket/$encoded_key;
}

# Return a presigned S3 PUT URL for a single-file upload.
# GET /presign-upload/<build>/<path>
# Responds 200 with the presigned URL as plain text so the client can PUT
# the file body directly to S3, bypassing this proxy entirely.
#
location ~ ^/presign-upload/([^/]+/.*[^/])$ {
if ($request_method != GET) {
return 400;
}

set_by_lua_file $canonical_path /etc/nginx/canonicalize_path.ljbc;

if ($canonical_path !~ '^[__SUPPORTED_CHARSET__]+$') {
return 400;
}

if ($canonical_path !~ '^([^/]+\:staging\-([0-9]{10}\.|)[0-9a-f]+\.[^./]+\.[0-9]+(\.[0-9]+|)|\.md_staging)/') {
return 400;
}

set $signature_mode "PRESIGN_PUT";
set $redirect_endpoint "__S3_ENDPOINT_PROTO____S3_ENDPOINT_HOST__:__S3_ENDPOINT_PORT__";
set $aws_access_key "";
set $aws_tgt_bucket "";
set $encoded_key "";
set $presign_query_string "";
rewrite_by_lua_file /etc/nginx/compute_aws_s3_signature.ljbc;
}

# Return a presigned S3 PUT URL for one multipart part.
# GET /presign-upload-part/<build>/<path>?partNumber=N&uploadId=X
# Responds 200 with the presigned URL as plain text.
#
location ~ ^/presign-upload-part/([^/]+/.*[^/])$ {
if ($request_method != GET) {
return 400;
}

set_by_lua_file $canonical_path /etc/nginx/canonicalize_path.ljbc;

if ($canonical_path !~ '^[__SUPPORTED_CHARSET__]+$') {
return 400;
}

if ($canonical_path !~ '^([^/]+\:staging\-([0-9]{10}\.|)[0-9a-f]+\.[^./]+\.[0-9]+(\.[0-9]+|)|\.md_staging)/') {
return 400;
}

set $signature_mode "PRESIGN_PART";
set $redirect_endpoint "__S3_ENDPOINT_PROTO____S3_ENDPOINT_HOST__:__S3_ENDPOINT_PORT__";
set $aws_access_key "";
set $aws_tgt_bucket "";
set $encoded_key "";
set $presign_query_string "";
rewrite_by_lua_file /etc/nginx/compute_aws_s3_signature.ljbc;
}

# Multipart upload: initiate.
# POST /upload-multipart/initiate/<build>/<path>
# Proxies POST /<key>?uploads to S3; returns XML with uploadId.
#
location ~ ^/upload-multipart/initiate/([^/]+/.*[^/])$ {
if ($request_method != POST) {
return 400;
}

set_by_lua_block $canonical_path {
local tmp = ngx.var.request_uri
tmp = tmp:gsub('?.*$', '', 1)
tmp = tmp:gsub('^/[^/]+/[^/]+/', '', 1)
tmp = tmp:gsub('+', '%%2B')
tmp = tmp:gsub('&amp;', '%%26')
tmp = ngx.unescape_uri(tmp)
return tmp
}

if ($canonical_path !~ '^[__SUPPORTED_CHARSET__]+$') {
return 400;
}

if ($canonical_path !~ '^([^/]+\:staging\-([0-9]{10}\.|)[0-9a-f]+\.[^./]+\.[0-9]+(\.[0-9]+|)|\.md_staging)/') {
return 400;
}

set $signature_mode "MULTIPART_INITIATE";
set $aws_access_key "";
set $aws_signature "";
set $x_amz_date "";
set $x_amz_acl "";
set $aws_tgt_bucket "${AWS_BUCKET_PREFIX}-staging";
set $encoded_key "";
rewrite_by_lua_file /etc/nginx/compute_aws_s3_signature.ljbc;

proxy_set_header Authorization "AWS $aws_access_key:$aws_signature";
proxy_set_header x-amz-date $x_amz_date;
proxy_set_header x-amz-acl $x_amz_acl;
proxy_set_header Host $aws_tgt_bucket.__S3_ENDPOINT_HOST__;
proxy_set_header Connection "keep-alive";

proxy_pass __S3_ENDPOINT_PROTO__$aws_tgt_bucket/$encoded_key?uploads;
}

# Multipart upload: upload one part.
# PUT /upload-multipart/part/<build>/<path>?partNumber=N&uploadId=X
# Proxies PUT /<key>?partNumber=N&uploadId=X to S3; returns ETag header.
#
location ~ ^/upload-multipart/part/([^/]+/.*[^/])$ {
if ($request_method != PUT) {
return 400;
}

set_by_lua_block $canonical_path {
local tmp = ngx.var.request_uri
tmp = tmp:gsub('?.*$', '', 1)
tmp = tmp:gsub('^/[^/]+/[^/]+/', '', 1)
tmp = tmp:gsub('+', '%%2B')
tmp = tmp:gsub('&amp;', '%%26')
tmp = ngx.unescape_uri(tmp)
return tmp
}

if ($canonical_path !~ '^[__SUPPORTED_CHARSET__]+$') {
return 400;
}

if ($canonical_path !~ '^([^/]+\:staging\-([0-9]{10}\.|)[0-9a-f]+\.[^./]+\.[0-9]+(\.[0-9]+|)|\.md_staging)/') {
return 400;
}

set $signature_mode "MULTIPART_UPLOAD_PART";
set $aws_access_key "";
set $aws_signature "";
set $x_amz_date "";
set $x_amz_acl "";
set $aws_tgt_bucket "${AWS_BUCKET_PREFIX}-staging";
set $encoded_key "";
rewrite_by_lua_file /etc/nginx/compute_aws_s3_signature.ljbc;

proxy_set_header Authorization "AWS $aws_access_key:$aws_signature";
proxy_set_header x-amz-date $x_amz_date;
proxy_set_header Host $aws_tgt_bucket.__S3_ENDPOINT_HOST__;
proxy_set_header Connection "keep-alive";

proxy_pass __S3_ENDPOINT_PROTO__$aws_tgt_bucket/$encoded_key?partNumber=$arg_partNumber&uploadId=$arg_uploadId;
}

# Multipart upload: complete.
# POST /upload-multipart/complete/<build>/<path>?uploadId=X
# Proxies POST /<key>?uploadId=X with XML body (part ETags) to S3.
#
location ~ ^/upload-multipart/complete/([^/]+/.*[^/])$ {
if ($request_method != POST) {
return 400;
}

set_by_lua_block $canonical_path {
local tmp = ngx.var.request_uri
tmp = tmp:gsub('?.*$', '', 1)
tmp = tmp:gsub('^/[^/]+/[^/]+/', '', 1)
tmp = tmp:gsub('+', '%%2B')
tmp = tmp:gsub('&amp;', '%%26')
tmp = ngx.unescape_uri(tmp)
return tmp
}

if ($canonical_path !~ '^[__SUPPORTED_CHARSET__]+$') {
return 400;
}

if ($canonical_path !~ '^([^/]+\:staging\-([0-9]{10}\.|)[0-9a-f]+\.[^./]+\.[0-9]+(\.[0-9]+|)|\.md_staging)/') {
return 400;
}

set $signature_mode "MULTIPART_COMPLETE";
set $aws_access_key "";
set $aws_signature "";
set $x_amz_date "";
set $x_amz_acl "";
set $aws_tgt_bucket "${AWS_BUCKET_PREFIX}-staging";
set $encoded_key "";
rewrite_by_lua_file /etc/nginx/compute_aws_s3_signature.ljbc;

proxy_set_header Authorization "AWS $aws_access_key:$aws_signature";
proxy_set_header x-amz-date $x_amz_date;
proxy_set_header Host $aws_tgt_bucket.__S3_ENDPOINT_HOST__;
proxy_set_header Connection "keep-alive";
proxy_set_header Content-Type "application/xml";

proxy_pass __S3_ENDPOINT_PROTO__$aws_tgt_bucket/$encoded_key?uploadId=$arg_uploadId;
}

# Multipart upload: abort.
# DELETE /upload-multipart/abort/<build>/<path>?uploadId=X
# Cancels an in-progress multipart upload and frees stored parts.
#
location ~ ^/upload-multipart/abort/([^/]+/.*[^/])$ {
if ($request_method != DELETE) {
return 400;
}

set_by_lua_block $canonical_path {
local tmp = ngx.var.request_uri
tmp = tmp:gsub('?.*$', '', 1)
tmp = tmp:gsub('^/[^/]+/[^/]+/', '', 1)
tmp = tmp:gsub('+', '%%2B')
tmp = tmp:gsub('&amp;', '%%26')
tmp = ngx.unescape_uri(tmp)
return tmp
}

if ($canonical_path !~ '^[__SUPPORTED_CHARSET__]+$') {
return 400;
}

if ($canonical_path !~ '^([^/]+\:staging\-([0-9]{10}\.|)[0-9a-f]+\.[^./]+\.[0-9]+(\.[0-9]+|)|\.md_staging)/') {
return 400;
}

set $signature_mode "MULTIPART_ABORT";
set $aws_access_key "";
set $aws_signature "";
set $x_amz_date "";
set $x_amz_acl "";
set $aws_tgt_bucket "${AWS_BUCKET_PREFIX}-staging";
set $encoded_key "";
rewrite_by_lua_file /etc/nginx/compute_aws_s3_signature.ljbc;

proxy_set_header Authorization "AWS $aws_access_key:$aws_signature";
proxy_set_header x-amz-date $x_amz_date;
proxy_set_header Host $aws_tgt_bucket.__S3_ENDPOINT_HOST__;
proxy_set_header Connection "keep-alive";

proxy_pass __S3_ENDPOINT_PROTO__$aws_tgt_bucket/$encoded_key?uploadId=$arg_uploadId;
}

# Add metadata for a build with the following syntax:
#
# /github/scality/$(repo)/$(workflow_name)/$(created_at)/$(artifacts_name)?key1=value1&key1=value2&key2=value3
Expand Down
130 changes: 127 additions & 3 deletions lua/compute_aws_s3_signature.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,10 @@ local function empty_if_nil (str)
end


-- Compute AWS S3 signature.
-- Compute AWS S3 signature with an explicit canonicalized resource.
-- Used by multipart modes that need to append subresource query params.
--
local function compute_S3_signature (canonicalized_amz_headers)
local canonicalized_resource = "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key
local function compute_S3_signature_with_resource (canonicalized_amz_headers, canonicalized_resource)
local aws_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
local http_content_md5 = empty_if_nil(ngx.var.http_content_md5)
local http_content_type = empty_if_nil(ngx.var.http_content_type)
Expand All @@ -94,6 +94,14 @@ local function compute_S3_signature (canonicalized_amz_headers)
end


-- Compute AWS S3 signature.
--
local function compute_S3_signature (canonicalized_amz_headers)
local canonicalized_resource = "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key
compute_S3_signature_with_resource(canonicalized_amz_headers, canonicalized_resource)
end


-- Compute AWS S3 presignature.
--
local function compute_S3_presignature (expires)
Expand Down Expand Up @@ -253,6 +261,122 @@ elseif signature_mode == "PRESIGN" then
--
return ngx.redirect(ngx.var.redirect_endpoint .. "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?" .. ngx.var.presign_query_string, ngx.HTTP_MOVED_TEMPORARILY);

elseif signature_mode == "MULTIPART_INITIATE" then

-- MULTIPART_INITIATE: POST /{key}?uploads
-- Initiates a multipart upload; S3 returns an uploadId.
--
ngx.var.encoded_key = get_encoded_key(ngx.var.canonical_path)
if ngx.var.aws_tgt_bucket == "" then
ngx.var.aws_tgt_bucket = aws_bucket_prefix .. "-staging"
end
ngx.var.x_amz_acl = "private"
compute_S3_signature_with_resource(
"x-amz-acl:" .. ngx.var.x_amz_acl .. "\nx-amz-date:" .. ngx.var.x_amz_date,
"/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploads"
)

elseif signature_mode == "MULTIPART_UPLOAD_PART" then

-- MULTIPART_UPLOAD_PART: PUT /{key}?partNumber=N&uploadId=X
-- Uploads one chunk; S3 returns an ETag for the part.
-- partNumber < uploadId alphabetically → correct V2 sort order.
--
ngx.var.encoded_key = get_encoded_key(ngx.var.canonical_path)
if ngx.var.aws_tgt_bucket == "" then
ngx.var.aws_tgt_bucket = aws_bucket_prefix .. "-staging"
end
local part_number = ngx.var.arg_partNumber or ""
local upload_id = ngx.var.arg_uploadId or ""
compute_S3_signature_with_resource(
"x-amz-date:" .. ngx.var.x_amz_date,
"/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?partNumber=" .. part_number .. "&uploadId=" .. upload_id
)

elseif signature_mode == "MULTIPART_COMPLETE" then

-- MULTIPART_COMPLETE: POST /{key}?uploadId=X
-- Sends XML body with part ETags; S3 assembles the final object.
--
ngx.var.encoded_key = get_encoded_key(ngx.var.canonical_path)
if ngx.var.aws_tgt_bucket == "" then
ngx.var.aws_tgt_bucket = aws_bucket_prefix .. "-staging"
end
local upload_id = ngx.var.arg_uploadId or ""
compute_S3_signature_with_resource(
"x-amz-date:" .. ngx.var.x_amz_date,
"/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id
)

elseif signature_mode == "MULTIPART_ABORT" then

-- MULTIPART_ABORT: DELETE /{key}?uploadId=X
-- Cancels an in-progress multipart upload and frees stored parts.
--
ngx.var.encoded_key = get_encoded_key(ngx.var.canonical_path)
if ngx.var.aws_tgt_bucket == "" then
ngx.var.aws_tgt_bucket = aws_bucket_prefix .. "-staging"
end
local upload_id = ngx.var.arg_uploadId or ""
compute_S3_signature_with_resource(
"x-amz-date:" .. ngx.var.x_amz_date,
"/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id
)

elseif signature_mode == "PRESIGN_PUT" then

-- PRESIGN_PUT: return a presigned S3 PUT URL for a single-file upload.
-- The client (GitHub Actions runner) GETs this endpoint to obtain a
-- presigned URL, then PUTs the file body directly to S3, bypassing nginx.
--
ngx.var.encoded_key = get_encoded_key(ngx.var.canonical_path)
local build_tgt = ngx.var.canonical_path:match("^[^/]+")
scan_tgt_buckets(build_tgt)

local expires = ngx.time() + 3600
local aws_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
local canonicalized_resource = "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key
-- Sign as PUT with no Content-MD5 and no Content-Type (presigned, not proxied).
local string_to_sign = "PUT\n\n\n" .. expires .. "\n" .. canonicalized_resource
local aws_signature = ngx.encode_base64(ngx.hmac_sha1(aws_secret_key, string_to_sign))
local presigned_url = ngx.var.redirect_endpoint .. canonicalized_resource .. "?" ..
ngx.encode_args({AWSAccessKeyId = ngx.var.aws_access_key, Expires = expires, Signature = aws_signature})

ngx.header.content_type = "text/plain"
ngx.say(presigned_url)
return ngx.exit(ngx.HTTP_OK)

elseif signature_mode == "PRESIGN_PART" then

-- PRESIGN_PART: return a presigned S3 PUT URL for one multipart part.
-- The client GETs ?partNumber=N&uploadId=X to obtain a presigned URL,
-- then PUTs the part body directly to S3.
--
ngx.var.encoded_key = get_encoded_key(ngx.var.canonical_path)
if ngx.var.aws_tgt_bucket == "" then
ngx.var.aws_tgt_bucket = aws_bucket_prefix .. "-staging"
end

local part_number = ngx.var.arg_partNumber or ""
local upload_id = ngx.var.arg_uploadId or ""
local expires = ngx.time() + 3600
local aws_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
-- subresources must appear in canonical resource (alphabetical: partNumber < uploadId)
local canonicalized_resource = "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key ..
"?partNumber=" .. part_number .. "&uploadId=" .. upload_id
local string_to_sign = "PUT\n\n\n" .. expires .. "\n" .. canonicalized_resource
local aws_signature = ngx.encode_base64(ngx.hmac_sha1(aws_secret_key, string_to_sign))
local presigned_url = ngx.var.redirect_endpoint .. "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key ..
"?partNumber=" .. part_number ..
"&uploadId=" .. ngx.escape_uri(upload_id) ..
"&AWSAccessKeyId=" .. ngx.escape_uri(ngx.var.aws_access_key) ..
"&Expires=" .. expires ..
"&Signature=" .. ngx.escape_uri(aws_signature)

ngx.header.content_type = "text/plain"
ngx.say(presigned_url)
return ngx.exit(ngx.HTTP_OK)

else

--
Expand Down
Loading
Loading