Test presigned url for upload

Author: matthiasL-scality

dist/index.js

@@ -158,9 +158,12 @@ export async function fileUploadMultipart(
   core.info(`Multipart: initiated, uploadId obtained for ${file}`)
 
   // 2. Upload all parts in parallel (MULTIPART_CONCURRENCY at a time).
+  // Each part: GET a presigned S3 URL from the proxy (auth + tiny payload),
+  // then PUT the part body directly to S3 — data bypasses the nginx proxy
+  // and the node NIC entirely.
   const etags: {partNumber: number; etag: string}[] = []
-  const partUrl = new URL(
-    path.join('/upload-multipart/part/', buildName, filePath),
+  const presignPartBaseUrl = new URL(
+    path.join('/presign-upload-part/', buildName, filePath),
     baseUrl
   ).toString()
 
@@ -171,29 +174,59 @@ export async function fileUploadMultipart(
     core.info(
       `Multipart: uploading part ${partNumber}/${partCount} (${Math.round(partSize / 1e6)}MB) for ${file}`
     )
+
+    // Step 2a — get presigned URL (authenticated, lightweight).
+    const presignResp = await client.get(presignPartBaseUrl, {
+      params: {partNumber, uploadId},
+      timeout: 30000
+    })
+    const s3PartUrl = (presignResp.data as string).trim()
+
+    // Step 2b — PUT part directly to S3 using raw https.request.
+    // Axios re-encodes presigned URL query strings via the URL API, which
+    // decodes %2B → + and re-serialises it as + (space in query strings).
+    // This corrupts the AWS Signature V2 and causes 403 at Scaleway.
+    // Using https.request preserves the query string exactly as returned
+    // by the proxy.
     const partStream = fs.createReadStream(file, {start, end})
-    let resp
-    try {
-      resp = await client.put(partUrl, partStream, {
-        params: {partNumber, uploadId},
-        headers: {'Content-Length': partSize.toString()},
-        maxBodyLength: Infinity,
-        maxContentLength: Infinity,
-        maxRedirects: 0,
-        timeout: 600000
-      })
-    } catch (e: unknown) {
-      if (axios.isAxiosError(e) && e.response) {
-        core.error(
-          `Multipart: part ${partNumber}/${partCount} failed with status ${e.response.status}: ${JSON.stringify(e.response.data)}`
-        )
-      }
-      throw e
-    }
-    const etag = resp.headers['etag'] as string
-    if (!etag) {
-      throw new Error(`No ETag returned for part ${partNumber} of ${file}`)
-    }
+    const s3Url = new URL(s3PartUrl)
+    const etag = await new Promise<string>((resolve, reject) => {
+      const req = https.request(
+        {
+          method: 'PUT',
+          hostname: s3Url.hostname,
+          port: s3Url.port ? parseInt(s3Url.port) : 443,
+          path: s3Url.pathname + s3Url.search,
+          headers: {'Content-Length': String(partSize)}
+        },
+        res => {
+          let body = ''
+          res.on('data', (chunk: Buffer) => {
+            body += chunk.toString()
+          })
+          res.on('end', () => {
+            if (res.statusCode === 200) {
+              const tag = res.headers['etag'] as string
+              if (!tag) {
+                reject(
+                  new Error(`No ETag returned for part ${partNumber} of ${file}`)
+                )
+              } else {
+                resolve(tag)
+              }
+            } else {
+              reject(
+                new Error(
+                  `Multipart: part ${partNumber}/${partCount} failed with status ${res.statusCode}: ${body}`
+                )
+              )
+            }
+          })
+        }
+      )
+      req.on('error', reject)
+      partStream.pipe(req)
+    })
     etags.push({partNumber, etag})
     core.info(`Multipart: part ${partNumber}/${partCount} done for ${file}`)
   }

dist/index.js.map

@@ -10,7 +10,7 @@ import * as process from 'process'
 import {
   artifactsName,
   artifactsPatternName,
-  fileUpload,
+  fileUploadPresigned,
   fileUploadMultipart,
   fileVersion,
   MULTIPART_THRESHOLD,
@@ -143,11 +143,7 @@ async function upload_one_file(
     )
     await fileUploadMultipart(client, url, name, file, artifactsPath)
   } else {
-    const uploadUrl: string = new URL(
-      path.join('/upload/', name, artifactsPath),
-      url
-    ).toString()
-    await fileUpload(client, uploadUrl, file)
+    await fileUploadPresigned(client, url, name, file, artifactsPath)
   }
   const elapsed = (Date.now() - uploadStart) / 1000
   const mbps = (fileSize / 1e6 / elapsed).toFixed(1)