Support private S3 buckets (#923)

Author: rafaykh90

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## master
 
+## 2.0.2
+- Support private ACL for S3 buckets [#923](https://github.com/mapbox/node-pre-gyp/pull/923)
+
 ## 2.0.1
 - Update abi_crosswalk.json for abi 137 / node 24 (https://github.com/mapbox/node-pre-gyp/pull/904)
 

README.md

@@ -85,6 +85,7 @@ Options include:
  - `--target=0.4.0`: Pass the target node or node-webkit version to compile against
  - `--target_arch=ia32`: Pass the target arch and override the host `arch`. Any value that is [supported by Node.js](https://nodejs.org/api/os.html#osarch) is valid.
  - `--target_platform=win32`: Pass the target platform and override the host `platform`. Valid values are `linux`, `darwin`, `win32`, `sunos`, `freebsd`, `openbsd`, and `aix`.
+ - `--acl=<acl>`: Set the S3 ACL when publishing binaries (e.g., `public-read`, `private`). Overrides the `binary.acl` setting in package.json.
 
 Both `--build-from-source` and `--fallback-to-build` can be passed alone or they can provide values. You can pass `--fallback-to-build=false` to override the option as declared in package.json. In addition to being able to pass `--build-from-source` you can also pass `--build-from-source=myapp` where `myapp` is the name of your module.
 
@@ -185,6 +186,33 @@ Your S3 server region.
 
 Set `s3ForcePathStyle` to true if the endpoint url should not be prefixed with the bucket name. If false (default), the server endpoint would be  constructed as `bucket_name.your_server.com`.
 
+###### acl
+
+The S3 Access Control List (ACL) to apply when publishing binaries. Defaults to `'public-read'` for backward compatibility. Common values include:
+
+- `public-read` - (default) Binary is publicly accessible by anyone
+- `private` - Binary requires AWS credentials to download
+- `authenticated-read` - Any authenticated AWS user can download
+- `bucket-owner-read` - Bucket owner gets READ access
+- `bucket-owner-full-control` - Bucket owner gets FULL_CONTROL access
+
+**For private binaries:**
+- Users installing your package will need AWS credentials configured (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables)
+- The `aws-sdk` package must be available at install time
+- If authentication fails, node-pre-gyp will fall back to building from source (if `--fallback-to-build` is specified)
+
+You can also specify the ACL via command-line flag: `node-pre-gyp publish --acl=private`
+
+Example for private binaries:
+```json
+"binary": {
+  "module_name": "your_module",
+  "module_path": "./lib/binding/",
+  "host": "https://your-bucket.s3.us-east-1.amazonaws.com",
+  "acl": "private"
+}
+```
+
 ##### The `binary` object has optional properties
 
 ###### remote_path

lib/install.js

@@ -10,6 +10,8 @@ const log = require('./util/log.js');
 const existsAsync = fs.exists || path.exists;
 const versioning = require('./util/versioning.js');
 const napi = require('./util/napi.js');
+const s3_setup = require('./util/s3_setup.js');
+const url = require('url');
 // for fetching binaries
 const fetch = require('node-fetch');
 const tar = require('tar');
@@ -23,6 +25,65 @@ try {
   // do nothing
 }
 
+function place_binary_authenticated(opts, targetDir, callback) {
+  log.info('install', 'Attempting authenticated S3 download');
+
+  // Check if AWS credentials are available
+  if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) {
+    const err = new Error('Binary is private but AWS credentials not found. Please configure AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables, or use --fallback-to-build to compile from source.');
+    err.statusCode = 403;
+    return callback(err);
+  }
+
+  try {
+    const config = s3_setup.detect(opts);
+    const s3 = s3_setup.get_s3(config);
+    const key_name = url.resolve(config.prefix, opts.package_name);
+
+    log.info('install', 'Downloading from S3:', config.bucket, key_name);
+
+    const s3_opts = {
+      Bucket: config.bucket,
+      Key: key_name
+    };
+
+    s3.getObject(s3_opts, (err, data) => {
+      if (err) {
+        log.error('install', 'Authenticated S3 download failed:', err.message);
+        return callback(err);
+      }
+
+      log.info('install', 'Authenticated download successful, extracting...');
+
+      const { Readable } = require('stream');
+      const dataStream = Readable.from(data.Body);
+
+      let extractions = 0;
+      const countExtractions = (entry) => {
+        extractions += 1;
+        log.info('install', `unpacking ${entry.path}`);
+      };
+
+      dataStream.pipe(extract(targetDir, countExtractions))
+        .on('error', (e) => {
+          callback(e);
+        })
+        .on('close', () => {
+          log.info('install', `extracted file count: ${extractions}`);
+          callback();
+        });
+    });
+  } catch (e) {
+    if (e.code === 'MODULE_NOT_FOUND' && e.message.includes('aws-sdk')) {
+      const err = new Error('Binary is private and requires aws-sdk for authenticated download. Please run: npm install aws-sdk');
+      err.statusCode = 403;
+      return callback(err);
+    }
+    log.error('install', 'Error setting up authenticated download:', e.message);
+    callback(e);
+  }
+}
+
 function place_binary(uri, targetDir, opts, callback) {
   log.log('GET', uri);
 
@@ -63,6 +124,14 @@ function place_binary(uri, targetDir, opts, callback) {
   fetch(sanitized, { agent })
     .then((res) => {
       if (!res.ok) {
+        // If we get 403 Forbidden, the binary might be private - try authenticated download
+        if (res.status === 403) {
+          log.info('install', 'Received 403 Forbidden - attempting authenticated download');
+          // Call place_binary_authenticated and return a special marker
+          // to prevent the promise chain from calling callback again
+          place_binary_authenticated(opts, targetDir, callback);
+          return { authenticated: true };
+        }
         throw new Error(`response status ${res.status} ${res.statusText} on ${sanitized}`);
       }
       const dataStream = res.body;
@@ -87,6 +156,9 @@ function place_binary(uri, targetDir, opts, callback) {
       });
     })
     .then((text) => {
+      if (text && text.authenticated) {
+        return; // Don't call callback - place_binary_authenticated will handle it
+      }
       log.info(text);
       callback();
     })

lib/mock/s3.js

@@ -36,6 +36,9 @@ function s3_mock() {
     },
     putObject(params, callback) {
       return s3.putObject(params, wcb(callback));
+    },
+    getObject(params, callback) {
+      return s3.getObject(params, wcb(callback));
     }
   };
 }

lib/node-pre-gyp.js

@@ -98,7 +98,8 @@ proto.configDefs = {
   debug: Boolean,    // 'build'
   directory: String, // bin
   proxy: String,     // 'install'
-  loglevel: String  // everywhere
+  loglevel: String,  // everywhere
+  acl: String        // 'publish' - S3 ACL for published binaries
 };
 
 /**

lib/publish.js

@@ -43,12 +43,13 @@ function publish(gyp, argv, callback) {
         // the object does not already exist
         log.info('publish', 'Preparing to put object');
         const s3_put_opts = {
-          ACL: 'public-read',
+          ACL: opts.acl,
           Body: fs.createReadStream(tarball),
           Key: key_name,
           Bucket: config.bucket
         };
-        log.info('publish', 'Putting object', s3_put_opts.ACL, s3_put_opts.Bucket, s3_put_opts.Key);
+        log.info('publish', 'Putting object with ACL:', s3_put_opts.ACL);
+        log.info('publish', 'Bucket:', s3_put_opts.Bucket, 'Key:', s3_put_opts.Key);
         try {
           s3.putObject(s3_put_opts, (err2, resp) => {
             log.info('publish', 'returned from putting object');

lib/util/s3_setup.js

@@ -84,6 +84,9 @@ module.exports.get_s3 = function(config) {
     },
     putObject(params, callback) {
       return s3.putObject(params, callback);
+    },
+    getObject(params, callback) {
+      return s3.getObject(params, callback);
     }
   };
 };

lib/util/versioning.js

@@ -307,7 +307,8 @@ module.exports.evaluate = function(package_json, options, napi_build_version) {
     toolset: options.toolset || '', // address https://github.com/mapbox/node-pre-gyp/issues/119
     bucket: package_json.binary.bucket,
     region: package_json.binary.region,
-    s3ForcePathStyle: package_json.binary.s3ForcePathStyle || false
+    s3ForcePathStyle: package_json.binary.s3ForcePathStyle || false,
+    acl: options.acl || package_json.binary.acl || 'public-read'
   };
     // support host mirror with npm config `--{module_name}_binary_host_mirror`
     // e.g.: https://github.com/node-inspector/v8-profiler/blob/master/package.json#L25

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "@mapbox/node-pre-gyp",
   "description": "Node.js native addon binary install tool",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "keywords": [
     "native",
     "addon",
@@ -61,5 +61,8 @@
     "test": "tape test/*test.js",
     "test:s3": "tape test/s3.test.js",
     "bucket": "node scripts/set-bucket.js"
+  },
+  "overrides": {
+    "js-yaml": "^3.14.2"
   }
 }