Caching the downloaded archive (#30)

* Implement filesystem caching of downloaded binary

Closes #24

* Fix failure to install multiple times on Mac

* Cater for different OSs sharing node_modules

Closes #25

* Update proposed filesystem caching after feedback

* Turn caching on by default, using default filesystem locations
* `CHROMIUM_CACHE` now overrides default cache location
* Add flag `CHROMIUM_CACHE_SKIP` to disable caching
* Many little refactors as suggested
* Updated readme accordingly
* Delete temporary files when the process exits

* Refactor setEnvVar test util method

* Ask wise and all-knowing friend for cache location

* Rename CHROMIUM_CACHE to NODE_CHROMIUM_CACHE

* Update readme

* Rename cacheDir variable now cachedir is in scope

dtolstyi would never let me get away with that :)

* Rename all env vars CHROMIUM_* > NODE_CHROMIUM_*

* More resilient progress bar and rename env var

Author: ricksbrown

README.MD

@@ -40,40 +40,40 @@ npm config set https-proxy http://<username>:<password>@<the.proxy.hostname>:<po
 npm config set no-proxy localhost,127.0.0.1,example.org
 ```
 
-Additionally proxy settings found in the environment variables `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` will be used if they are not defined in the `.npmrc` file. 
+Additionally proxy settings found in the environment variables `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` will be used if they are not defined in the `.npmrc` file.
 
 ### Install a concrete revision
-If you want to specify the revision of Chromium to be installed, just set the environment variable `CHROMIUM_REVISION` to the number of the revision you want to install, as in:
+If you want to specify the revision of Chromium to be installed, just set the environment variable `NODE_CHROMIUM_REVISION` to the number of the revision you want to install, as in:
 ```shell script
-export CHROMIUM_REVISION=729994
+export NODE_CHROMIUM_REVISION=729994
 ```
 
 Note - may also be set in .npmrc like so:
 
 ```ini
-chromium_revision=729994
+node_chromium_revision=729994
 ```
 
 ### Use a Download Mirror
-You may download a specific revision from an alternate download host using the environment variable `CHROMIUM_DOWNLOAD_HOST`, for example:
+You may download a specific revision from an alternate download host using the environment variable `NODE_CHROMIUM_DOWNLOAD_HOST`, for example:
 
 ```bash
-export CHROMIUM_REVISION=737027
-export CHROMIUM_DOWNLOAD_HOST=https://npm.taobao.org/mirrors/chromium-browser-snapshots/
+export NODE_CHROMIUM_REVISION=737027
+export NODE_CHROMIUM_DOWNLOAD_HOST=https://npm.taobao.org/mirrors/chromium-browser-snapshots/
 
 # If running on Linux x64 this will download binary from:
 # https://npm.taobao.org/mirrors/chromium-browser-snapshots/Linux_x64/737027/chrome-linux.zip?alt=media
 ```
 
-Notes on `CHROMIUM_DOWNLOAD_HOST`:
+Notes on `NODE_CHROMIUM_DOWNLOAD_HOST`:
 
 * The default download host is `https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/`
-* Mirrors are expected to host binaries in the structure: `<CHROMIUM_DOWNLOAD_HOST>/<PLATFORM_ARCHITECTURE>/<REVISION>/<OS_CHROMIUM_FILE_NAME>.zip?alt=media` for example see the taobao mirror [chromium-browser-snapshots](https://npm.taobao.org/mirrors/chromium-browser-snapshots/).
+* Mirrors are expected to host binaries in the structure: `<NODE_CHROMIUM_DOWNLOAD_HOST>/<PLATFORM_ARCHITECTURE>/<REVISION>/<OS_CHROMIUM_FILE_NAME>.zip?alt=media` for example see the taobao mirror [chromium-browser-snapshots](https://npm.taobao.org/mirrors/chromium-browser-snapshots/).
 * May also be set in .npmrc like so:
 
 ```ini
-chromium_download_host=https://npm.taobao.org/mirrors/chromium-browser-snapshots/
-chromium_revision=737027
+node_chromium_download_host=https://npm.taobao.org/mirrors/chromium-browser-snapshots/
+node_chromium_revision=737027
 ```
 
 ## Selenium WebDriver Headless (without UI) tests
@@ -121,7 +121,27 @@ async function takeScreenshot(driver, name) {
 
 start();
 ```
-## 
+
+### Cache Downloaded Binaries
+By default downloaded chromium binaries are cached in the appropriate cache directory for your operating system.
+
+You may override the cache path by setting the `NODE_CHROMIUM_CACHE_PATH` environment variable to a directory path, for example:
+
+```bash
+export NODE_CHROMIUM_CACHE_PATH=/path/to/cache/dir/
+
+# or in .npmrc like so:
+# node_chromium_cache_path=/path/to/cache/dir/
+```
+
+You may disable caching by setting `NODE_CHROMIUM_CACHE_DISABLE` to `true`:
+
+```bash
+export NODE_CHROMIUM_CACHE_DISABLE=true
+
+# or in .npmrc like so:
+# node_chromium_cache_disable=true
+```
 
 ## License
 MIT

cache.js

@@ -0,0 +1,57 @@
+'use strict';
+const path = require('path');
+const fs = require('fs');
+const cachedir = require('cachedir');
+const config = require('./config');
+
+/**
+ * Retrieve a Chromium archive from filesystem cache.
+ * @param {string} revision The Chromium revision to retrieve.
+ * @returns {string} The path to the cached Chromium archive. Falsy if not found.
+ */
+function get(revision) {
+    const cachePath = buildCachePath(revision);
+    if (fs.existsSync(cachePath)) {
+        return cachePath;
+    }
+
+    return '';
+}
+
+/**
+ * Store a Chromium archive in filesystem cache for future use.
+ * Has no effect if the user has not configured a cache location.
+ * @param {string} revision The Chromium revision in the archive file.
+ * @param {string} filePath The path to the Chromium archive file to store in cache.
+ */
+function put(revision, filePath) {
+    const cachePath = buildCachePath(revision);
+    if (cachePath && filePath) {
+        try {
+            fs.mkdirSync(path.dirname(cachePath), {recursive: true});
+            fs.copyFileSync(filePath, cachePath);
+        } catch (error) {
+            // Don't die on cache fail
+            console.error('Could not cache file', cachePath, error);
+        }
+    }
+}
+
+/**
+ * Get the unique cache path for this revision, on this platform and architecture.
+ * @param {string} revision The revision of this Chromium binary, essentially a unique cache key.
+ * @returns {string} The cache path, or falsy if caching is not enabled.
+ */
+function buildCachePath(revision) {
+    if (!revision || config.getEnvVar('NODE_CHROMIUM_CACHE_DISABLE').toLowerCase() === 'true') {
+        return '';
+    }
+
+    const cachePath = config.getEnvVar('NODE_CHROMIUM_CACHE_PATH') || cachedir('node-chromium');
+    return path.join(cachePath, `chromium-${revision}-${process.platform}-${process.arch}.zip`);
+}
+
+module.exports = {
+    get,
+    put
+};

index.js

@@ -15,5 +15,6 @@ function getBinaryPath() {
 }
 
 module.exports = {
-    path: getBinaryPath()
+    path: getBinaryPath(),
+    install: require('./install')
 };

install.js

@@ -1,14 +1,17 @@
 'use strict';
 
+const path = require('path');
 const fs = require('fs');
 const extractZip = require('extract-zip');
 const got = require('got');
 const tmp = require('tmp');
 const debug = require('debug')('node-chromium');
+const rimraf = require('rimraf');
 const ProgressBar = require('progress');
 
 const config = require('./config');
 const utils = require('./utils');
+const cache = require('./cache');
 
 let progressBar = null;
 
@@ -27,13 +30,26 @@ function createTempFile() {
     });
 }
 
+/**
+ * Downloads the Chromium archive from the default CDN or mirror if configured.
+ * If the required archive is retrieved from the cache directory then the download will be skipped.
+ * @param {string} revision The Chromium revision to download.
+ */
 async function downloadChromiumRevision(revision) {
-    const tmpPath = await createTempFile();
+    const cacheEntry = cache.get(revision);
+    if (cacheEntry) {
+        debug('Found Chromium archive in cache, skipping download');
+
+        return Promise.resolve(cacheEntry);
+    }
 
     debug('Downloading Chromium archive from Google CDN');
     const url = utils.getDownloadUrl(revision);
-
-    return _downloadFile(url, tmpPath);
+    const tmpPath = await createTempFile();
+    return _downloadFile(url, tmpPath).then(tmpPath => {
+        cache.put(revision, tmpPath);
+        return tmpPath;
+    });
 }
 
 function _downloadFile(url, destPath) {
@@ -60,54 +76,63 @@ function _downloadFile(url, destPath) {
  * @param progress Information about progress so far.
  */
 function onProgress(progress) {
+    const fakeProgressBar = {tick: () => {}};
     try {
         if (!progressBar) {
             const formatBytes = bytes => {
                 const mb = bytes / 1024 / 1024;
                 return `${Math.round(mb * 10) / 10} MB`;
             };
 
-            progressBar = new ProgressBar(`Downloading Chromium - ${formatBytes(progress.total)} [:bar] :percent :etas `, {
-                width: 20,
-                total: progress.total
-            });
+            if (progress.total) {
+                progressBar = new ProgressBar(`Downloading Chromium - ${formatBytes(progress.total)} [:bar] :percent :etas `, {
+                    width: 20,
+                    total: progress.total
+                });
+            } else {
+                progressBar = fakeProgressBar;
+                console.info('\tPlease wait, this may take a while...');
+            }
         }
 
         progressBar.tick(progress.transferred - progressBar.curr);
     } catch (error) {
         // Don't die on progress bar failure, log it and stop progress
         console.error('Error displaying progress bar. Continuing anyway...', error);
-        progressBar = {tick: () => {}};
+        progressBar = fakeProgressBar;
     }
 }
 
 function unzipArchive(archivePath, outputFolder) {
     debug('Started extracting archive', archivePath);
 
     return new Promise((resolve, reject) => {
-        extractZip(archivePath, {dir: outputFolder}, error => {
-            if (error) {
-                console.error('An error occurred while trying to extract archive', error);
-                reject(error);
-            } else {
-                debug('Archive was successfully extracted');
-                resolve(true);
-            }
+        const osOutputFolder = path.join(outputFolder, utils.getOsChromiumFolderName());
+        rimraf(osOutputFolder, () => {
+            extractZip(archivePath, {dir: outputFolder}, error => {
+                if (error) {
+                    console.error('An error occurred while trying to extract archive', error);
+                    reject(error);
+                } else {
+                    debug('Archive was successfully extracted');
+                    resolve(true);
+                }
+            });
         });
     });
 }
 
 async function install() {
-    const chromiumRevision = config.getEnvVar('CHROMIUM_REVISION');
+    const chromiumRevision = config.getEnvVar('NODE_CHROMIUM_REVISION');
     try {
         console.info('Step 1. Retrieving Chromium revision number');
         const revision = chromiumRevision || await utils.getLatestRevisionNumber();
 
         console.info(`Step 2. Downloading Chromium revision ${revision}`);
-        const tmpPath = await downloadChromiumRevision(revision);
+        const archivePath = await downloadChromiumRevision(revision);
 
         console.info('Step 3. Setting up Chromium binaries');
-        await unzipArchive(tmpPath, config.BIN_OUT_PATH);
+        await unzipArchive(archivePath, config.BIN_OUT_PATH);
 
         console.info('Process is successfully finished');
     } catch (error) {
@@ -120,4 +145,6 @@ if (require.main === module) {
     install();
 }
 
+tmp.setGracefulCleanup(); // Ensure temporary files are cleaned up when process exits
+
 module.exports = install;