Merge pull request #52 from VikramTiwari/updates-for-license-key

fix: update config to use license key

Author: VikramTiwari

.github/workflows/nodejs.yml

@@ -9,7 +9,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [8.x, 10.x, 12.x, 13.x]
+        node-version: [10.x, 12.x, 13.x]
 
     steps:
     - uses: actions/checkout@v1
@@ -24,3 +24,4 @@ jobs:
         npm test
       env:
         CI: true
+        MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}

.travis.yml

@@ -20,6 +20,8 @@ npm install --save geo-from-ip
 yarn add --save geo-from-ip
 ```
 
+- Set `MAXMIND_LICENSE_KEY=<your_maxmind_license_key>` in your environment variables. Read more about [this change on MaxMind's blog](https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/).
+
 - Use package to get geo data from IP
 
 ```javascript

README.md

@@ -1,118 +1,122 @@
-'use strict'
+"use strict";
 
-const debug = require('debug')('geo-from-ip:updater')
-const fs = require('fs')
-const zlib = require('zlib')
-const async = require('async')
-const request = require('request')
+const debug = require("debug")("geo-from-ip:updater");
+const fs = require("fs");
+const path = require("path");
+const util = require("util");
+const streamPipeline = util.promisify(require("stream").pipeline);
+const fetch = require("node-fetch");
+const targz = require("targz");
 
-const config = require('../mmdb/config')
-debug(`Running update script with config: ${JSON.stringify(config)}`)
+if (process.env.MAXMIND_LICENSE_KEY === undefined) {
+  debug(
+    "Seems like you forgot to add MAXMIND_LICENSE_KEY to your environment variables. Read more: https://github.com/VikramTiwari/geo-from-ip#how-to-use"
+  );
+  process.exit(1);
+}
+
+const config = require("../mmdb/config");
 
 /**
- * check if remote file is newer
- * @param  {String}   dest   path for local file
- * @param  {String}   remote path for remote file
- * @param  {Function} cb     true/false
+ * uncompresses the zipped file into folders and does cleanup of remaining files and folders
+ * @param {String} zipped path to zipped file
+ * @param {Object} database database object
  */
-function isRemoteNewer (dest, remote, cb) {
-  // if no file
-  if (!fs.existsSync(dest)) {
-    cb(null, false)
-  } else {
-    // if dest file is not a file, remove it
-    const stats = fs.statSync(dest)
-    if (!stats.isFile()) {
-      debug(`${dest} is not a file`)
-      fs.unlink(dest, () => {
-        debug(`${dest} deleted`)
-      })
-    }
-
-    request(
+async function uncompress(zipped, database) {
+  new Promise((resolve, reject) => {
+    targz.decompress(
       {
-        url: remote,
-        headers: {
-          'If-Modified-Since': stats.mtime.toUTCString()
+        src: zipped,
+        dest: config.dbDir,
+        tar: {
+          ignore: function(name) {
+            return path.extname(name) !== ".mmdb";
+          }
         }
       },
-      res => {
-        if (res === null) {
-          cb(null, true)
+      err => {
+        if (err) {
+          debug(err);
+          reject();
         } else {
-          cb(null, false)
+          fs.readdirSync(config.dbDir).forEach(file => {
+            if (fs.lstatSync(`${config.dbDir}/${file}`).isDirectory()) {
+              fs.renameSync(
+                `${config.dbDir}/${file}/${database.filename}.mmdb`,
+                `${config.dbDir}/${database.filename}.mmdb`
+              );
+              fs.rmdirSync(`${config.dbDir}/${file}`);
+            }
+          });
+          fs.unlinkSync(`${config.dbDir}/${database.filename}.tar.gz`);
+          resolve();
         }
       }
-    )
-  }
+    );
+  });
 }
 
 /**
- * downloads file
- * @param  {String}   dest   path for local file
- * @param  {String}   remote path for remote file
- * @param  {Function} cb     true/false
+ * download database and unzip
+ *
+ * @param {Object} database database to download
  */
-function download (dest, remote, cb) {
-  request
-    .get(remote)
-    .pipe(zlib.createGunzip())
-    .pipe(fs.createWriteStream(dest))
-    .on('error', error => {
-      cb(error, false)
-    })
-    .on('finish', () => {
-      cb(null, true)
-    })
+async function download(database) {
+  const zipped = `${config.dbDir}/${database.filename}.tar.gz`;
+  const response = await fetch(database.remote);
+  await streamPipeline(response.body, fs.createWriteStream(zipped));
+  debug("download complete, uncompressing");
+  await uncompress(zipped, database);
+  debug("ready!");
 }
 
 /**
- * runs downloads for all the files
- * @param  {String}   item   which database
- * @param  {Function} cb     true/false
+ * check if remote file is newer
+ *
+ * @param  {Object}   database  database to download
  */
-function dl (item, cb) {
-  const dest = config.dbDir + '/' + item.local
+async function isRemoteNewer(database) {
+  const mmdb = `${config.dbDir}/${database.filename}.mmdb`;
+  // if no file
+  if (!fs.existsSync(mmdb)) {
+    debug("file does not exist");
+    return true;
+  } else {
+    // if dest file is not a file, remove it
+    const stats = fs.statSync(mmdb);
+    if (!stats.isFile()) {
+      debug(`${mmdb} is not a file`);
+      fs.unlinkSync(mmdb, () => {
+        debug(`${mmdb} deleted`);
+      });
+    }
 
-  isRemoteNewer(dest, item.remote, (err, newer) => {
-    if (err) {
-      cb(err, false)
-    } else if (!err && newer) {
-      cb(null, false)
-    } else if (!err && !newer) {
-      // begin downloading
-      debug(`downloading ${dest}`)
+    const response = await fetch(database.remote, {
+      method: "GET",
+      headers: {
+        "If-Modified-Since": stats.mtime.toUTCString()
+      }
+    });
 
-      download(dest, item.remote, (err, done) => {
-        if (err) {
-          cb(err, false)
-        } else if (!err && !done) {
-          cb(null, false)
-        } else if (!err && done) {
-          cb(null, true)
-        }
-      })
+    if (response === null) {
+      return true;
     }
-  })
+    return false;
+  }
 }
 
 /**
- * Runs a sync function to download the files
- *
- * @param {any} cb
+ * sync databases to local
  */
-function sync (cb) {
-  async.eachSeries(config.geoIpDbs, dl, err => {
-    cb(err, true)
-  })
+function sync() {
+  config.geoIpDbs.forEach(async database => {
+    if (await isRemoteNewer(database)) {
+      debug("remote is newer, downloading");
+      await download(database);
+    }
+  });
 }
 
-async.eachSeries(config.geoIpDbs, dl, err => {
-  if (err) {
-    debug(`🚨 Error: ${err}`)
-  } else {
-    debug(`🚀 All set!`)
-  }
-})
+sync();
 
-module.exports.sync = sync
+module.exports.sync = sync;

lib/update.js

@@ -3,8 +3,8 @@
 // where your GeoIP databases are stored
 exports.dbDir = __dirname
 
-// local-filename, remote geolite-url
+// local:filename, remote:geolite-url
 exports.geoIpDbs = [{
-  local: 'GeoLite2-City.mmdb',
-  remote: 'https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz'
+  filename: 'GeoLite2-City',
+  remote: `https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`
 }]

mmdb/config.js

@@ -1,7 +1,7 @@
 ## Free Databases [https://dev.maxmind.com/geoip/geoip2/geolite2/]
 
-> GeoLite2-City.mmdb [https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz]
-> GeoLite2-Country.mmdb [https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz]
+> GeoLite2-City.mmdb [https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City]
+> GeoLite2-Country.mmdb [https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country]
 
 ## Paid databases [https://www.maxmind.com/en/geoip2-databases]