From 3e3337ddeb4b39848776ef562dec352648982569 Mon Sep 17 00:00:00 2001 From: Bimlendra Date: Wed, 9 May 2018 18:36:09 +0530 Subject: [PATCH 1/6] Added script for agent downloader --- functions/uploader/agent/agent_downloader.js | 94 ++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 functions/uploader/agent/agent_downloader.js diff --git a/functions/uploader/agent/agent_downloader.js b/functions/uploader/agent/agent_downloader.js new file mode 100644 index 00000000..8a15adc3 --- /dev/null +++ b/functions/uploader/agent/agent_downloader.js @@ -0,0 +1,94 @@ +const fetch = require(`node-fetch`); +const error = require(`debug`)(`ia:uploader:agent:error`); +const unzip = require(`unzip`); +const util = require(`util`); +const { exec } = require('child-process-promise'); + +const basicHeaderRequest = { + 'content-type': `application/json; charset=UTF-8`, + 'authorization': `BEARER ` +}; +fetchAgentFromDF(); +/** + * Post entities to DialogFlow + * + * Maximum 1,000 records per request + * Maximum 30,000 records per entity + * + * @param entityname {string} + * @param entities {array} + * @param first {int} suppose to be 0 + * @returns {Promise} + */ + +function fetchAgentFromDF () { + return getAccessToken() + .then(accesstoken => { + return fetch('https://dialogflow.googleapis.com/v2/projects/project_id/agent:export?access_token='+accesstoken, {method: `POST`}); + }) + .then(res => res.json()) + .then(data => { + console.log('Bimmy'+util.inspect(data.response.agentContent, false, null)); + base64_decode(data.response.agentContent, 'agent.zip'); + return base64_decode(data.response.agentContent, 'agent.zip');; + }) + .then(isDone => { + fs.createReadStream('agent.zip').pipe(unzip.Extract({ path: './dynamic_agent' })); + return Promise.resolve("Done"); + }) + .catch(e => { + error(`Get error in posting entity to DF, error: ${JSON.stringify(e)}`); + return Promise.reject(e); + }); +} + +function getAccessToken () { + console.log('getting access token'); + return exec('gcloud auth print-access-token') + .then(values => { + var stdout = values.stdout; + var stderr = values.stderr; + if (stdout) { + var filteredstdout = stdout.replace(/\n$/, ''); + console.log(util.inspect(filteredstdout, false, null)); + return Promise.resolve(filteredstdout); + } else if (stderr) { + error(stderr); + return Promise.reject(new Error('ERROR: ' + stderr)); + } else { + error('Having trouble with GCloud execution'); + return Promise.reject(new Error('ERROR: Having trouble with GCloud execution')); + } + }); +} + + +var fs = require('fs'); + +// function to encode file data to base64 encoded string +function base64_encode(file) { + // read binary data + var bitmap = fs.readFileSync(file); + // convert binary data to base64 encoded string + return new Buffer(bitmap).toString('base64'); +} + +// function to create file from base64 encoded string +function base64_decode(base64str, file) { + // create buffer object from base64 encoded string, it is important to tell the constructor that the string is base64 encoded + var bitmap = new Buffer(base64str, 'base64'); + // write buffer to file + fs.writeFileSync(file, bitmap); + console.log('******** File created from base64 encoded string ********'); + return true; +} + +// convert image to base64 encoded string +//var base64str = base64_encode('kitten.jpg'); +//console.log(base64str); +// convert base64 string back to image +//base64_decode(base64str, 'copy.jpg'); + +module.exports = { + fetchAgentFromDF, +}; From 75a3b836c76f811b350c42ce296f92711bff5d1b Mon Sep 17 00:00:00 2001 From: Bimlendra Date: Wed, 9 May 2018 19:08:04 +0530 Subject: [PATCH 2/6] resolved test fail --- functions/package-lock.json | 238 ++++++++++++++++++- functions/package.json | 4 +- functions/uploader/agent/agent_downloader.js | 69 ++---- 3 files changed, 260 insertions(+), 51 deletions(-) diff --git a/functions/package-lock.json b/functions/package-lock.json index 42a4eb41..415e2c7c 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -837,6 +837,15 @@ } } }, + "binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", + "requires": { + "buffers": "0.1.1", + "chainsaw": "0.1.0" + } + }, "body-parser": { "version": "1.18.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", @@ -939,6 +948,11 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.0.0.tgz", "integrity": "sha512-83apNb8KK0Se60UE1+4Ukbe3HbfELJ6UlI4ldtOGs7So4KD26orJM8hIY9lxdzP+UpItH1Yh/Y8GUvNFWFFRxA==" }, + "buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" + }, "builtin-modules": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", @@ -1067,6 +1081,21 @@ "type-detect": "4.0.8" } }, + "chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", + "requires": { + "traverse": "0.3.9" + }, + "dependencies": { + "traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" + } + } + }, "chalk": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.1.tgz", @@ -1112,6 +1141,27 @@ "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, + "child-process-promise": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/child-process-promise/-/child-process-promise-2.2.1.tgz", + "integrity": "sha1-RzChHvYQ+tRQuPIjx50x172tgHQ=", + "requires": { + "cross-spawn": "4.0.2", + "node-version": "1.1.3", + "promise-polyfill": "6.1.0" + }, + "dependencies": { + "cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", + "requires": { + "lru-cache": "4.1.2", + "which": "1.3.0" + } + } + } + }, "circular-json": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", @@ -2548,6 +2598,27 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, + "fstream": { + "version": "0.1.31", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-0.1.31.tgz", + "integrity": "sha1-czfwWPu7vvqMn1YaKMqwhJICyYg=", + "requires": { + "graceful-fs": "3.0.11", + "inherits": "2.0.3", + "mkdirp": "0.5.1", + "rimraf": "2.6.2" + }, + "dependencies": { + "graceful-fs": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.11.tgz", + "integrity": "sha1-dhPHeKGv6mLyXGMKCG1/Osu92Bg=", + "requires": { + "natives": "1.1.3" + } + } + } + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -4053,8 +4124,7 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "isobject": { "version": "3.0.1", @@ -4368,6 +4438,38 @@ "object-visit": "1.0.1" } }, + "match-stream": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/match-stream/-/match-stream-0.0.2.tgz", + "integrity": "sha1-mesFAJOzTf+t5CG5rAtBCpz6F88=", + "requires": { + "buffers": "0.1.1", + "readable-stream": "1.0.34" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "mathjs": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-4.1.2.tgz", @@ -4556,8 +4658,7 @@ "minimist": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "mixin-deep": { "version": "1.3.1", @@ -4582,7 +4683,6 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "dev": true, "requires": { "minimist": "0.0.8" } @@ -4668,6 +4768,11 @@ "to-regex": "3.0.2" } }, + "natives": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/natives/-/natives-1.1.3.tgz", + "integrity": "sha512-BZGSYV4YOLxzoTK73l0/s/0sH9l8SHs2ocReMH1f8JYSh5FUWu4ZrKCpJdRkWXV6HFR/pZDz7bwWOVAY07q77g==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -4719,6 +4824,11 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.4.tgz", "integrity": "sha512-8Df0906+tq/omxuCZD6PqhPaQDYuyJ1d+VITgxoIA8zvQd1ru+nMJcDChHH324MWitIgbVkAkQoGEEVJNpn/PA==" }, + "node-version": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/node-version/-/node-version-1.1.3.tgz", + "integrity": "sha512-rEwE51JWn0yN3Wl5BXeGn5d52OGbSXzWiiXRjAQeuyvcGKyvuSILW2rb3G7Xh+nexzLwhTpek6Ehxd6IjvHePg==" + }, "node-zip": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/node-zip/-/node-zip-1.1.1.tgz", @@ -4871,6 +4981,11 @@ "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, + "over": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/over/-/over-0.0.5.tgz", + "integrity": "sha1-8phS5w/X4l82DgE6jsRMgq7bVwg=" + }, "p-limit": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.2.0.tgz", @@ -5144,6 +5259,11 @@ "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=", "dev": true }, + "promise-polyfill": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-6.1.0.tgz", + "integrity": "sha1-36lpQ+qcEh/KTem1hoyznTRy4Fc=" + }, "properties-reader": { "version": "0.0.15", "resolved": "https://registry.npmjs.org/properties-reader/-/properties-reader-0.0.15.tgz", @@ -5184,6 +5304,40 @@ "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, + "pullstream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/pullstream/-/pullstream-0.4.1.tgz", + "integrity": "sha1-1vs79a7Wl+gxFQ6xACwlo/iuExQ=", + "requires": { + "over": "0.0.5", + "readable-stream": "1.0.34", + "setimmediate": "1.0.5", + "slice-stream": "1.0.0" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "pump": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", @@ -5479,7 +5633,6 @@ "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", - "dev": true, "requires": { "glob": "7.1.2" } @@ -5614,6 +5767,11 @@ } } }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + }, "setprototypeof": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", @@ -5679,6 +5837,37 @@ "is-fullwidth-code-point": "2.0.0" } }, + "slice-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/slice-stream/-/slice-stream-1.0.0.tgz", + "integrity": "sha1-WzO9ZvATsaf4ZGCwPUY97DmtPqA=", + "requires": { + "readable-stream": "1.0.34" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "snakeize": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz", @@ -6275,6 +6464,42 @@ } } }, + "unzip": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/unzip/-/unzip-0.1.11.tgz", + "integrity": "sha1-iXScY7BY19kNYZ+GuYqhU107l/A=", + "requires": { + "binary": "0.3.0", + "fstream": "0.1.31", + "match-stream": "0.0.2", + "pullstream": "0.4.1", + "readable-stream": "1.0.34", + "setimmediate": "1.0.5" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "urix": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", @@ -6380,7 +6605,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/which/-/which-1.3.0.tgz", "integrity": "sha512-xcJpopdamTuY5duC/KnTTNBraPK54YwpenP4lzxU8H91GudWpFv38u0CKjclE1Wi2EH2EDz5LRcHcKbCIzqGyg==", - "dev": true, "requires": { "isexe": "2.0.0" } diff --git a/functions/package.json b/functions/package.json index 5ebddeff..821f18bf 100644 --- a/functions/package.json +++ b/functions/package.json @@ -22,6 +22,7 @@ "ask-sdk": "^2.0.1", "axios": "^0.18.0", "bespoken-tools": "^1.2.8", + "child-process-promise": "^2.2.1", "dashbot": "^9.8.0", "debug": "^3.1.0", "firebase-admin": "^5.12.0", @@ -35,7 +36,8 @@ "object.entries": "^1.0.4", "raven": "^2.6.0", "replaceall": "^0.1.6", - "supports-color": "^5.4.0" + "supports-color": "^5.4.0", + "unzip": "^0.1.11" }, "devDependencies": { "axios-mock-adapter": "^1.15.0", diff --git a/functions/uploader/agent/agent_downloader.js b/functions/uploader/agent/agent_downloader.js index 8a15adc3..edde7d2a 100644 --- a/functions/uploader/agent/agent_downloader.js +++ b/functions/uploader/agent/agent_downloader.js @@ -1,40 +1,26 @@ -const fetch = require(`node-fetch`); +const axios = require('axios'); +const debug = require(`debug`)(`ia:uploader:agent:debug`); const error = require(`debug`)(`ia:uploader:agent:error`); +const fs = require('fs'); const unzip = require(`unzip`); const util = require(`util`); const { exec } = require('child-process-promise'); -const basicHeaderRequest = { - 'content-type': `application/json; charset=UTF-8`, - 'authorization': `BEARER ` -}; fetchAgentFromDF(); -/** - * Post entities to DialogFlow - * - * Maximum 1,000 records per request - * Maximum 30,000 records per entity - * - * @param entityname {string} - * @param entities {array} - * @param first {int} suppose to be 0 - * @returns {Promise} - */ function fetchAgentFromDF () { return getAccessToken() .then(accesstoken => { - return fetch('https://dialogflow.googleapis.com/v2/projects/project_id/agent:export?access_token='+accesstoken, {method: `POST`}); + return axios('https://dialogflow.googleapis.com/v2/projects/project_id/agent:export?access_token=' + accesstoken, {method: `POST`}); }) .then(res => res.json()) .then(data => { - console.log('Bimmy'+util.inspect(data.response.agentContent, false, null)); - base64_decode(data.response.agentContent, 'agent.zip'); - return base64_decode(data.response.agentContent, 'agent.zip');; + debug(util.inspect(data.response.agentContent, false, null)); + return base64Decode(data.response.agentContent, 'agent.zip'); }) .then(isDone => { fs.createReadStream('agent.zip').pipe(unzip.Extract({ path: './dynamic_agent' })); - return Promise.resolve("Done"); + return Promise.resolve('Done'); }) .catch(e => { error(`Get error in posting entity to DF, error: ${JSON.stringify(e)}`); @@ -43,14 +29,14 @@ function fetchAgentFromDF () { } function getAccessToken () { - console.log('getting access token'); + debug('getting access token'); return exec('gcloud auth print-access-token') .then(values => { var stdout = values.stdout; var stderr = values.stderr; if (stdout) { var filteredstdout = stdout.replace(/\n$/, ''); - console.log(util.inspect(filteredstdout, false, null)); + debug(util.inspect(filteredstdout, false, null)); return Promise.resolve(filteredstdout); } else if (stderr) { error(stderr); @@ -62,32 +48,29 @@ function getAccessToken () { }); } - -var fs = require('fs'); - // function to encode file data to base64 encoded string -function base64_encode(file) { - // read binary data - var bitmap = fs.readFileSync(file); - // convert binary data to base64 encoded string - return new Buffer(bitmap).toString('base64'); -} +// function base64Encode(file) { +// read binary data +// var bitmap = fs.readFileSync(file); +// convert binary data to base64 encoded string +// return new Buffer(bitmap).toString('base64'); +// } // function to create file from base64 encoded string -function base64_decode(base64str, file) { - // create buffer object from base64 encoded string, it is important to tell the constructor that the string is base64 encoded - var bitmap = new Buffer(base64str, 'base64'); - // write buffer to file - fs.writeFileSync(file, bitmap); - console.log('******** File created from base64 encoded string ********'); - return true; +function base64Decode (base64str, file) { + // create buffer object from base64 encoded string, it is important to tell the constructor that the string is base64 encoded + var bitmap = Buffer.alloc()(base64str, 'base64'); + // write buffer to file + fs.writeFileSync(file, bitmap); + debug('******** File created from base64 encoded string ********'); + return true; } // convert image to base64 encoded string -//var base64str = base64_encode('kitten.jpg'); -//console.log(base64str); -// convert base64 string back to image -//base64_decode(base64str, 'copy.jpg'); +// var base64str = base64_encode('kitten.jpg'); +// debug(base64str); +// convert base64 string back to image +// base64_decode(base64str, 'copy.jpg'); module.exports = { fetchAgentFromDF, From c0c957917a21c6be1daecd927a4303e2caae87dc Mon Sep 17 00:00:00 2001 From: Bimlendra Date: Fri, 11 May 2018 18:33:40 +0530 Subject: [PATCH 3/6] Integrated zip/unzip functions with promise & agent import script is also ready, having issue with axios for import script --- functions/package-lock.json | 536 +++++++++++++++++++ functions/package.json | 6 +- functions/uploader/agent/agent_downloader.js | 34 +- functions/uploader/agent/agent_uploader.js | 48 ++ functions/uploader/utils/base64.js | 37 ++ functions/uploader/utils/prepare_agent.js | 41 ++ 6 files changed, 671 insertions(+), 31 deletions(-) create mode 100644 functions/uploader/agent/agent_uploader.js create mode 100644 functions/uploader/utils/base64.js create mode 100644 functions/uploader/utils/prepare_agent.js diff --git a/functions/package-lock.json b/functions/package-lock.json index 415e2c7c..722d3a5a 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -492,6 +492,82 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" + }, + "archiver": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-0.11.0.tgz", + "integrity": "sha1-mBd9p6bAGSt/J5jzDNbquKvXZpA=", + "requires": { + "async": "0.9.2", + "buffer-crc32": "0.2.13", + "glob": "3.2.11", + "lazystream": "0.1.0", + "lodash": "2.4.2", + "readable-stream": "1.0.34", + "tar-stream": "0.4.7", + "zip-stream": "0.4.1" + }, + "dependencies": { + "async": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" + }, + "glob": { + "version": "3.2.11", + "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.11.tgz", + "integrity": "sha1-Spc/Y1uRkPcV0QmH1cAP0oFevj0=", + "requires": { + "inherits": "2.0.3", + "minimatch": "0.3.0" + } + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "lodash": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", + "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=" + }, + "lru-cache": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz", + "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=" + }, + "minimatch": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.3.0.tgz", + "integrity": "sha1-J12O2qxPG7MyZHIInnlJyDlGmd0=", + "requires": { + "lru-cache": "2.7.3", + "sigmund": "1.0.1" + } + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -846,6 +922,37 @@ "chainsaw": "0.1.0" } }, + "bl": { + "version": "0.9.5", + "resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz", + "integrity": "sha1-wGt5evCF6gC8Unr8jvzxHeIjIFQ=", + "requires": { + "readable-stream": "1.0.34" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "body-parser": { "version": "1.18.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", @@ -933,6 +1040,11 @@ "isarray": "1.0.0" } }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, "buffer-equal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz", @@ -1307,6 +1419,39 @@ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" }, + "compress-commons": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-0.1.6.tgz", + "integrity": "sha1-DHQIcP3ljLpRbwrAyCLjOguF36M=", + "requires": { + "buffer-crc32": "0.2.13", + "crc32-stream": "0.3.4", + "readable-stream": "1.0.34" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "compressible": { "version": "2.0.13", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.13.tgz", @@ -1449,6 +1594,38 @@ "vary": "1.1.2" } }, + "crc32-stream": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-0.3.4.tgz", + "integrity": "sha1-c7wltF+sHbZjIjGnv86JJ+nwZVI=", + "requires": { + "buffer-crc32": "0.2.13", + "readable-stream": "1.0.34" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "create-error-class": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", @@ -1739,6 +1916,24 @@ "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.1.1.tgz", "integrity": "sha1-RNZW3p2kFWlEZzNTZfsxR7hXK3w=" }, + "easy-zip": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/easy-zip/-/easy-zip-0.0.4.tgz", + "integrity": "sha1-sto31nUCIYYKrvAWjekS6/6VfZM=", + "requires": { + "async": "2.6.0" + }, + "dependencies": { + "async": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", + "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==", + "requires": { + "lodash": "4.17.10" + } + } + } + }, "ecc-jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", @@ -2593,6 +2788,46 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "fs-extra": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-6.0.1.tgz", + "integrity": "sha512-GnyIkKhhzXZUWFCaJzvyDLEEgDkPfb4/TPvJCJVuS8MWZgoSsErf++QpiAlDnKFcqhRlm+tIOcencCjyJE6ZCA==", + "requires": { + "graceful-fs": "4.1.11", + "jsonfile": "4.0.0", + "universalify": "0.1.1" + } + }, + "fs-promise": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/fs-promise/-/fs-promise-2.0.3.tgz", + "integrity": "sha1-9k5PhUvPaJqovdy6JokW2z20aFQ=", + "requires": { + "any-promise": "1.3.0", + "fs-extra": "2.1.2", + "mz": "2.7.0", + "thenify-all": "1.6.0" + }, + "dependencies": { + "fs-extra": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-2.1.2.tgz", + "integrity": "sha1-BGxwFjzvmq1GsOSn+kZ/si1x3jU=", + "requires": { + "graceful-fs": "4.1.11", + "jsonfile": "2.4.0" + } + }, + "jsonfile": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", + "requires": { + "graceful-fs": "4.1.11" + } + } + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -4209,6 +4444,14 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "4.1.11" + } + }, "jsonwebtoken": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.1.0.tgz", @@ -4277,6 +4520,37 @@ "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" }, + "lazystream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-0.1.0.tgz", + "integrity": "sha1-GyXWPHcqTCDwpe0KnXf0hLbhaSA=", + "requires": { + "readable-stream": "1.0.34" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "lcid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", @@ -4744,6 +5018,21 @@ "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, + "mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "requires": { + "any-promise": "1.3.0", + "object-assign": "4.1.1", + "thenify-all": "1.6.0" + } + }, + "n": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/n/-/n-2.1.10.tgz", + "integrity": "sha512-7SB/P9HsNCDl21WyAkFcdNtD+E22akltkSDuRaKqr7y1PmfKfrUIZYfPRSjS0f8MtSctgqzVkbLo6lvdzAEfmQ==" + }, "nan": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", @@ -4824,6 +5113,11 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.4.tgz", "integrity": "sha512-8Df0906+tq/omxuCZD6PqhPaQDYuyJ1d+VITgxoIA8zvQd1ru+nMJcDChHH324MWitIgbVkAkQoGEEVJNpn/PA==" }, + "node-int64": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.3.3.tgz", + "integrity": "sha1-LW5rLs5d6FiLQ9iNG8QbJs0fqE0=" + }, "node-version": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/node-version/-/node-version-1.1.3.tgz", @@ -5362,6 +5656,11 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" + }, "qs": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", @@ -5797,6 +6096,11 @@ "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.0.tgz", "integrity": "sha512-xTCx2vohXC2EWWDqY/zb4+5Mu28D+HYNSOuFzsyRDRvI/e1ICb69afwaUwfjr+25ZXldbOLyp+iDUZHq8UnTag==" }, + "sigmund": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", + "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=" + }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", @@ -6213,6 +6517,40 @@ "string-width": "2.1.1" } }, + "tar-stream": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-0.4.7.tgz", + "integrity": "sha1-Hx0s6evHtCdlJDyg6PG3v9oKrc0=", + "requires": { + "bl": "0.9.5", + "end-of-stream": "1.4.1", + "readable-stream": "1.1.14", + "xtend": "4.0.1" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "text-encoding": { "version": "0.6.4", "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", @@ -6225,6 +6563,22 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "thenify": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.0.tgz", + "integrity": "sha1-5p44obq+lpsBCCB5eLn2K4hgSDk=", + "requires": { + "any-promise": "1.3.0" + } + }, + "thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=", + "requires": { + "thenify": "3.3.0" + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -6423,6 +6777,11 @@ "object-keys": "1.0.11" } }, + "universalify": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", + "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -6779,6 +7138,183 @@ } } } + }, + "zip-dir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/zip-dir/-/zip-dir-1.0.2.tgz", + "integrity": "sha1-JT+QeurWKiGs2HIdi4gDKyQRwFE=", + "requires": { + "async": "1.5.2", + "jszip": "2.5.0" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + } + } + }, + "zip-folder": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/zip-folder/-/zip-folder-1.0.0.tgz", + "integrity": "sha1-cKd0T9F4mi/rQa00GbMun9h5V7I=", + "requires": { + "archiver": "0.11.0" + } + }, + "zip-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-0.4.1.tgz", + "integrity": "sha1-TqeVqM4Z6fq0mjHR0IdyFBWfA6M=", + "requires": { + "compress-commons": "0.1.6", + "lodash": "2.4.2", + "readable-stream": "1.0.34" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "lodash": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", + "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, + "zipfolder": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/zipfolder/-/zipfolder-1.0.4.tgz", + "integrity": "sha1-+VvP1TFREA4chIbHb52+2+QpNs0=", + "requires": { + "archiver": "0.13.1", + "q": "1.5.1" + }, + "dependencies": { + "archiver": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-0.13.1.tgz", + "integrity": "sha1-r8azVhpjZDs4omeIkSqDhU7KoPE=", + "requires": { + "async": "0.9.2", + "buffer-crc32": "0.2.13", + "glob": "4.3.5", + "lazystream": "0.1.0", + "lodash": "2.4.2", + "readable-stream": "1.0.34", + "tar-stream": "1.1.5", + "zip-stream": "0.5.2" + } + }, + "async": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" + }, + "compress-commons": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-0.2.9.tgz", + "integrity": "sha1-Qi2SdDDAGr0GzUVbbfwEy0z4ADw=", + "requires": { + "buffer-crc32": "0.2.13", + "crc32-stream": "0.3.4", + "node-int64": "0.3.3", + "readable-stream": "1.0.34" + } + }, + "glob": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-4.3.5.tgz", + "integrity": "sha1-gPuwjKVA8jiszl0R0em8QedRc9M=", + "requires": { + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "2.0.10", + "once": "1.4.0" + } + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "lodash": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", + "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=" + }, + "minimatch": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-2.0.10.tgz", + "integrity": "sha1-jQh8OcazjAAbl/ynzm0OHoCvusc=", + "requires": { + "brace-expansion": "1.1.11" + } + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "tar-stream": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz", + "integrity": "sha1-vpIYwTDCACnhB7D5Z/sj3gV50Tw=", + "requires": { + "bl": "0.9.5", + "end-of-stream": "1.4.1", + "readable-stream": "1.0.34", + "xtend": "4.0.1" + } + }, + "zip-stream": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-0.5.2.tgz", + "integrity": "sha1-Mty8UG0Nq00hNyYlvX66rDwv/1Y=", + "requires": { + "compress-commons": "0.2.9", + "lodash": "3.2.0", + "readable-stream": "1.0.34" + }, + "dependencies": { + "lodash": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz", + "integrity": "sha1-S/UKMkP5rrC6xBpV09WZBnWkYvs=" + } + } + } + } } } } diff --git a/functions/package.json b/functions/package.json index 821f18bf..ac5a458c 100644 --- a/functions/package.json +++ b/functions/package.json @@ -25,6 +25,7 @@ "child-process-promise": "^2.2.1", "dashbot": "^9.8.0", "debug": "^3.1.0", + "easy-zip": "0.0.4", "firebase-admin": "^5.12.0", "firebase-functions": "^1.0.2", "glob": "^7.1.2", @@ -32,12 +33,15 @@ "lodash": "^4.17.10", "mathjs": "^4.1.2", "mustache": "^2.3.0", + "n": "^2.1.10", "node-fetch": "^2.1.2", + "node-zip": "^1.1.1", "object.entries": "^1.0.4", "raven": "^2.6.0", "replaceall": "^0.1.6", "supports-color": "^5.4.0", - "unzip": "^0.1.11" + "unzip": "^0.1.11", + "zipfolder": "^1.0.4" }, "devDependencies": { "axios-mock-adapter": "^1.15.0", diff --git a/functions/uploader/agent/agent_downloader.js b/functions/uploader/agent/agent_downloader.js index edde7d2a..175ca2bc 100644 --- a/functions/uploader/agent/agent_downloader.js +++ b/functions/uploader/agent/agent_downloader.js @@ -1,26 +1,18 @@ const axios = require('axios'); const debug = require(`debug`)(`ia:uploader:agent:debug`); const error = require(`debug`)(`ia:uploader:agent:error`); -const fs = require('fs'); -const unzip = require(`unzip`); const util = require(`util`); const { exec } = require('child-process-promise'); - -fetchAgentFromDF(); +const {prepareAgentToFetchFromDF} = require('../utils/prepare_agent'); function fetchAgentFromDF () { return getAccessToken() .then(accesstoken => { - return axios('https://dialogflow.googleapis.com/v2/projects/project_id/agent:export?access_token=' + accesstoken, {method: `POST`}); + return axios('https://dialogflow.googleapis.com/v2/projects/music-a88c1/agent:export?access_token=' + accesstoken, {method: `POST`}); }) - .then(res => res.json()) + .then(res => res.data) .then(data => { - debug(util.inspect(data.response.agentContent, false, null)); - return base64Decode(data.response.agentContent, 'agent.zip'); - }) - .then(isDone => { - fs.createReadStream('agent.zip').pipe(unzip.Extract({ path: './dynamic_agent' })); - return Promise.resolve('Done'); + return prepareAgentToFetchFromDF(data.response.agentContent, '../../agent/', 'agent.zip'); }) .catch(e => { error(`Get error in posting entity to DF, error: ${JSON.stringify(e)}`); @@ -48,24 +40,6 @@ function getAccessToken () { }); } -// function to encode file data to base64 encoded string -// function base64Encode(file) { -// read binary data -// var bitmap = fs.readFileSync(file); -// convert binary data to base64 encoded string -// return new Buffer(bitmap).toString('base64'); -// } - -// function to create file from base64 encoded string -function base64Decode (base64str, file) { - // create buffer object from base64 encoded string, it is important to tell the constructor that the string is base64 encoded - var bitmap = Buffer.alloc()(base64str, 'base64'); - // write buffer to file - fs.writeFileSync(file, bitmap); - debug('******** File created from base64 encoded string ********'); - return true; -} - // convert image to base64 encoded string // var base64str = base64_encode('kitten.jpg'); // debug(base64str); diff --git a/functions/uploader/agent/agent_uploader.js b/functions/uploader/agent/agent_uploader.js new file mode 100644 index 00000000..5425b913 --- /dev/null +++ b/functions/uploader/agent/agent_uploader.js @@ -0,0 +1,48 @@ +const debug = require(`debug`)(`ia:uploader:agent:agent_uploader:debug`); +const error = require(`debug`)(`ia:uploader:agent:agent_uploader:error`); +const fetch = require(`node-fetch`); +const util = require(`util`); +const { exec } = require('child-process-promise'); +const {prepareAgentToPostToDF} = require('../utils/prepare_agent'); + +postAgentToDF(); + +function postAgentToDF () { + return Promise.all([getAccessToken(), prepareAgentToPostToDF('my_agent', './')]) + .then(values => { + const [accesstoken, base64] = values; + return fetch('https://dialogflow.googleapis.com/v2/projects/music-a88c1/agent:import?access_token=' + accesstoken, {method: `POST`, body: JSON.stringify({'agentContent': base64})}); + }) + .then(res => res.json()) + .then(data => { + debug(util.inspect(data, false, null)); + }) + .catch(e => { + error(`Get error in posting entity to DF, error: ${util.inspect(e, false, null)}`); + return Promise.reject(e); + }); +} + +function getAccessToken () { + debug('getting access token'); + return exec('gcloud auth print-access-token') + .then(values => { + var stdout = values.stdout; + var stderr = values.stderr; + if (stdout) { + var filteredstdout = stdout.replace(/\n$/, ''); + debug(util.inspect(filteredstdout, false, null)); + return Promise.resolve(filteredstdout); + } else if (stderr) { + error(stderr); + return Promise.reject(new Error('ERROR: ' + stderr)); + } else { + error('Having trouble with GCloud execution'); + return Promise.reject(new Error('ERROR: Having trouble with GCloud execution')); + } + }); +} + +module.exports = { + postAgentToDF, +}; diff --git a/functions/uploader/utils/base64.js b/functions/uploader/utils/base64.js new file mode 100644 index 00000000..380c15af --- /dev/null +++ b/functions/uploader/utils/base64.js @@ -0,0 +1,37 @@ +const error = require(`debug`)(`ia:uploader:base64:error`); +const fs = require('fs'); + +// function to encode file data to base64 encoded string +function base64Encode (file) { +// read binary data + return new Promise(function (resolve, reject) { + fs.readFile(file, function (err, bitmap) { + if (err) return reject(err); + else return resolve(Buffer.from(bitmap).toString('base64')); + }); + }) + .catch(function (err) { + error(err); + }); +} + +// function to create file from base64 encoded string +function base64Decode (base64str, file) { +// create buffer object from base64 encoded string, it is important to tell the constructor that the string is base64 encoded + var bitmap = Buffer.from(base64str, 'base64'); + // write buffer to file + return new Promise(function (resolve, reject) { + fs.writeFile(file, bitmap, 'utf8', function (err) { + if (err) return reject(err); + else return resolve(bitmap); + }); + }) + .catch(function (err) { + error(err); + }); +} + +module.exports = { + base64Encode, + base64Decode, +}; diff --git a/functions/uploader/utils/prepare_agent.js b/functions/uploader/utils/prepare_agent.js new file mode 100644 index 00000000..a87fcba0 --- /dev/null +++ b/functions/uploader/utils/prepare_agent.js @@ -0,0 +1,41 @@ +const error = require(`debug`)(`ia:prepare_agent:agent:error`); +const fs = require('fs'); +const unzip = require(`unzip`); +const zip = require('zipfolder'); + +const {base64Encode, base64Decode} = require('../utils/base64'); + +// function to encode file data to base64 encoded string +function prepareAgentToPostToDF (folderPath, zipPath) { + return zip.zipFolder({folderPath: folderPath, targetFolderPath: zipPath}) + .then(function (path) { + return Promise.resolve(base64Encode(path)); + }, function (err) { + return Promise.reject(err); + }); +} + +// function to create file from base64 encoded string +function prepareAgentToFetchFromDF (base64str, folderPath, zipPath) { + return base64Decode(base64str, zipPath) + .then(isDone => { + return fs.createReadStream(zipPath).pipe(unzip.Extract({ path: folderPath })); + }) + .then(close => { + return new Promise(function (resolve, reject) { + fs.unlink(zipPath, function (err) { + if (err) return reject(err); + else return resolve(zipPath); + }); + }); + }) + .catch(e => { + error(`Get error in generating agent.zip, error: ${JSON.stringify(e)}`); + return Promise.reject(e); + }); +} + +module.exports = { + prepareAgentToPostToDF, + prepareAgentToFetchFromDF, +}; From 8cf6d353ade448c61b4d05f56c1c6d73d3a12928 Mon Sep 17 00:00:00 2001 From: Bimlendra Date: Sat, 19 May 2018 17:43:14 +0530 Subject: [PATCH 4/6] resolved all requests & axios is working for import script now, few structure changes --- functions/package-lock.json | 334 +++--------------- functions/package.json | 3 - functions/uploader/agent/agent-downloader.js | 35 ++ functions/uploader/agent/agent-uploader.js | 36 ++ functions/uploader/agent/agent_downloader.js | 51 --- functions/uploader/config.js | 5 + functions/uploader/index.js | 9 + functions/uploader/utils/base64.js | 2 +- .../get-access-token.js} | 28 +- .../{prepare_agent.js => prepare-agent.js} | 14 +- 10 files changed, 151 insertions(+), 366 deletions(-) create mode 100644 functions/uploader/agent/agent-downloader.js create mode 100644 functions/uploader/agent/agent-uploader.js delete mode 100644 functions/uploader/agent/agent_downloader.js rename functions/uploader/{agent/agent_uploader.js => utils/get-access-token.js} (55%) rename functions/uploader/utils/{prepare_agent.js => prepare-agent.js} (69%) diff --git a/functions/package-lock.json b/functions/package-lock.json index 722d3a5a..57a42201 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -492,82 +492,6 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" }, - "any-promise": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" - }, - "archiver": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/archiver/-/archiver-0.11.0.tgz", - "integrity": "sha1-mBd9p6bAGSt/J5jzDNbquKvXZpA=", - "requires": { - "async": "0.9.2", - "buffer-crc32": "0.2.13", - "glob": "3.2.11", - "lazystream": "0.1.0", - "lodash": "2.4.2", - "readable-stream": "1.0.34", - "tar-stream": "0.4.7", - "zip-stream": "0.4.1" - }, - "dependencies": { - "async": { - "version": "0.9.2", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", - "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" - }, - "glob": { - "version": "3.2.11", - "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.11.tgz", - "integrity": "sha1-Spc/Y1uRkPcV0QmH1cAP0oFevj0=", - "requires": { - "inherits": "2.0.3", - "minimatch": "0.3.0" - } - }, - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "lodash": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", - "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=" - }, - "lru-cache": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz", - "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=" - }, - "minimatch": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.3.0.tgz", - "integrity": "sha1-J12O2qxPG7MyZHIInnlJyDlGmd0=", - "requires": { - "lru-cache": "2.7.3", - "sigmund": "1.0.1" - } - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "0.0.1", - "string_decoder": "0.10.31" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } - } - }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -1419,39 +1343,6 @@ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" }, - "compress-commons": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-0.1.6.tgz", - "integrity": "sha1-DHQIcP3ljLpRbwrAyCLjOguF36M=", - "requires": { - "buffer-crc32": "0.2.13", - "crc32-stream": "0.3.4", - "readable-stream": "1.0.34" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "0.0.1", - "string_decoder": "0.10.31" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } - } - }, "compressible": { "version": "2.0.13", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.13.tgz", @@ -2788,46 +2679,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, - "fs-extra": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-6.0.1.tgz", - "integrity": "sha512-GnyIkKhhzXZUWFCaJzvyDLEEgDkPfb4/TPvJCJVuS8MWZgoSsErf++QpiAlDnKFcqhRlm+tIOcencCjyJE6ZCA==", - "requires": { - "graceful-fs": "4.1.11", - "jsonfile": "4.0.0", - "universalify": "0.1.1" - } - }, - "fs-promise": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/fs-promise/-/fs-promise-2.0.3.tgz", - "integrity": "sha1-9k5PhUvPaJqovdy6JokW2z20aFQ=", - "requires": { - "any-promise": "1.3.0", - "fs-extra": "2.1.2", - "mz": "2.7.0", - "thenify-all": "1.6.0" - }, - "dependencies": { - "fs-extra": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-2.1.2.tgz", - "integrity": "sha1-BGxwFjzvmq1GsOSn+kZ/si1x3jU=", - "requires": { - "graceful-fs": "4.1.11", - "jsonfile": "2.4.0" - } - }, - "jsonfile": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", - "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", - "requires": { - "graceful-fs": "4.1.11" - } - } - } - }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -4444,14 +4295,6 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, - "jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", - "requires": { - "graceful-fs": "4.1.11" - } - }, "jsonwebtoken": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.1.0.tgz", @@ -5018,16 +4861,6 @@ "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, - "mz": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", - "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "requires": { - "any-promise": "1.3.0", - "object-assign": "4.1.1", - "thenify-all": "1.6.0" - } - }, "n": { "version": "2.1.10", "resolved": "https://registry.npmjs.org/n/-/n-2.1.10.tgz", @@ -5217,6 +5050,51 @@ "isobject": "3.0.1" } }, + "omit-deep": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/omit-deep/-/omit-deep-0.3.0.tgz", + "integrity": "sha1-IcivNJm8rdKWUaIyy8rLxSRF6+w=", + "requires": { + "is-plain-object": "2.0.4", + "unset-value": "0.1.2" + }, + "dependencies": { + "has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=", + "requires": { + "get-value": "2.0.6", + "has-values": "0.1.4", + "isobject": "2.1.0" + }, + "dependencies": { + "isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=", + "requires": { + "isarray": "1.0.0" + } + } + } + }, + "has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E=" + }, + "unset-value": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-0.1.2.tgz", + "integrity": "sha1-UGgQuGfyfCpabpsEgzYx9t5Y0xA=", + "requires": { + "has-value": "0.3.1", + "isobject": "3.0.1" + } + } + } + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -6096,11 +5974,6 @@ "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.0.tgz", "integrity": "sha512-xTCx2vohXC2EWWDqY/zb4+5Mu28D+HYNSOuFzsyRDRvI/e1ICb69afwaUwfjr+25ZXldbOLyp+iDUZHq8UnTag==" }, - "sigmund": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", - "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=" - }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", @@ -6517,40 +6390,6 @@ "string-width": "2.1.1" } }, - "tar-stream": { - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-0.4.7.tgz", - "integrity": "sha1-Hx0s6evHtCdlJDyg6PG3v9oKrc0=", - "requires": { - "bl": "0.9.5", - "end-of-stream": "1.4.1", - "readable-stream": "1.1.14", - "xtend": "4.0.1" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "readable-stream": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", - "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "0.0.1", - "string_decoder": "0.10.31" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } - } - }, "text-encoding": { "version": "0.6.4", "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", @@ -6563,22 +6402,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "thenify": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.0.tgz", - "integrity": "sha1-5p44obq+lpsBCCB5eLn2K4hgSDk=", - "requires": { - "any-promise": "1.3.0" - } - }, - "thenify-all": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", - "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=", - "requires": { - "thenify": "3.3.0" - } - }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -6777,11 +6600,6 @@ "object-keys": "1.0.11" } }, - "universalify": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", - "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=" - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -7139,68 +6957,6 @@ } } }, - "zip-dir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/zip-dir/-/zip-dir-1.0.2.tgz", - "integrity": "sha1-JT+QeurWKiGs2HIdi4gDKyQRwFE=", - "requires": { - "async": "1.5.2", - "jszip": "2.5.0" - }, - "dependencies": { - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" - } - } - }, - "zip-folder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/zip-folder/-/zip-folder-1.0.0.tgz", - "integrity": "sha1-cKd0T9F4mi/rQa00GbMun9h5V7I=", - "requires": { - "archiver": "0.11.0" - } - }, - "zip-stream": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-0.4.1.tgz", - "integrity": "sha1-TqeVqM4Z6fq0mjHR0IdyFBWfA6M=", - "requires": { - "compress-commons": "0.1.6", - "lodash": "2.4.2", - "readable-stream": "1.0.34" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "lodash": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", - "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=" - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "0.0.1", - "string_decoder": "0.10.31" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } - } - }, "zipfolder": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/zipfolder/-/zipfolder-1.0.4.tgz", diff --git a/functions/package.json b/functions/package.json index ac5a458c..17e8f956 100644 --- a/functions/package.json +++ b/functions/package.json @@ -25,7 +25,6 @@ "child-process-promise": "^2.2.1", "dashbot": "^9.8.0", "debug": "^3.1.0", - "easy-zip": "0.0.4", "firebase-admin": "^5.12.0", "firebase-functions": "^1.0.2", "glob": "^7.1.2", @@ -33,9 +32,7 @@ "lodash": "^4.17.10", "mathjs": "^4.1.2", "mustache": "^2.3.0", - "n": "^2.1.10", "node-fetch": "^2.1.2", - "node-zip": "^1.1.1", "object.entries": "^1.0.4", "raven": "^2.6.0", "replaceall": "^0.1.6", diff --git a/functions/uploader/agent/agent-downloader.js b/functions/uploader/agent/agent-downloader.js new file mode 100644 index 00000000..3ca0f67b --- /dev/null +++ b/functions/uploader/agent/agent-downloader.js @@ -0,0 +1,35 @@ +const axios = require('axios'); +const debug = require(`debug`)(`ia:uploader:agent:agent-downloader:debug`); +const error = require(`debug`)(`ia:uploader:agent:agent-downloader:error`); +const {getAccessToken} = require('../utils/get-access-token'); +const {prepareAgentToFetchFromDF} = require('../utils/prepare-agent'); + +const config = require('../config'); +const mustache = require('mustache'); + +function downloadAgent () { + return getAccessToken() + .then(accessToken => { + var agentId = config.agent.ID; + return axios(mustache.render( + config.dfendpoints.AGENT_EXPORT_URL, + { + agentId, + accessToken, + } + ), {method: `POST`}); + }) + .then(res => res.data) + .then(data => { + debug(data); + return prepareAgentToFetchFromDF(data.response.agentContent, '../../agent/', 'agent.zip'); + }) + .catch(e => { + error(`Get error in downloading agent from DF, error: ${e}`); + return Promise.reject(e); + }); +} + +module.exports = { + downloadAgent, +}; diff --git a/functions/uploader/agent/agent-uploader.js b/functions/uploader/agent/agent-uploader.js new file mode 100644 index 00000000..f4a6964c --- /dev/null +++ b/functions/uploader/agent/agent-uploader.js @@ -0,0 +1,36 @@ +const axios = require('axios'); +const debug = require(`debug`)(`ia:uploader:agent:agent-uploader:debug`); +const error = require(`debug`)(`ia:uploader:agent:agent-uploader:error`); +const util = require(`util`); +const {getAccessToken} = require('../utils/get-access-token'); +const {prepareAgentToPostToDF} = require('../utils/prepare-agent'); + +const config = require('../config'); +const mustache = require('mustache'); + +function uploadAgent () { + return Promise.all([getAccessToken(), prepareAgentToPostToDF('../../agent', './')]) + .then(values => { + const [accessToken, base64] = values; + var agentId = config.agent.ID; + return axios(mustache.render( + config.dfendpoints.AGENT_IMPORT_URL, + { + agentId, + accessToken, + } + ), {headers: {'content-type': `application/json; charset=UTF-8`}, method: `POST`, data: JSON.stringify({'agentContent': base64})}); + }) + .then(res => res.data) + .then(data => { + debug(util.inspect(data, false, null)); + }) + .catch(e => { + error(`Get error in uploading agent from DF, error: ${e}`); + return Promise.reject(e); + }); +} + +module.exports = { + uploadAgent, +}; diff --git a/functions/uploader/agent/agent_downloader.js b/functions/uploader/agent/agent_downloader.js deleted file mode 100644 index 175ca2bc..00000000 --- a/functions/uploader/agent/agent_downloader.js +++ /dev/null @@ -1,51 +0,0 @@ -const axios = require('axios'); -const debug = require(`debug`)(`ia:uploader:agent:debug`); -const error = require(`debug`)(`ia:uploader:agent:error`); -const util = require(`util`); -const { exec } = require('child-process-promise'); -const {prepareAgentToFetchFromDF} = require('../utils/prepare_agent'); - -function fetchAgentFromDF () { - return getAccessToken() - .then(accesstoken => { - return axios('https://dialogflow.googleapis.com/v2/projects/music-a88c1/agent:export?access_token=' + accesstoken, {method: `POST`}); - }) - .then(res => res.data) - .then(data => { - return prepareAgentToFetchFromDF(data.response.agentContent, '../../agent/', 'agent.zip'); - }) - .catch(e => { - error(`Get error in posting entity to DF, error: ${JSON.stringify(e)}`); - return Promise.reject(e); - }); -} - -function getAccessToken () { - debug('getting access token'); - return exec('gcloud auth print-access-token') - .then(values => { - var stdout = values.stdout; - var stderr = values.stderr; - if (stdout) { - var filteredstdout = stdout.replace(/\n$/, ''); - debug(util.inspect(filteredstdout, false, null)); - return Promise.resolve(filteredstdout); - } else if (stderr) { - error(stderr); - return Promise.reject(new Error('ERROR: ' + stderr)); - } else { - error('Having trouble with GCloud execution'); - return Promise.reject(new Error('ERROR: Having trouble with GCloud execution')); - } - }); -} - -// convert image to base64 encoded string -// var base64str = base64_encode('kitten.jpg'); -// debug(base64str); -// convert base64 string back to image -// base64_decode(base64str, 'copy.jpg'); - -module.exports = { - fetchAgentFromDF, -}; diff --git a/functions/uploader/config.js b/functions/uploader/config.js index 6249e9c3..823d404c 100644 --- a/functions/uploader/config.js +++ b/functions/uploader/config.js @@ -3,6 +3,11 @@ module.exports = { DF_ENTITY_GET_URL: 'https://api.dialogflow.com/v1/entities/{{entityname}}?v=20150910', DF_ENTITY_POST_URL: 'https://api.dialogflow.com/v1/entities/{{entityname}}/entries?v=20150910', DF_ENTITY_DELETE_URL: 'https://api.dialogflow.com/v1/entities/{{entityname}}?v=20150910', + AGENT_EXPORT_URL: 'https://dialogflow.googleapis.com/v2/projects/{{agentId}}/agent:export?access_token={{accessToken}}', + AGENT_IMPORT_URL: 'https://dialogflow.googleapis.com/v2/projects/{{agentId}}/agent:import?access_token={{accessToken}}', + }, + agent: { + ID: 'agent_id_will_goes_here' }, uploader: { collection: { diff --git a/functions/uploader/index.js b/functions/uploader/index.js index 8dc7fe05..7d91b75c 100644 --- a/functions/uploader/index.js +++ b/functions/uploader/index.js @@ -9,6 +9,8 @@ const error = require(`debug`)(`ia:uploader:index:error`); const {uploadCollection} = require('./entities/collection'); const {uploadGenres} = require('./entities/genres'); +const {downloadAgent} = require('./agent/agent-downloader'); +const {uploadAgent} = require('./agent/agent-uploader'); const ALL = `all`; @@ -16,7 +18,14 @@ const entities = { collection: uploadCollection, genres: uploadGenres, }; + +const agent = { + upload: uploadAgent, + download: downloadAgent, +}; + const all = { + agent: agent, entities: entities, }; diff --git a/functions/uploader/utils/base64.js b/functions/uploader/utils/base64.js index 380c15af..de268e12 100644 --- a/functions/uploader/utils/base64.js +++ b/functions/uploader/utils/base64.js @@ -1,4 +1,4 @@ -const error = require(`debug`)(`ia:uploader:base64:error`); +const error = require(`debug`)(`ia:uploader:utils:base64:error`); const fs = require('fs'); // function to encode file data to base64 encoded string diff --git a/functions/uploader/agent/agent_uploader.js b/functions/uploader/utils/get-access-token.js similarity index 55% rename from functions/uploader/agent/agent_uploader.js rename to functions/uploader/utils/get-access-token.js index 5425b913..c7c578ec 100644 --- a/functions/uploader/agent/agent_uploader.js +++ b/functions/uploader/utils/get-access-token.js @@ -1,21 +1,16 @@ -const debug = require(`debug`)(`ia:uploader:agent:agent_uploader:debug`); -const error = require(`debug`)(`ia:uploader:agent:agent_uploader:error`); -const fetch = require(`node-fetch`); +const debug = require(`debug`)(`ia:uploader:utils:get-access-token:debug`); +const error = require(`debug`)(`ia:uploader:utils:get-access-token:error`); const util = require(`util`); const { exec } = require('child-process-promise'); -const {prepareAgentToPostToDF} = require('../utils/prepare_agent'); -postAgentToDF(); - -function postAgentToDF () { - return Promise.all([getAccessToken(), prepareAgentToPostToDF('my_agent', './')]) - .then(values => { - const [accesstoken, base64] = values; - return fetch('https://dialogflow.googleapis.com/v2/projects/music-a88c1/agent:import?access_token=' + accesstoken, {method: `POST`, body: JSON.stringify({'agentContent': base64})}); - }) - .then(res => res.json()) - .then(data => { - debug(util.inspect(data, false, null)); +function getBasicHeaderRequest () { + return getAccessToken() + .then(accessToken => { + const basicHeaderRequest = { + 'content-type': `application/json; charset=UTF-8`, + 'authorization': `BEARER ${accessToken}` + }; + return Promise.resolve(basicHeaderRequest); }) .catch(e => { error(`Get error in posting entity to DF, error: ${util.inspect(e, false, null)}`); @@ -44,5 +39,6 @@ function getAccessToken () { } module.exports = { - postAgentToDF, + getBasicHeaderRequest, + getAccessToken, }; diff --git a/functions/uploader/utils/prepare_agent.js b/functions/uploader/utils/prepare-agent.js similarity index 69% rename from functions/uploader/utils/prepare_agent.js rename to functions/uploader/utils/prepare-agent.js index a87fcba0..b66cbb06 100644 --- a/functions/uploader/utils/prepare_agent.js +++ b/functions/uploader/utils/prepare-agent.js @@ -1,17 +1,18 @@ -const error = require(`debug`)(`ia:prepare_agent:agent:error`); +const error = require(`debug`)(`ia:uploader:agent:utils:prepare_agent:error`); +const debug = require(`debug`)(`ia:uploader:agent:utils:prepare_agent:debug`); const fs = require('fs'); const unzip = require(`unzip`); const zip = require('zipfolder'); -const {base64Encode, base64Decode} = require('../utils/base64'); +const {base64Encode, base64Decode} = require('./base64'); // function to encode file data to base64 encoded string function prepareAgentToPostToDF (folderPath, zipPath) { return zip.zipFolder({folderPath: folderPath, targetFolderPath: zipPath}) .then(function (path) { - return Promise.resolve(base64Encode(path)); - }, function (err) { - return Promise.reject(err); + var encodedBase64 = base64Encode(path); + debug(encodedBase64); + return Promise.resolve(encodedBase64); }); } @@ -22,6 +23,7 @@ function prepareAgentToFetchFromDF (base64str, folderPath, zipPath) { return fs.createReadStream(zipPath).pipe(unzip.Extract({ path: folderPath })); }) .then(close => { + debug(close); return new Promise(function (resolve, reject) { fs.unlink(zipPath, function (err) { if (err) return reject(err); @@ -30,7 +32,7 @@ function prepareAgentToFetchFromDF (base64str, folderPath, zipPath) { }); }) .catch(e => { - error(`Get error in generating agent.zip, error: ${JSON.stringify(e)}`); + error(`Get error in generating agent.zip, error: ${e}`); return Promise.reject(e); }); } From 73339896b9248a5073faeb4f9bb5d77a61225135 Mon Sep 17 00:00:00 2001 From: Bimlendra Date: Fri, 25 May 2018 21:41:54 +0530 Subject: [PATCH 5/6] test cases added for agent --- .../uploader/agent/agent-downloader.spec.js | 12 ++++++++++++ .../tests/uploader/agent/agent-uploader.spec.js | 12 ++++++++++++ functions/tests/uploader/utils/base64.spec.js | 17 +++++++++++++++++ .../uploader/utils/get-access-token.spec.js | 17 +++++++++++++++++ .../tests/uploader/utils/prepare-agent.spec.js | 17 +++++++++++++++++ 5 files changed, 75 insertions(+) create mode 100644 functions/tests/uploader/agent/agent-downloader.spec.js create mode 100644 functions/tests/uploader/agent/agent-uploader.spec.js create mode 100644 functions/tests/uploader/utils/base64.spec.js create mode 100644 functions/tests/uploader/utils/get-access-token.spec.js create mode 100644 functions/tests/uploader/utils/prepare-agent.spec.js diff --git a/functions/tests/uploader/agent/agent-downloader.spec.js b/functions/tests/uploader/agent/agent-downloader.spec.js new file mode 100644 index 00000000..e8ad4ad8 --- /dev/null +++ b/functions/tests/uploader/agent/agent-downloader.spec.js @@ -0,0 +1,12 @@ +const {expect} = require('chai'); +const agentDownloader = require('../../../uploader/agent/agent-downloader'); + +describe('uploader', () => { + describe('agent-downloader', () => { + describe('downloadAgent', () => { + it('should be defined', () => { + expect(agentDownloader.downloadAgent).to.be.ok; + }); + }); + }); +}); diff --git a/functions/tests/uploader/agent/agent-uploader.spec.js b/functions/tests/uploader/agent/agent-uploader.spec.js new file mode 100644 index 00000000..c5575fbf --- /dev/null +++ b/functions/tests/uploader/agent/agent-uploader.spec.js @@ -0,0 +1,12 @@ +const {expect} = require('chai'); +const agentUploader = require('../../../uploader/agent/agent-uploader'); + +describe('uploader', () => { + describe('agent-uploader', () => { + describe('uploadAgent', () => { + it('should be defined', () => { + expect(agentUploader.uploadAgent).to.be.ok; + }); + }); + }); +}); diff --git a/functions/tests/uploader/utils/base64.spec.js b/functions/tests/uploader/utils/base64.spec.js new file mode 100644 index 00000000..1de1a52b --- /dev/null +++ b/functions/tests/uploader/utils/base64.spec.js @@ -0,0 +1,17 @@ +const {expect} = require('chai'); +const base64 = require('../../../uploader/utils/base64'); + +describe('uploader', () => { + describe('base64', () => { + describe('base64Encode', () => { + it('should be defined', () => { + expect(base64.base64Encode).to.be.ok; + }); + }); + describe('base64Decode', () => { + it('should be defined', () => { + expect(base64.base64Decode).to.be.ok; + }); + }); + }); +}); diff --git a/functions/tests/uploader/utils/get-access-token.spec.js b/functions/tests/uploader/utils/get-access-token.spec.js new file mode 100644 index 00000000..9e71738e --- /dev/null +++ b/functions/tests/uploader/utils/get-access-token.spec.js @@ -0,0 +1,17 @@ +const {expect} = require('chai'); +const accessToken = require('../../../uploader/utils/get-access-token'); + +describe('uploader', () => { + describe('get-access-token', () => { + describe('getBasicHeaderRequest', () => { + it('should be defined', () => { + expect(accessToken.getBasicHeaderRequest).to.be.ok; + }); + }); + describe('getAccessToken', () => { + it('should be defined', () => { + expect(accessToken.getAccessToken).to.be.ok; + }); + }); + }); +}); diff --git a/functions/tests/uploader/utils/prepare-agent.spec.js b/functions/tests/uploader/utils/prepare-agent.spec.js new file mode 100644 index 00000000..f1bcc34c --- /dev/null +++ b/functions/tests/uploader/utils/prepare-agent.spec.js @@ -0,0 +1,17 @@ +const {expect} = require('chai'); +const prepareAgent = require('../../../uploader/utils/prepare-agent'); + +describe('uploader', () => { + describe('prepare-agent', () => { + describe('prepareAgentToPostToDF', () => { + it('should be defined', () => { + expect(prepareAgent.prepareAgentToPostToDF).to.be.ok; + }); + }); + describe('prepareAgentToFetchFromDF', () => { + it('should be defined', () => { + expect(prepareAgent.prepareAgentToFetchFromDF).to.be.ok; + }); + }); + }); +}); From 25c815e812ff87be894bad54bb4c1f60bad17c3b Mon Sep 17 00:00:00 2001 From: Bimlendra Date: Sat, 26 May 2018 22:50:28 +0530 Subject: [PATCH 6/6] added utils as a layer in test case --- functions/tests/uploader/utils/base64.spec.js | 18 ++++++++++-------- .../uploader/utils/get-access-token.spec.js | 18 ++++++++++-------- .../tests/uploader/utils/prepare-agent.spec.js | 18 ++++++++++-------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/functions/tests/uploader/utils/base64.spec.js b/functions/tests/uploader/utils/base64.spec.js index 1de1a52b..48fff6bd 100644 --- a/functions/tests/uploader/utils/base64.spec.js +++ b/functions/tests/uploader/utils/base64.spec.js @@ -2,15 +2,17 @@ const {expect} = require('chai'); const base64 = require('../../../uploader/utils/base64'); describe('uploader', () => { - describe('base64', () => { - describe('base64Encode', () => { - it('should be defined', () => { - expect(base64.base64Encode).to.be.ok; + describe('utils', () => { + describe('base64', () => { + describe('base64Encode', () => { + it('should be defined', () => { + expect(base64.base64Encode).to.be.ok; + }); }); - }); - describe('base64Decode', () => { - it('should be defined', () => { - expect(base64.base64Decode).to.be.ok; + describe('base64Decode', () => { + it('should be defined', () => { + expect(base64.base64Decode).to.be.ok; + }); }); }); }); diff --git a/functions/tests/uploader/utils/get-access-token.spec.js b/functions/tests/uploader/utils/get-access-token.spec.js index 9e71738e..91dff8e4 100644 --- a/functions/tests/uploader/utils/get-access-token.spec.js +++ b/functions/tests/uploader/utils/get-access-token.spec.js @@ -2,15 +2,17 @@ const {expect} = require('chai'); const accessToken = require('../../../uploader/utils/get-access-token'); describe('uploader', () => { - describe('get-access-token', () => { - describe('getBasicHeaderRequest', () => { - it('should be defined', () => { - expect(accessToken.getBasicHeaderRequest).to.be.ok; + describe('utils', () => { + describe('get-access-token', () => { + describe('getBasicHeaderRequest', () => { + it('should be defined', () => { + expect(accessToken.getBasicHeaderRequest).to.be.ok; + }); }); - }); - describe('getAccessToken', () => { - it('should be defined', () => { - expect(accessToken.getAccessToken).to.be.ok; + describe('getAccessToken', () => { + it('should be defined', () => { + expect(accessToken.getAccessToken).to.be.ok; + }); }); }); }); diff --git a/functions/tests/uploader/utils/prepare-agent.spec.js b/functions/tests/uploader/utils/prepare-agent.spec.js index f1bcc34c..80f0c24e 100644 --- a/functions/tests/uploader/utils/prepare-agent.spec.js +++ b/functions/tests/uploader/utils/prepare-agent.spec.js @@ -2,15 +2,17 @@ const {expect} = require('chai'); const prepareAgent = require('../../../uploader/utils/prepare-agent'); describe('uploader', () => { - describe('prepare-agent', () => { - describe('prepareAgentToPostToDF', () => { - it('should be defined', () => { - expect(prepareAgent.prepareAgentToPostToDF).to.be.ok; + describe('utils', () => { + describe('prepare-agent', () => { + describe('prepareAgentToPostToDF', () => { + it('should be defined', () => { + expect(prepareAgent.prepareAgentToPostToDF).to.be.ok; + }); }); - }); - describe('prepareAgentToFetchFromDF', () => { - it('should be defined', () => { - expect(prepareAgent.prepareAgentToFetchFromDF).to.be.ok; + describe('prepareAgentToFetchFromDF', () => { + it('should be defined', () => { + expect(prepareAgent.prepareAgentToFetchFromDF).to.be.ok; + }); }); }); });