[WIP] #148 Create instruction of how to create dev/draft copy of our Dialog Flow App and add helpful scripts

functions/package.json

@@ -36,7 +36,9 @@
     "object.entries": "^1.0.4",
     "raven": "^2.6.0",
     "replaceall": "^0.1.6",
-    "supports-color": "^5.4.0"
+    "supports-color": "^5.4.0",
+    "unzip": "^0.1.11",
+    "zipfolder": "^1.0.4"
   },
   "devDependencies": {
     "axios-mock-adapter": "^1.15.0",

functions/tests/uploader/agent/agent-downloader.spec.js

@@ -0,0 +1,12 @@
+const {expect} = require('chai');
+const agentDownloader = require('../../../uploader/agent/agent-downloader');
+
+describe('uploader', () => {
+  describe('agent-downloader', () => {
+    describe('downloadAgent', () => {
+      it('should be defined', () => {
+        expect(agentDownloader.downloadAgent).to.be.ok;
+      });
+    });
+  });
+});

functions/tests/uploader/agent/agent-uploader.spec.js

@@ -0,0 +1,12 @@
+const {expect} = require('chai');
+const agentUploader = require('../../../uploader/agent/agent-uploader');
+
+describe('uploader', () => {
+  describe('agent-uploader', () => {
+    describe('uploadAgent', () => {
+      it('should be defined', () => {
+        expect(agentUploader.uploadAgent).to.be.ok;
+      });
+    });
+  });
+});

functions/tests/uploader/utils/base64.spec.js

@@ -0,0 +1,36 @@
+const {expect} = require('chai');
+const base64 = require('../../../uploader/utils/base64');
+const fs = require('fs');
+const base64Str = 'UEsDBBQAAAAAAJBqvEwAAAAAAAAAAAAAAAANACAAc2FtcGxlX2FnZW50L1VUDQAHyLQLW+K0C1vctAtbdXgLAAEE6AMAAAToAwAAUEsBAhQDFAAAAAAAkGq8TAAAAAAAAAAAAAAAAA0AIAAAAAAAAAAAAO1BAAAAAHNhbXBsZV9hZ2VudC9VVA0AB8i0C1vitAtb3LQLW3V4CwABBOgDAAAE6AMAAFBLBQYAAAAAAQABAFsAAABLAAAAAAA=';
+
+describe('uploader', () => {
+  describe('utils', () => {
+    describe('base64', () => {
+      describe('base64Encode', () => {
+        it('should be defined', () => {
+          expect(base64.base64Encode).to.be.ok;
+        });
+
+        it('should return encoded string', () => {
+          base64.base64Encode('./tests/uploader/utils/fixtures/sample_agent.zip')
+            .then(value => {
+              expect(value).to.be.a('string');
+            });
+        });
+      });
+
+      describe('base64Decode', () => {
+        it('should be defined', () => {
+          expect(base64.base64Decode).to.be.ok;
+        });
+
+        it('should return success', () => {
+          base64.base64Decode(base64Str, './tests/uploader/utils/fixtures/sample_agent.zip')
+            .then(value => {
+              expect(value).to.be.eql('successe');
+            });
+        });
+      });
+    });
+  });
+});

functions/tests/uploader/utils/get-access-token.spec.js

@@ -0,0 +1,20 @@
+const {expect} = require('chai');
+const accessToken = require('../../../uploader/utils/get-access-token');
+
+describe('uploader', () => {
+  describe('utils', () => {
+    describe('get-access-token', () => {
+      describe('getBasicHeaderRequest', () => {
+        it('should be defined', () => {
+          expect(accessToken.getBasicHeaderRequest).to.be.ok;
+        });
+      });
+      describe('getAccessToken', () => {
+        it('should be defined', () => {
+          expect(accessToken.getAccessToken).to.be.ok;
+        });
+
+      });
+    });
+  });
+});

functions/tests/uploader/utils/prepare-agent.spec.js

@@ -0,0 +1,19 @@
+const {expect} = require('chai');
+const prepareAgent = require('../../../uploader/utils/prepare-agent');
+
+describe('uploader', () => {
+  describe('utils', () => {
+    describe('prepare-agent', () => {
+      describe('prepareAgentToPostToDF', () => {
+        it('should be defined', () => {
+          expect(prepareAgent.prepareAgentToPostToDF).to.be.ok;
+        });
+      });
+      describe('prepareAgentToFetchFromDF', () => {
+        it('should be defined', () => {
+          expect(prepareAgent.prepareAgentToFetchFromDF).to.be.ok;
+        });
+      });
+    });
+  });
+});

functions/uploader/agent/agent-downloader.js

@@ -0,0 +1,35 @@
+const axios = require('axios');
+const debug = require(`debug`)(`ia:uploader:agent:agent-downloader:debug`);
+const error = require(`debug`)(`ia:uploader:agent:agent-downloader:error`);
+const {getAccessToken} = require('../utils/get-access-token');
+const {prepareAgentToFetchFromDF} = require('../utils/prepare-agent');
+
+const config = require('../config');
+const mustache = require('mustache');
+
+function downloadAgent () {
+  return getAccessToken()
+    .then(accessToken => {
+      var agentId = config.agent.ID;
+      return axios(mustache.render(
+        config.dfendpoints.AGENT_EXPORT_URL,
+        {
+          agentId,
+          accessToken,
+        }
+      ), {method: `POST`});
+    })
+    .then(res => res.data)
+    .then(data => {
+      debug(data);
+      return prepareAgentToFetchFromDF(data.response.agentContent, '../../agent/', 'agent.zip');
+    })
+    .catch(e => {
+      error(`Get error in downloading agent from DF, error: ${e}`);
+      return Promise.reject(e);
+    });
+}
+
+module.exports = {
+  downloadAgent,
+};

functions/uploader/agent/agent-uploader.js

@@ -0,0 +1,36 @@
+const axios = require('axios');
+const debug = require(`debug`)(`ia:uploader:agent:agent-uploader:debug`);
+const error = require(`debug`)(`ia:uploader:agent:agent-uploader:error`);
+const util = require(`util`);
+const {getAccessToken} = require('../utils/get-access-token');
+const {prepareAgentToPostToDF} = require('../utils/prepare-agent');
+
+const config = require('../config');
+const mustache = require('mustache');
+
+function uploadAgent () {
+  return Promise.all([getAccessToken(), prepareAgentToPostToDF('../../agent', './')])
+    .then(values => {
+      const [accessToken, base64] = values;
+      var agentId = config.agent.ID;
+      return axios(mustache.render(
+        config.dfendpoints.AGENT_IMPORT_URL,
+        {
+          agentId,
+          accessToken,
+        }
+      ), {headers: {'content-type': `application/json; charset=UTF-8`}, method: `POST`, data: JSON.stringify({'agentContent': base64})});
+    })
+    .then(res => res.data)
+    .then(data => {
+      debug(util.inspect(data, false, null));
+    })
+    .catch(e => {
+      error(`Get error in uploading agent from DF, error: ${e}`);
+      return Promise.reject(e);
+    });
+}
+
+module.exports = {
+  uploadAgent,
+};

functions/uploader/config.js

@@ -1,19 +1,24 @@
 module.exports = {
   dfendpoints: {
     DF_ENTITY_POST_URL: 'https://dialogflow.googleapis.com/v2/projects/internet-archive/agent/entityTypes/{{entityid}}/entities:batchUpdate?access_token={{accesstoken}}',
-    uploader: {
-      collection: {
-        ID: 'etree',
-        FIELD: 'creator',
-        LIMIT: 30000,
-        ENTITY_ID: 'ENTITY_ID will goes here',
-      },
-      genres: {
-        ID: 'georgeblood',
-        FIELD: 'subject',
-        LIMIT: 30000,
-        ENTITY_ID: 'ENTITY_ID will goes here',
-      },
+    AGENT_EXPORT_URL: 'https://dialogflow.googleapis.com/v2/projects/{{agentId}}/agent:export?access_token={{accessToken}}',
+    AGENT_IMPORT_URL: 'https://dialogflow.googleapis.com/v2/projects/{{agentId}}/agent:import?access_token={{accessToken}}',
+  },
+  agent: {
+    ID: 'agent_id_will_goes_here'
+  },
+  uploader: {
+    collection: {
+      ID: 'etree',
+      FIELD: 'creator',
+      LIMIT: 30000,
+      ENTITY_ID: 'ENTITY_ID will goes here',
+    },
+    genres: {
+      ID: 'georgeblood',
+      FIELD: 'subject',
+      LIMIT: 30000,
+      ENTITY_ID: 'ENTITY_ID will goes here',
     },
   },
 };

functions/uploader/index.js

@@ -9,14 +9,23 @@ const error = require(`debug`)(`ia:uploader:index:error`);
 
 const {uploadCollection} = require('./entities/collection');
 const {uploadGenres} = require('./entities/genres');
+const {downloadAgent} = require('./agent/agent-downloader');
+const {uploadAgent} = require('./agent/agent-uploader');
 
 const ALL = `all`;
 
 const entities = {
   collection: uploadCollection,
   genres: uploadGenres,
 };
+
+const agent = {
+  upload: uploadAgent,
+  download: downloadAgent,
+};
+
 const all = {
+  agent: agent,
   entities: entities,
 };
 

functions/uploader/utils/base64.js

@@ -0,0 +1,41 @@
+const error = require(`debug`)(`ia:uploader:utils:base64:error`);
+const fs = require('fs');
+
+// function to encode file data to base64 encoded string
+function base64Encode (file) {
+// read binary data
+  return new Promise(function (resolve, reject) {
+    fs.readFile(file, function (err, bitmap) {
+      console.log(Buffer.from(bitmap).toString('base64'));
+      if (err) return reject(err);
+      else return resolve(Buffer.from(bitmap).toString('base64'));
+    });
+  })
+    .catch(function (err) {
+      console.log(err);
+    });
+}
+
+// function to create file from base64 encoded string
+function base64Decode (base64str, file) {
+// create buffer object from base64 encoded string, it is important to tell the constructor that the string is base64 encoded
+  var bitmap = Buffer.from(base64str, 'base64');
+  // write buffer to file
+  return new Promise(function (resolve, reject) {
+    fs.writeFile(file, bitmap, 'utf8', function (err) {
+      console.log(file);
+      console.log(bitmap);
+      console.log(err);
+      if (err) return reject(err);
+      else return resolve('success');
+    });
+  })
+    .catch(function (err) {
+      console.log(err);
+    });
+}
+
+module.exports = {
+  base64Encode,
+  base64Decode,
+};

functions/uploader/utils/get-access-token.js

@@ -0,0 +1,44 @@
+const debug = require(`debug`)(`ia:uploader:utils:get-access-token:debug`);
+const error = require(`debug`)(`ia:uploader:utils:get-access-token:error`);
+const util = require(`util`);
+const { exec } = require('child-process-promise');
+
+function getBasicHeaderRequest () {
+  return getAccessToken()
+    .then(accessToken => {
+      const basicHeaderRequest = {
+        'content-type': `application/json; charset=UTF-8`,
+        'authorization': `BEARER ${accessToken}`
+      };
+      return Promise.resolve(basicHeaderRequest);
+    })
+    .catch(e => {
+      error(`Get error in fetching basicHeaderRequest, error: ${util.inspect(e, false, null)}`);
+      return Promise.reject(e);
+    });
+}
+
+function getAccessToken () {
+  debug('getting access token');
+  return exec('gcloud auth print-access-token')
+    .then(values => {
+      var stdout = values.stdout;
+      var stderr = values.stderr;
+      if (stdout) {
+        var filteredstdout = stdout.replace(/\n$/, '');
+        debug(util.inspect(filteredstdout, false, null));
+        return Promise.resolve(filteredstdout);
+      } else if (stderr) {
+        error(stderr);
+        return Promise.reject(new Error('ERROR: ' + stderr));
+      } else {
+        error('Having trouble with GCloud execution');
+        return Promise.reject(new Error('ERROR: Having trouble with GCloud execution'));
+      }
+    });
+}
+
+module.exports = {
+  getBasicHeaderRequest,
+  getAccessToken,
+};

functions/uploader/utils/prepare-agent.js

@@ -0,0 +1,43 @@
+const error = require(`debug`)(`ia:uploader:agent:utils:prepare_agent:error`);
+const debug = require(`debug`)(`ia:uploader:agent:utils:prepare_agent:debug`);
+const fs = require('fs');
+const unzip = require(`unzip`);
+const zip = require('zipfolder');
+
+const {base64Encode, base64Decode} = require('./base64');
+
+// function to encode file data to base64 encoded string
+function prepareAgentToPostToDF (folderPath, zipPath) {
+  return zip.zipFolder({folderPath: folderPath, targetFolderPath: zipPath})
+    .then(function (path) {
+      var encodedBase64 = base64Encode(path);
+      debug(encodedBase64);
+      return Promise.resolve(encodedBase64);
+    });
+}
+
+// function to create file from base64 encoded string
+function prepareAgentToFetchFromDF (base64str, folderPath, zipPath) {
+  return base64Decode(base64str, zipPath)
+    .then(isDone => {
+      return fs.createReadStream(zipPath).pipe(unzip.Extract({ path: folderPath }));
+    })
+    .then(close => {
+      debug(close);
+      return new Promise(function (resolve, reject) {
+        fs.unlink(zipPath, function (err) {
+          if (err) return reject(err);
+          else return resolve(zipPath);
+        });
+      });
+    })
+    .catch(e => {
+      error(`Get error in generating agent.zip, error: ${e}`);
+      return Promise.reject(e);
+    });
+}
+
+module.exports = {
+  prepareAgentToPostToDF,
+  prepareAgentToFetchFromDF,
+};