Add log files. Update tests. Try to better handle session connection issues. Simplify how ant works.

Author: rutlabaga

bin/run-grunt

@@ -133,6 +133,10 @@
 		<runDataPackJob packCommand="packDeploy" job="${vlocity.dataPackJob}" />
 	</target>
 
+    <target name="runTestJob" description="">
+        <runDataPackJob packCommand="runTestJob" job="TestJob" />
+    </target>
+
 	<macrodef name="runDataPackJob" >
 		<attribute name="packCommand" />
 		<attribute name="job" />
@@ -142,7 +146,11 @@
 					<echo> ------------------------------------------- </echo>
 					<echo> ------------------------------------------- </echo>
 					<echo> Running - @{packCommand} - @{job}</echo>
-						<exec executable="${basedir}/bin/run-grunt" dir="${basedir}">
+                        <exec executable="npm" dir="${basedir}">
+                            <arg value="install" />
+                        </exec>
+						<exec executable="node" dir="${basedir}">
+                            <arg value="./node_modules/grunt-cli/bin/grunt" />
 							<arg value="@{packCommand}"/>
 							<arg value="-job"/>
 							<arg value="@{job}"/>

build.xml

@@ -3,14 +3,16 @@ expansionPath: testJobRunning
 buildFile: testJobRunning/testBuildFile.resource
 queries: 
   - VlocityDataPackType: VlocityUITemplate
-    query: Select Id from %vlocity_namespace%__VlocityUITemplate__c where Name LIKE 'testpack-%' AND %vlocity_namespace%__Active__c = true
+    query: Select Id from %vlocity_namespace%__VlocityUITemplate__c where Name LIKE 'datapacktest-%' AND %vlocity_namespace%__Active__c = true
+  - VlocityDataPackType: VlocityUILayout
+    query: Select Id from %vlocity_namespace%__VlocityUILayout__c where Name LIKE 'datapacktest-%'
 delete: true 
 activate: true 
 compileOnBuild: true
 preJobApex:
   Deploy: TestJob.cls
 preStepApex:
-  Deploy: TestJob.cls
+  Deploy: DeactivateTemplatesAndLayouts.cls
 postStepApex:
   Deploy: TestJob.cls
 postJobApex:

dataPacksJobs/TestJob.yaml

@@ -286,12 +286,14 @@ module.exports = function (grunt) {
         fs.removeSync('./test/testJobRunning');
         fs.copySync('./test/testJobData', './test/testJobRunning');
 
-        var commands = ['Deploy', 
-                        'GetDiffs',
-                        'Export', 
-                        'GetDiffsAndDeploy', 
-                        'BuildFile',
-                        'JavaScript' ];
+        var commands = [
+                            'Deploy', 
+                            'GetDiffs',
+                            'Export', 
+                            'GetDiffsAndDeploy', 
+                            'BuildFile',
+                            'JavaScript' 
+                        ];
 
         if (grunt.option('test')) {
             commands = [ grunt.option('test') ];

grunt/dataPacksJobTask.js

@@ -176,15 +176,32 @@ DataPacks.prototype.runDataPackProcess = function(dataPackData, options, onSucce
     self.vlocity.checkLogin(function() {
         self.vlocity.jsForceConnection.apex.post(self.dataPacksEndpoint, dataPackData, function(err, result) {
 
-            if (typeof result == "string") {
-                result = JSON.parse(result);
-            }
+            if (err) { 
+                if (dataPackData.isRetry) {
+                    console.error('\x1b[31m', 'ERROR >>' ,'\x1b[0m', dataPackId, err);
+                    
+                    if (onError) onError(err);
+                    else if (onSuccess) onSuccess(err);
+                    else throw err;
+                } else {
+                     console.error('\x1b[31m', 'RETRYING FOR ERROR >>' ,'\x1b[0m', dataPackId, err);
+
+                    self.vlocity.isLoggedIn = false;
+                    dataPackData.isRetry = true;
+
+                    setTimeout(function() { self.runDataPackProcess(dataPackData, options, onSuccess, onError); }, 1000);
+                }
+            } else {
+                dataPackData.isRetry = false;
 
-            if (self.vlocity.verbose) { 
-                console.log('Result', result);
-            }
+                if (typeof result == "string") {
+                   result = JSON.parse(result);
+                }
+
+                if (self.vlocity.verbose) { 
+                    console.log('Result', result);
+                }
 
-            if (result) {
                 if (result.Total > 0) {
                     if (dataPackData.processType == "Export" 
                         && dataPackData.processData
@@ -205,29 +222,26 @@ DataPacks.prototype.runDataPackProcess = function(dataPackData, options, onSucce
                         console.log('\x1b[32m', 'Activated >>', '\x1b[0m', activatedEntity.VlocityDataPackKey);
                     });
                 }
-            }
-            
-            if (err) { 
-                console.error('\x1b[31m', 'ERROR >>' ,'\x1b[0m', dataPackId, err);
-
-                // Need to hook this in an interesting way
-                var packError = { VlocityDataPackId: dataPackId, AllErrorMessage: err };
-                
-                if (onError) onError(err);
-                else if (onSuccess) onSuccess(err);
-                else throw err;
-            } else if (/(Ready|InProgress)/.test(result.Status)) {
-                dataPackData.processData = result;
-                
-                setTimeout(function() { self.runDataPackProcess(dataPackData, options, onSuccess, onError); }, result.Async ? 3000 : 1);
-            } else if (/(Complete|Deleted)/.test(result.Status)) {
-
-                if (onSuccess) onSuccess(result);
-                else console.log(result);
-            } else if (/Error/.test(result.Status)) {
-                if (onError) onError(result);
-                else if (onSuccess) onSuccess(result);
-                else console.log(result);
+
+                if (result.activationError) {
+                    result.activationError.forEach(function(activatedEntity) {
+                        console.log('\x1b[31m', 'Activation Error >>', '\x1b[0m', activatedEntity.VlocityDataPackKey, '---', activatedEntity.ActivationMessage);
+                    });
+                }
+                 
+                if (/(Ready|InProgress)/.test(result.Status)) {
+                    dataPackData.processData = result;
+                    
+                    setTimeout(function() { self.runDataPackProcess(dataPackData, options, onSuccess, onError); }, result.Async ? 3000 : 1);
+                } else if (/(Complete|Deleted)/.test(result.Status)) {
+
+                    if (onSuccess) onSuccess(result);
+                    else console.log(result);
+                } else if (/Error/.test(result.Status)) {
+                    if (onError) onError(result);
+                    else if (onSuccess) onSuccess(result);
+                    else console.log(result);
+                }
             }
         });
     });

node_vlocity/datapacks.js

@@ -322,6 +322,10 @@ DataPacksBuilder.prototype.initializeImportStatus = function(importPath, manifes
                                 jobInfo.currentStatus[generatedDataPackKey] = 'Ready';
                             }
 
+                            if (sobjectData.Name) {
+                                jobInfo.generatedKeysToNames[generatedDataPackKey] = sobjectData.Name;
+                            }
+
                             if (jobInfo.allParents.indexOf(generatedDataPackKey) == -1) {
                                 jobInfo.allParents.push(generatedDataPackKey);
                             }

node_vlocity/datapacksbuilder.js

@@ -468,6 +468,10 @@ DataPacksExpand.prototype.preprocessDataPack = function(currentData, jobInfo) {
 
                 jobInfo.vlocityKeysToNewNamesMap[currentData.VlocityDataPackKey] = generatedKey;
 
+                if (currentData.Name) {
+                    jobInfo.generatedKeysToNames[generatedKey] = currentData.Name;
+                }
+
                 // make sure we don't overwrite keys later
                 jobInfo.vlocityKeysToNewNamesMap[generatedKey] = generatedKey;
 

node_vlocity/datapacksexpand.js

@@ -69,10 +69,6 @@ DataPacksJob.prototype.runJob = function(jobData, jobName, action, onSuccess, on
 DataPacksJob.prototype.runJobWithInfo = function(jobInfo, action, onSuccess, onError) {
     var self = this;
 
-    if (!jobInfo.startTime) {
-        jobInfo.startTime = Date.now();
-    }
-
     // Will not continue a single DataPack, but will continue when there are breaks in the job
     if (action == 'Continue' || action == 'Retry') {
         Object.assign(jobInfo, JSON.parse(fs.readFileSync(CURRENT_INFO_FILE, 'utf8')));
@@ -134,6 +130,14 @@ DataPacksJob.prototype.runJobWithInfo = function(jobInfo, action, onSuccess, onE
         action = jobInfo.jobAction;
     }
 
+    if (jobInfo.startTime == null) {
+        jobInfo.startTime = Date.now();
+    }
+
+    if (!jobInfo.logName) {
+        jobInfo.logName = self.vlocity.datapacksexpand.generateFolderOrFilename(jobInfo.jobName + '-' + new Date(Date.now()).toISOString() + '-' + jobInfo.jobAction, 'yaml');
+    }
+
     // Initialize all that need it
     if (jobInfo.extendedManifest == null) {
         jobInfo.extendedManifest = {};
@@ -167,9 +171,6 @@ DataPacksJob.prototype.runJobWithInfo = function(jobInfo, action, onSuccess, onE
         jobInfo.vlocityRecordSourceKeyMap = {};
     }
 
-    if (jobInfo.startTime == null) {
-        jobInfo.startTime = Date.now();
-    }
 
     if (!jobInfo.addedToExportBuildFile) {
         jobInfo.addedToExportBuildFile = [];
@@ -184,6 +185,10 @@ DataPacksJob.prototype.runJobWithInfo = function(jobInfo, action, onSuccess, onE
         jobInfo.VlocityDataPackIds = {};
     }
 
+    if (!jobInfo.generatedKeysToNames) {
+        jobInfo.generatedKeysToNames = {};
+    }
+
     jobInfo.supportParallel = jobInfo.defaultMaxParallel > 1;
 
     var toolingApi = self.vlocity.jsForceConnection.tooling;
@@ -296,6 +301,9 @@ DataPacksJob.prototype.buildManifestFromQueries = function(jobInfo, onComplete)
         var totalFound = 0;
 
         async.eachSeries(jobInfo.queries, function(queryData, callback) {
+            if (!queryData || !queryData.VlocityDataPackType) {
+                return callback();
+            }
 
             if (!jobInfo.manifest[queryData.VlocityDataPackType]) {
                 jobInfo.manifest[queryData.VlocityDataPackType] = [];
@@ -752,7 +760,7 @@ DataPacksJob.prototype.activateAll = function(dataPackData, jobInfo, onComplete,
 
                 dataPackData.dataPacks.forEach(function(dataPack) {
 
-                   if (dataPack.ActivationStatus == 'Ready' && dataPack.VlocityDataPackStatus == 'Success') {
+                    if (dataPack.ActivationStatus == 'Ready' && dataPack.VlocityDataPackStatus == 'Success') {
 
                         // If it is the only one in the deploy and it fails to activate it must be set to error. Otherwise retry the deploy and activation separate from others.
                         if (dataPackData.dataPacks.length == 1) {
@@ -767,6 +775,16 @@ DataPacksJob.prototype.activateAll = function(dataPackData, jobInfo, onComplete,
                         } else {
                             jobInfo.currentStatus[dataPack.VlocityDataPackKey] = 'ReadySeparate';
                         }
+                    } else if (dataPack.ActivationStatus == 'Error') {
+
+                        var message = 'Activation Error >> ' +  dataPack.VlocityDataPackKey + ' --- ' + dataPack.ActivationMessage;
+
+                        jobInfo.hasError = true;
+                        jobInfo.currentStatus[dataPack.VlocityDataPackKey] = 'Error';
+
+                        if (jobInfo.errors.indexOf(message) == -1) {
+                            jobInfo.errors.push(message);
+                        }
                     }
                 });
 

node_vlocity/datapacksjob.js

@@ -2,6 +2,7 @@ var fs = require("fs-extra");
 var path  = require('path');
 var stringify = require('json-stable-stringify');
 var async = require('async');
+var yaml = require('js-yaml');
 
 // use consts for setting the namespace prefix so that we easily can reference it later on in this file
 const namespacePrefix = 'vlocity_namespace';
@@ -647,17 +648,49 @@ DataPacksUtils.prototype.printJobStatus = function(jobInfo) {
         return;
     }
 
-    Object.keys(jobInfo.currentStatus).forEach(function(dataPackKey) {
+    var keysByStatus = {};
+    
+    Object.keys(jobInfo.currentStatus).forEach(function(dataPackKey) { 
+
+        var status = jobInfo.currentStatus[dataPackKey];
         if (jobInfo.currentStatus[dataPackKey] == 'Ready' 
             || jobInfo.currentStatus[dataPackKey] == 'Header' 
             || jobInfo.currentStatus[dataPackKey] == 'Added'
             || jobInfo.currentStatus[dataPackKey] == 'ReadySeparate') {
+            status = 'Remaining';
                 totalRemaining++;
         } else if (jobInfo.currentStatus[dataPackKey] == 'Error') {
             errorsCount++;
         } else if (jobInfo.currentStatus[dataPackKey] == 'Success') {
             successfulCount++;
         }
+
+        if (!keysByStatus[status]) {
+            keysByStatus[status] = {};
+        }
+
+        // For Exports
+        var keyForStatus = jobInfo.vlocityKeysToNewNamesMap[dataPackKey] ? jobInfo.vlocityKeysToNewNamesMap[dataPackKey] : dataPackKey;
+
+        if (keyForStatus.indexOf('/') != -1) {
+
+            var slashIndex = keyForStatus.indexOf('/');
+            var beforeSlash = keyForStatus.substring(0, slashIndex);
+            var afterSlash = keyForStatus.substring(slashIndex+1);
+
+            if (!keysByStatus[status][beforeSlash]) {
+                keysByStatus[status][beforeSlash] = [];
+            }
+            var dataPackName = jobInfo.generatedKeysToNames[keyForStatus];
+
+            if (dataPackName && afterSlash.indexOf(dataPackName) == -1) {
+                afterSlash = dataPackName + ' - ' + afterSlash;
+            }
+
+            if (keysByStatus[status][beforeSlash].indexOf(afterSlash) == -1) {
+                keysByStatus[status][beforeSlash].push(afterSlash);
+            }
+        }
     });
 
     if (jobInfo.extendedManifest) {
@@ -693,4 +726,25 @@ DataPacksUtils.prototype.printJobStatus = function(jobInfo) {
     if (jobInfo.hasError) {
         jobInfo.errorMessage = jobInfo.errors.join('\n');
     }
+
+    Object.keys(keysByStatus).forEach(function(statusKey) {
+        Object.keys(keysByStatus[statusKey]).forEach(function(typeKey) {
+            keysByStatus[statusKey][typeKey].sort();
+        });
+    });
+
+    var logInfo = {
+        Job: jobInfo.jobName,
+        Action: jobInfo.jobAction,
+        ProjectPath: jobInfo.projectPath,
+        TotalTime: Math.floor((elapsedTime / 60)) + 'm ' + Math.floor((elapsedTime % 60)) + 's',
+        Errors: jobInfo.errors,
+        Status: keysByStatus
+    };
+
+    try {
+        fs.outputFileSync(path.join('logs', jobInfo.logName), yaml.dump(logInfo, { lineWidth: 1000 }));
+    } catch (e) {
+        console.log(e);
+    }
 };

node_vlocity/datapacksutils.js

@@ -0,0 +1,3 @@
+{
+    "VlocityCard/datapacktest-card": "Reference by Name"
+}