test(cli): add check for intermediates matching aggregates in aws tests (#3252)

bernardobridge · web-flow · commit 56dbfe8f982b · 2024-07-03T15:00:57.000+01:00
diff --git a/packages/artillery/test/cloud-e2e/fargate/bom.test.js b/packages/artillery/test/cloud-e2e/fargate/bom.test.js
@@ -2,7 +2,10 @@ const { test, before, beforeEach } = require('tap');
 const { $ } = require('zx');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -59,4 +62,5 @@ test('Run mixed-hierarchy', async (t) => {
   );
 
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/fargate/cw-adot.test.js b/packages/artillery/test/cloud-e2e/fargate/cw-adot.test.js
@@ -9,7 +9,10 @@ const {
   getTestTags
 } = require('../../helpers');
 const { getTestId, getXRayTraces } = require('./fixtures/adot/helpers.js');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 // NOTE: This test reports to Artillery Dashboard to dogfood and improve visibility
@@ -46,6 +49,7 @@ test('traces succesfully arrive to cloudwatch', async (t) => {
   const testId = getTestId(output.stdout);
   const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 
   let traceMap;
   try {
diff --git a/packages/artillery/test/cloud-e2e/fargate/dd-adot.test.js b/packages/artillery/test/cloud-e2e/fargate/dd-adot.test.js
@@ -8,7 +8,10 @@ const {
   deleteFile,
   getTestTags
 } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 const { getDatadogSpans, getTestId } = require('./fixtures/adot/helpers.js');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
@@ -53,6 +56,7 @@ test('traces succesfully arrive to datadog', async (t) => {
   const testId = getTestId(output.stdout);
   const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 
   let spanList;
   try {
diff --git a/packages/artillery/test/cloud-e2e/fargate/ensure-plugin.test.js b/packages/artillery/test/cloud-e2e/fargate/ensure-plugin.test.js
@@ -3,7 +3,10 @@ const { $ } = require('zx');
 const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -40,6 +43,8 @@ test('Run uses ensure', async (t) => {
       300,
       'Should have 300 "200 OK" responses'
     );
+
     checkForNegativeValues(t, report);
+    checkAggregateCounterSums(t, report);
   }
 });
diff --git a/packages/artillery/test/cloud-e2e/fargate/expect-plugin.test.js b/packages/artillery/test/cloud-e2e/fargate/expect-plugin.test.js
@@ -3,7 +3,10 @@ const { $ } = require('zx');
 const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -41,7 +44,9 @@ test('CLI should exit with non-zero exit code when there are failed expectations
       10,
       'Should have 10 "200 OK" responses'
     );
+
     checkForNegativeValues(t, report);
+    checkAggregateCounterSums(t, report);
   }
 });
 
diff --git a/packages/artillery/test/cloud-e2e/fargate/misc.test.js b/packages/artillery/test/cloud-e2e/fargate/misc.test.js
@@ -3,7 +3,10 @@ const { $ } = require('zx');
 const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -78,7 +81,9 @@ test('Kitchen Sink Test - multiple features together', async (t) => {
     40,
     'Should have 40 /pony "200 OK" responses'
   );
+
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 });
 
 test('Run lots-of-output', async (t) => {
diff --git a/packages/artillery/test/cloud-e2e/fargate/processors.test.js b/packages/artillery/test/cloud-e2e/fargate/processors.test.js
@@ -3,7 +3,10 @@ const { $ } = require('zx');
 const fs = require('fs');
 const path = require('path');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -37,7 +40,9 @@ test('Run with typescript processor and external package', async (t) => {
     2,
     'Should have emitted 2 errors'
   );
+
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 });
 
 test('Run a test with an ESM processor', async (t) => {
@@ -63,5 +68,7 @@ test('Run a test with an ESM processor', async (t) => {
     10,
     'Should have emitted 10 custom metrics from ts processor'
   );
+
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-bom.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-bom.test.js
@@ -6,7 +6,10 @@ const {
   generateTmpReportPath,
   getImageArchitecture
 } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 
@@ -61,5 +64,7 @@ tap.test('Run mixed-hierarchy test in Lambda Container', async (t) => {
     20,
     'Should have 20 "200 OK" responses'
   );
+
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-dotenv.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-dotenv.test.js
@@ -6,7 +6,10 @@ const {
   generateTmpReportPath,
   getImageArchitecture
 } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 const A9_PATH = process.env.A9_PATH || 'artillery';
@@ -44,5 +47,7 @@ tap.test('Run dotenv test in Lambda Container', async (t) => {
     50,
     'Should have custom counter for env variable fruit'
   );
+
   checkForNegativeValues(t, report);
+  checkAggregateCounterSums(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-ensure.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-ensure.test.js
@@ -7,7 +7,10 @@ const {
   getTestTags,
   getImageArchitecture
 } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 //NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
 const tags = getTestTags(['type:acceptance']);
@@ -46,6 +49,8 @@ tap.test('Lambda Container run uses ensure', async (t) => {
       300,
       'Should have 300 "200 OK" responses'
     );
+
     checkForNegativeValues(t, report);
+    checkAggregateCounterSums(t, report);
   }
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-expect.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-expect.test.js
@@ -7,7 +7,10 @@ const {
   getTestTags,
   getImageArchitecture
 } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 let reportFilePath;
@@ -54,7 +57,9 @@ tap.test(
         10,
         'Should have 10 "200 OK" responses'
       );
+
       checkForNegativeValues(t, report);
+      checkAggregateCounterSums(t, report);
     }
   }
 );
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-smoke.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-smoke.test.js
@@ -6,7 +6,10 @@ const {
   generateTmpReportPath,
   getImageArchitecture
 } = require('../../helpers');
-const { checkForNegativeValues } = require('../../helpers/expectations');
+const {
+  checkForNegativeValues,
+  checkAggregateCounterSums
+} = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 
@@ -74,6 +77,8 @@ tap.test(
       2,
       'Should have emitted 2 errors'
     );
+
     checkForNegativeValues(t, report);
+    checkAggregateCounterSums(t, report);
   }
 );
diff --git a/packages/artillery/test/helpers/expectations.js b/packages/artillery/test/helpers/expectations.js
@@ -38,6 +38,35 @@ const checkForNegativeValues = (t, report) => {
   }
 };
 
+const checkAggregateCounterSums = (t, report) => {
+  const aggregateCounters = report.aggregate?.counters;
+
+  if (!aggregateCounters || Object.keys(aggregateCounters).length === 0) {
+    t.fail('No aggregate counters found in the report');
+  }
+
+  let intermediateCounters = {};
+
+  for (const intermediate of report.intermediate) {
+    for (const key in intermediate.counters) {
+      if (intermediateCounters[key]) {
+        intermediateCounters[key] += intermediate.counters[key];
+      } else {
+        intermediateCounters[key] = intermediate.counters[key];
+      }
+    }
+  }
+
+  for (const key in aggregateCounters) {
+    if (aggregateCounters[key] !== intermediateCounters[key]) {
+      t.fail(
+        `Aggregate counter sum mismatch for ${key}. Aggregate ${aggregateCounters[key]} != Intermediate ${intermediateCounters[key]}`
+      );
+    }
+  }
+};
+
 module.exports = {
-  checkForNegativeValues
+  checkForNegativeValues,
+  checkAggregateCounterSums
 };
