Skip to content

Commit 56dbfe8

Browse files
test(cli): add check for intermediates matching aggregates in aws tests (#3252)
1 parent 96b6074 commit 56dbfe8

13 files changed

+101
-13
lines changed

packages/artillery/test/cloud-e2e/fargate/bom.test.js

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ const { test, before, beforeEach } = require('tap');
22
const { $ } = require('zx');
33
const fs = require('fs');
44
const { generateTmpReportPath, getTestTags } = require('../../helpers');
5-
const { checkForNegativeValues } = require('../../helpers/expectations');
5+
const {
6+
checkForNegativeValues,
7+
checkAggregateCounterSums
8+
} = require('../../helpers/expectations');
69

710
const A9_PATH = process.env.A9_PATH || 'artillery';
811

@@ -59,4 +62,5 @@ test('Run mixed-hierarchy', async (t) => {
5962
);
6063

6164
checkForNegativeValues(t, report);
65+
checkAggregateCounterSums(t, report);
6266
});

packages/artillery/test/cloud-e2e/fargate/cw-adot.test.js

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@ const {
99
getTestTags
1010
} = require('../../helpers');
1111
const { getTestId, getXRayTraces } = require('./fixtures/adot/helpers.js');
12-
const { checkForNegativeValues } = require('../../helpers/expectations');
12+
const {
13+
checkForNegativeValues,
14+
checkAggregateCounterSums
15+
} = require('../../helpers/expectations');
1316

1417
const A9_PATH = process.env.A9_PATH || 'artillery';
1518
// NOTE: This test reports to Artillery Dashboard to dogfood and improve visibility
@@ -46,6 +49,7 @@ test('traces succesfully arrive to cloudwatch', async (t) => {
4649
const testId = getTestId(output.stdout);
4750
const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));
4851
checkForNegativeValues(t, report);
52+
checkAggregateCounterSums(t, report);
4953

5054
let traceMap;
5155
try {

packages/artillery/test/cloud-e2e/fargate/dd-adot.test.js

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ const {
88
deleteFile,
99
getTestTags
1010
} = require('../../helpers');
11-
const { checkForNegativeValues } = require('../../helpers/expectations');
11+
const {
12+
checkForNegativeValues,
13+
checkAggregateCounterSums
14+
} = require('../../helpers/expectations');
1215
const { getDatadogSpans, getTestId } = require('./fixtures/adot/helpers.js');
1316

1417
const A9_PATH = process.env.A9_PATH || 'artillery';
@@ -53,6 +56,7 @@ test('traces succesfully arrive to datadog', async (t) => {
5356
const testId = getTestId(output.stdout);
5457
const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));
5558
checkForNegativeValues(t, report);
59+
checkAggregateCounterSums(t, report);
5660

5761
let spanList;
5862
try {

packages/artillery/test/cloud-e2e/fargate/ensure-plugin.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@ const { $ } = require('zx');
33
const chalk = require('chalk');
44
const fs = require('fs');
55
const { generateTmpReportPath, getTestTags } = require('../../helpers');
6-
const { checkForNegativeValues } = require('../../helpers/expectations');
6+
const {
7+
checkForNegativeValues,
8+
checkAggregateCounterSums
9+
} = require('../../helpers/expectations');
710

811
const A9_PATH = process.env.A9_PATH || 'artillery';
912

@@ -40,6 +43,8 @@ test('Run uses ensure', async (t) => {
4043
300,
4144
'Should have 300 "200 OK" responses'
4245
);
46+
4347
checkForNegativeValues(t, report);
48+
checkAggregateCounterSums(t, report);
4449
}
4550
});

packages/artillery/test/cloud-e2e/fargate/expect-plugin.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@ const { $ } = require('zx');
33
const chalk = require('chalk');
44
const fs = require('fs');
55
const { generateTmpReportPath, getTestTags } = require('../../helpers');
6-
const { checkForNegativeValues } = require('../../helpers/expectations');
6+
const {
7+
checkForNegativeValues,
8+
checkAggregateCounterSums
9+
} = require('../../helpers/expectations');
710

811
const A9_PATH = process.env.A9_PATH || 'artillery';
912

@@ -41,7 +44,9 @@ test('CLI should exit with non-zero exit code when there are failed expectations
4144
10,
4245
'Should have 10 "200 OK" responses'
4346
);
47+
4448
checkForNegativeValues(t, report);
49+
checkAggregateCounterSums(t, report);
4550
}
4651
});
4752

packages/artillery/test/cloud-e2e/fargate/misc.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@ const { $ } = require('zx');
33
const chalk = require('chalk');
44
const fs = require('fs');
55
const { generateTmpReportPath, getTestTags } = require('../../helpers');
6-
const { checkForNegativeValues } = require('../../helpers/expectations');
6+
const {
7+
checkForNegativeValues,
8+
checkAggregateCounterSums
9+
} = require('../../helpers/expectations');
710

811
const A9_PATH = process.env.A9_PATH || 'artillery';
912

@@ -78,7 +81,9 @@ test('Kitchen Sink Test - multiple features together', async (t) => {
7881
40,
7982
'Should have 40 /pony "200 OK" responses'
8083
);
84+
8185
checkForNegativeValues(t, report);
86+
checkAggregateCounterSums(t, report);
8287
});
8388

8489
test('Run lots-of-output', async (t) => {

packages/artillery/test/cloud-e2e/fargate/processors.test.js

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@ const { $ } = require('zx');
33
const fs = require('fs');
44
const path = require('path');
55
const { generateTmpReportPath, getTestTags } = require('../../helpers');
6-
const { checkForNegativeValues } = require('../../helpers/expectations');
6+
const {
7+
checkForNegativeValues,
8+
checkAggregateCounterSums
9+
} = require('../../helpers/expectations');
710

811
const A9_PATH = process.env.A9_PATH || 'artillery';
912

@@ -37,7 +40,9 @@ test('Run with typescript processor and external package', async (t) => {
3740
2,
3841
'Should have emitted 2 errors'
3942
);
43+
4044
checkForNegativeValues(t, report);
45+
checkAggregateCounterSums(t, report);
4146
});
4247

4348
test('Run a test with an ESM processor', async (t) => {
@@ -63,5 +68,7 @@ test('Run a test with an ESM processor', async (t) => {
6368
10,
6469
'Should have emitted 10 custom metrics from ts processor'
6570
);
71+
6672
checkForNegativeValues(t, report);
73+
checkAggregateCounterSums(t, report);
6774
});

packages/artillery/test/cloud-e2e/lambda/lambda-bom.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ const {
66
generateTmpReportPath,
77
getImageArchitecture
88
} = require('../../helpers');
9-
const { checkForNegativeValues } = require('../../helpers/expectations');
9+
const {
10+
checkForNegativeValues,
11+
checkAggregateCounterSums
12+
} = require('../../helpers/expectations');
1013

1114
const tags = getTestTags(['type:acceptance']);
1215

@@ -61,5 +64,7 @@ tap.test('Run mixed-hierarchy test in Lambda Container', async (t) => {
6164
20,
6265
'Should have 20 "200 OK" responses'
6366
);
67+
6468
checkForNegativeValues(t, report);
69+
checkAggregateCounterSums(t, report);
6570
});

packages/artillery/test/cloud-e2e/lambda/lambda-dotenv.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ const {
66
generateTmpReportPath,
77
getImageArchitecture
88
} = require('../../helpers');
9-
const { checkForNegativeValues } = require('../../helpers/expectations');
9+
const {
10+
checkForNegativeValues,
11+
checkAggregateCounterSums
12+
} = require('../../helpers/expectations');
1013

1114
const tags = getTestTags(['type:acceptance']);
1215
const A9_PATH = process.env.A9_PATH || 'artillery';
@@ -44,5 +47,7 @@ tap.test('Run dotenv test in Lambda Container', async (t) => {
4447
50,
4548
'Should have custom counter for env variable fruit'
4649
);
50+
4751
checkForNegativeValues(t, report);
52+
checkAggregateCounterSums(t, report);
4853
});

packages/artillery/test/cloud-e2e/lambda/lambda-ensure.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ const {
77
getTestTags,
88
getImageArchitecture
99
} = require('../../helpers');
10-
const { checkForNegativeValues } = require('../../helpers/expectations');
10+
const {
11+
checkForNegativeValues,
12+
checkAggregateCounterSums
13+
} = require('../../helpers/expectations');
1114

1215
//NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
1316
const tags = getTestTags(['type:acceptance']);
@@ -46,6 +49,8 @@ tap.test('Lambda Container run uses ensure', async (t) => {
4649
300,
4750
'Should have 300 "200 OK" responses'
4851
);
52+
4953
checkForNegativeValues(t, report);
54+
checkAggregateCounterSums(t, report);
5055
}
5156
});

packages/artillery/test/cloud-e2e/lambda/lambda-expect.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ const {
77
getTestTags,
88
getImageArchitecture
99
} = require('../../helpers');
10-
const { checkForNegativeValues } = require('../../helpers/expectations');
10+
const {
11+
checkForNegativeValues,
12+
checkAggregateCounterSums
13+
} = require('../../helpers/expectations');
1114

1215
const tags = getTestTags(['type:acceptance']);
1316
let reportFilePath;
@@ -54,7 +57,9 @@ tap.test(
5457
10,
5558
'Should have 10 "200 OK" responses'
5659
);
60+
5761
checkForNegativeValues(t, report);
62+
checkAggregateCounterSums(t, report);
5863
}
5964
}
6065
);

packages/artillery/test/cloud-e2e/lambda/lambda-smoke.test.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ const {
66
generateTmpReportPath,
77
getImageArchitecture
88
} = require('../../helpers');
9-
const { checkForNegativeValues } = require('../../helpers/expectations');
9+
const {
10+
checkForNegativeValues,
11+
checkAggregateCounterSums
12+
} = require('../../helpers/expectations');
1013

1114
const tags = getTestTags(['type:acceptance']);
1215

@@ -74,6 +77,8 @@ tap.test(
7477
2,
7578
'Should have emitted 2 errors'
7679
);
80+
7781
checkForNegativeValues(t, report);
82+
checkAggregateCounterSums(t, report);
7883
}
7984
);

packages/artillery/test/helpers/expectations.js

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,35 @@ const checkForNegativeValues = (t, report) => {
3838
}
3939
};
4040

41+
const checkAggregateCounterSums = (t, report) => {
42+
const aggregateCounters = report.aggregate?.counters;
43+
44+
if (!aggregateCounters || Object.keys(aggregateCounters).length === 0) {
45+
t.fail('No aggregate counters found in the report');
46+
}
47+
48+
let intermediateCounters = {};
49+
50+
for (const intermediate of report.intermediate) {
51+
for (const key in intermediate.counters) {
52+
if (intermediateCounters[key]) {
53+
intermediateCounters[key] += intermediate.counters[key];
54+
} else {
55+
intermediateCounters[key] = intermediate.counters[key];
56+
}
57+
}
58+
}
59+
60+
for (const key in aggregateCounters) {
61+
if (aggregateCounters[key] !== intermediateCounters[key]) {
62+
t.fail(
63+
`Aggregate counter sum mismatch for ${key}. Aggregate ${aggregateCounters[key]} != Intermediate ${intermediateCounters[key]}`
64+
);
65+
}
66+
}
67+
};
68+
4169
module.exports = {
42-
checkForNegativeValues
70+
checkForNegativeValues,
71+
checkAggregateCounterSums
4372
};

0 commit comments

Comments
 (0)