ci(test): add ability to parallelise aws tests (#2544)

* ci(test): add ability to parallelise aws tests

* ci(test): name npm script better

Author: bernardobridge

.github/workflows/run-aws-tests.yml

@@ -38,11 +38,32 @@ jobs:
     with:
       ref: ${{ github.event.pull_request.head.sha || null }} # this should only be run with this ref if is-collaborator has been run and passed
 
+  generate-test-matrix:
+    runs-on: ubuntu-latest
+    needs: publish-branch-image
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha || null }}
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+      - id: generate-matrix
+        run: |
+          RESULT=$(node .github/workflows/scripts/get-tests-in-package-location.js)
+          echo "matrix=$RESULT" >> $GITHUB_OUTPUT
+
   run-tests:
     if: contains( github.event.pull_request.labels.*.name, 'run-aws-tests' )
-    needs: publish-branch-image
+    needs: generate-test-matrix
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        testName: ${{fromJson(needs.generate-test-matrix.outputs.matrix).names}}
     permissions:
       contents: read
       id-token: write
@@ -73,10 +94,7 @@ jobs:
           node-version: 18.x
       - run: npm install
       - run: npm run build
-      - run: npm run test:aws --workspace artillery
-        env:
-          FORCE_COLOR: 1
-    
-      - run: npm run test:aws --workspace artillery-engine-playwright
+      # runs the single test file from `package` workspace in the `file`, as defined in the matrix output
+      - run: npm run test:aws:ci --workspace ${{fromJson(needs.generate-test-matrix.outputs.matrix).namesToFiles[matrix.testName].package }} -- --files ${{ fromJson(needs.generate-test-matrix.outputs.matrix).namesToFiles[matrix.testName].file }}
         env:
           FORCE_COLOR: 1

.github/workflows/scripts/get-tests-in-package-location.js

@@ -0,0 +1,64 @@
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * This script is used to discover all the tests in different test directories that match a specific suffix
+ * and generate a JSON file that can be used to run the tests in parallel leveraging Github Actions
+ */
+
+const testLocations = [
+  {
+    location: 'test/cloud-e2e/fargate',
+    package: 'artillery',
+    suffix: '.test.js'
+  },
+  {
+    location: 'test/cloud-e2e/lambda',
+    package: 'artillery',
+    suffix: '.test.js'
+  },
+  {
+    location: 'test',
+    package: 'artillery-engine-playwright',
+    suffix: '.aws.js'
+  }
+];
+
+const tests = {
+  names: [],
+  namesToFiles: {}
+};
+
+const addTest = (fileName, baseLocation, packageName, suffix) => {
+  if (!fileName.endsWith(suffix)) {
+    return;
+  }
+  const testName = fileName.replace(suffix, '');
+  const jobName = `${packageName}/${testName}`;
+  tests.names.push(jobName);
+  tests.namesToFiles[jobName] = {
+    file: `${baseLocation}/${fileName}`,
+    package: packageName
+  };
+};
+
+// Recursively scan a directory to find files, and add tests to the tests object
+function scanDirectory(location, baseLocation, packageName, suffix) {
+  fs.readdirSync(location).forEach((file) => {
+    const absolute = path.join(location, file);
+    if (fs.statSync(absolute).isDirectory()) {
+      scanDirectory(absolute, baseLocation, packageName, suffix);
+    } else {
+      addTest(file, baseLocation, packageName, suffix);
+    }
+  });
+}
+
+// Scan all the test locations
+for (const { package, location, suffix } of testLocations) {
+  const fullLocation = `packages/${package}/${location}`;
+  scanDirectory(fullLocation, location, package, suffix);
+}
+
+// Output the tests object as a JSON string to be used by Github Actions
+console.log(JSON.stringify(tests));

packages/artillery-engine-playwright/package.json

@@ -5,7 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap test/*.test.js --color --no-coverage --timeout 300",
-    "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap test/*.aws.js --color --no-coverage --timeout 420"
+    "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap test/*.aws.js --color --no-coverage --timeout 420",
+    "test:aws:ci": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=420"
   },
   "keywords": [],
   "author": "",

packages/artillery/package.json

@@ -48,6 +48,7 @@
     "test": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' &&  npm run test:unit && npm run test:acceptance",
     "test:windows": "set ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' & npm run test:unit && tap --no-coverage --timeout=420 --color test/cli/*.test.js",
     "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=4200 test/cloud-e2e/**/*.test.js",
+    "test:aws:ci": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=4200",
     "test:aws:windows": "set ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' & tap --no-coverage --timeout=420 --color test/cloud-e2e/fargate/*.test.js --grep \"Run simple-bom\"",
     "lint": "eslint --ext \".js,.ts,.tsx\" .",
     "lint-fix": "npm run lint -- --fix"