Merge pull request #5 from grafana/validate-tests

[feat] Validate test paths before executing test run

Author: DefCon-007

.github/workflows/test.yaml

@@ -11,7 +11,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Setup Grafana k6
-      uses: grafana/setup-k6-action@v0.0.1
+      uses: grafana/setup-k6-action@main
       with:
         k6-version: '0.49.0'
     - uses: ./
@@ -21,4 +21,21 @@ jobs:
           ./dev/protocol*.js
         flags: --vus 10 --duration 30s
         parallel: true
-        cloud-run-locally: false
+        cloud-run-locally: false
+  verify-scripts:
+    runs-on: ubuntu-latest
+    env:
+      K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }}
+      K6_CLOUD_PROJECT_ID:  ${{ secrets.K6_CLOUD_PROJECT_ID }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup Grafana k6
+      uses: grafana/setup-k6-action@main
+      with:
+        k6-version: '0.49.0'
+    - uses: ./
+      continue-on-error: true
+      with:
+        path: |
+          ./dev/verify-script-tests/**.js
+        only-verify-scripts: true

action.yaml

@@ -29,6 +29,10 @@ inputs:
     description: 'If true, comment the cloud test run URL on the pull request'
     default: "true"
     required: false
+  only-verify-scripts: 
+    description: 'If true, only check if the test scripts are valid and skip test execution'
+    default: "false"
+    required: false
 
 runs:
   using: node20

dev/verify-script-tests/invalid-test.js

@@ -0,0 +1,15 @@
+import { sleep } from "k6";
+import http from "k6/http";
+
+export const options = {
+  scenarios: {
+    createBrowser: {
+      executor: "constant-arrival-rate",
+    },
+  },
+};
+
+export default function () {
+  http.get("http://test.k6.io");
+  sleep(1);
+}

dev/verify-script-tests/no-test.js

@@ -0,0 +1,2 @@
+// Sample JS file with no tests
+console.log("No tests in this file");

dev/verify-script-tests/valid-test.js

@@ -0,0 +1,7 @@
+import http from 'k6/http';
+import { sleep } from 'k6';
+
+export default function () {
+  http.get('http://test.k6.io');
+  sleep(1);
+}

dist/index.js

@@ -34346,6 +34346,7 @@ const child_process_1 = __nccwpck_require__(2081);
 const fs = __importStar(__nccwpck_require__(5630));
 const githubHelper_1 = __nccwpck_require__(2179);
 const k6OutputParser_1 = __nccwpck_require__(5035);
+const k6helper_1 = __nccwpck_require__(1034);
 const TEST_PIDS = [];
 run();
 /**
@@ -34359,8 +34360,28 @@ async function run() {
         const failFast = core.getInput('fail-fast', { required: false }) === 'true';
         const flags = core.getInput('flags', { required: false });
         const cloudRunLocally = core.getInput('cloud-run-locally', { required: false }) === 'true';
+        const onlyVerifyScripts = core.getInput('only-verify-scripts', { required: false }) === 'true';
         const shouldCommentCloudTestRunUrlOnPR = core.getInput('cloud-comment-on-pr', { required: false }) === 'true';
         const allPromises = [];
+        core.debug(`🔍 Found following ${testPaths.length} test run files:`);
+        testPaths.forEach((testPath, index) => {
+            core.debug(`${index + 1}. ${testPath}`);
+        });
+        if (testPaths.length === 0) {
+            throw new Error('No test files found');
+        }
+        const verifiedTestPaths = await (0, k6helper_1.validateTestPaths)(testPaths);
+        if (verifiedTestPaths.length === 0) {
+            throw new Error('No valid test files found');
+        }
+        console.log(`🧪 Found ${verifiedTestPaths.length} valid K6 tests out of total ${testPaths.length} test files.`);
+        verifiedTestPaths.forEach((testPath, index) => {
+            console.log(`  ${index + 1}. ${testPath}`);
+        });
+        if (onlyVerifyScripts) {
+            console.log('🔍 Only verifying scripts. Skipping test execution');
+            return;
+        }
         const isCloud = await isCloudIntegrationEnabled();
         const commands = testPaths.map(testPath => generateCommand(testPath)), TOTAL_TEST_RUNS = commands.length, TEST_RESULT_URLS_MAP = new Proxy({}, {
             set: (target, key, value) => {
@@ -34375,9 +34396,6 @@ async function run() {
             }
         });
         ;
-        if (commands.length === 0) {
-            throw new Error('No test files found');
-        }
         let allTestsPassed = true;
         if (parallel) {
             const childProcesses = [];
@@ -34665,6 +34683,45 @@ function parseK6Output(data, testRunUrlsMap, totalTestRuns) {
 exports.parseK6Output = parseK6Output;
 
 
+/***/ }),
+
+/***/ 1034:
+/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => {
+
+"use strict";
+
+Object.defineProperty(exports, "__esModule", ({ value: true }));
+exports.validateTestPaths = void 0;
+// Common helper functions used in the action 
+const child_process_1 = __nccwpck_require__(2081);
+async function validateTestPaths(testPaths) {
+    if (testPaths.length === 0) {
+        throw new Error('No test files found');
+    }
+    console.log(`🔍 Validating test run files.`);
+    const validK6TestPaths = [], command = "k6", defaultArgs = ["inspect", "--execution-requirements"];
+    const allPromises = [];
+    testPaths.forEach(async (testPath) => {
+        const args = [...defaultArgs, testPath];
+        const child = (0, child_process_1.spawn)(command, args, {
+            stdio: ['inherit', 'ignore', 'inherit'], // 'ignore' is for stdout
+            detached: false,
+        });
+        allPromises.push(new Promise(resolve => {
+            child.on('exit', (code, signal) => {
+                if (code === 0) {
+                    validK6TestPaths.push(testPath);
+                }
+                resolve();
+            });
+        }));
+    });
+    await Promise.all(allPromises);
+    return validK6TestPaths;
+}
+exports.validateTestPaths = validateTestPaths;
+
+
 /***/ }),
 
 /***/ 9491:

src/index.ts

@@ -4,6 +4,7 @@ import { spawn } from 'child_process';
 import * as fs from 'fs-extra';
 import { generatePRComment } from './githubHelper';
 import { parseK6Output } from './k6OutputParser';
+import { validateTestPaths } from './k6helper';
 import { TestRunUrlsMap } from './types';
 
 const TEST_PIDS: number[] = [];
@@ -21,9 +22,35 @@ export async function run(): Promise<void> {
         const failFast = core.getInput('fail-fast', { required: false }) === 'true'
         const flags = core.getInput('flags', { required: false })
         const cloudRunLocally = core.getInput('cloud-run-locally', { required: false }) === 'true'
+        const onlyVerifyScripts = core.getInput('only-verify-scripts', { required: false }) === 'true'
         const shouldCommentCloudTestRunUrlOnPR = core.getInput('cloud-comment-on-pr', { required: false }) === 'true'
         const allPromises: Promise<void>[] = [];
 
+        core.debug(`🔍 Found following ${testPaths.length} test run files:`);
+        testPaths.forEach((testPath, index) => {
+            core.debug(`${index + 1}. ${testPath}`);
+        });
+
+        if (testPaths.length === 0) {
+            throw new Error('No test files found')
+        }
+
+        const verifiedTestPaths = await validateTestPaths(testPaths);
+
+        if (verifiedTestPaths.length === 0) {
+            throw new Error('No valid test files found')
+        }
+
+        console.log(`🧪 Found ${verifiedTestPaths.length} valid K6 tests out of total ${testPaths.length} test files.`);
+        verifiedTestPaths.forEach((testPath, index) => {
+            console.log(`  ${index + 1}. ${testPath}`);
+        });
+
+        if (onlyVerifyScripts) {
+            console.log('🔍 Only verifying scripts. Skipping test execution');
+            return;
+        }
+
         const isCloud = await isCloudIntegrationEnabled()
 
         const commands = testPaths.map(testPath => generateCommand(testPath)),
@@ -41,9 +68,6 @@ export async function run(): Promise<void> {
                 }
             });
         ;
-        if (commands.length === 0) {
-            throw new Error('No test files found')
-        }
 
         let allTestsPassed = true;
 

src/k6helper.ts

@@ -0,0 +1,47 @@
+// Common helper functions used in the action 
+import { spawn } from 'child_process';
+
+export async function validateTestPaths(testPaths: string[]): Promise<string[]> {
+    /**
+     * Validates the test paths by running `k6 inspect --execution-requirements` on each test file.
+     * A test path is considered valid if the command returns an exit code of 0.
+     *
+     * @export
+     * @param {string[]} testPaths - List of test paths to validate
+     * @return {Promise<string[]>} - List of valid test paths
+     */
+
+    if (testPaths.length === 0) {
+        throw new Error('No test files found')
+    }
+
+    console.log(`🔍 Validating test run files.`);
+
+    const validK6TestPaths: string[] = [],
+        command = "k6",
+        defaultArgs = ["inspect", "--execution-requirements"];
+
+    const allPromises = [] as any[];
+
+    testPaths.forEach(async testPath => {
+        const args = [...defaultArgs, testPath];
+
+        const child = spawn(command, args, {
+            stdio: ['inherit', 'ignore', 'inherit'], // 'ignore' is for stdout
+            detached: false,
+        });
+
+        allPromises.push(new Promise<void>(resolve => {
+            child.on('exit', (code: number, signal: string) => {
+                if (code === 0) {
+                    validK6TestPaths.push(testPath);
+                }
+                resolve();
+            });
+        }));
+    });
+
+    await Promise.all(allPromises);
+
+    return validK6TestPaths;
+}