Show list of failed checks

Author: DefCon-007

src/githubHelper.ts

@@ -7,10 +7,11 @@ import { RestEndpointMethodTypes } from '@octokit/plugin-rest-endpoint-methods'
 import {
   cleanScriptPath,
   extractTestRunId,
+  fetchChecks,
   fetchTestRunSummary,
 } from './k6helper'
 import {
-  generateMetricsSummary,
+  generateMarkdownSummary,
   getTestRunStatusMarkdown,
 } from './markdownRenderer'
 import { TestRunUrlsMap } from './types'
@@ -176,9 +177,11 @@ export async function generatePRComment(
   core.debug('Generating PR comment')
 
   const resultSummaryStrings = ['# Performance Test Results 🚀\n\n']
-  let testRunIndex = 1
+  let testRunIndex = 0
 
   for (const [scriptPath, testRunUrl] of Object.entries(testRunUrlsMap)) {
+    testRunIndex++
+
     resultSummaryStrings.push(
       `## ${testRunIndex}. 🔗 [${cleanScriptPath(scriptPath)}](${testRunUrl})\n`
     )
@@ -200,16 +203,18 @@ export async function generatePRComment(
     }
 
     resultSummaryStrings.push(
-      getTestRunStatusMarkdown(testRunSummary.test_run_status)
+      getTestRunStatusMarkdown(testRunSummary.run_status)
     )
 
-    const markdownSummary = generateMetricsSummary(
-      testRunSummary.metrics_summary
+    const checks = await fetchChecks(testRunId)
+
+    const markdownSummary = generateMarkdownSummary(
+      testRunSummary.metrics_summary,
+      checks
     )
 
     resultSummaryStrings.push(markdownSummary)
     resultSummaryStrings.push('\n')
-    testRunIndex++
   }
 
   const comment = resultSummaryStrings.join('\n')

src/k6helper.ts

@@ -4,7 +4,11 @@ import { ChildProcess, spawn } from 'child_process'
 import path from 'path'
 import { apiRequest } from './apiUtils'
 import { parseK6Output } from './k6OutputParser'
-import { TestRunSummary, TestRunUrlsMap } from './types'
+import { Check, ChecksResponse, TestRunSummary, TestRunUrlsMap } from './types'
+
+function getK6CloudBaseUrl(): string {
+  return process.env.K6_CLOUD_BASE_URL || 'https://api.k6.io'
+}
 
 /**
  * Validates the test paths by running `k6 inspect --execution-requirements` on each test file.
@@ -211,8 +215,31 @@ export function extractTestRunId(testRunUrl: string): string | null {
 export async function fetchTestRunSummary(
   testRunId: string
 ): Promise<TestRunSummary | undefined> {
-  const baseUrl = process.env.K6_CLOUD_BASE_URL || 'https://api.k6.io/cloud/v5'
-  const url = `${baseUrl}/test_runs(${testRunId})/result_summary?$select=metrics_summary`
+  const baseUrl = getK6CloudBaseUrl()
+  const url = `${baseUrl}/cloud/v5/test_runs(${testRunId})/result_summary?$select=metrics_summary`
 
   return apiRequest<TestRunSummary>(url)
 }
+
+/**
+ * Fetches the checks for a test run from the Grafana Cloud K6 API.
+ * Uses retry mechanism with exponential backoff for reliability.
+ * Will automatically retry on transient errors and server errors, but not on client errors like 404 or 401.
+ *
+ * @param {string} testRunId - The ID of the test run to fetch checks for
+ * @returns {Promise<Check[]>} Array of checks or empty array if there was an error
+ */
+export async function fetchChecks(testRunId: string): Promise<Check[]> {
+  const baseUrl = getK6CloudBaseUrl()
+  const url = `${baseUrl}/loadtests/v4/test_runs(${testRunId})/checks?$select=name,metric_summary&$filter=group_id eq null`
+
+  const response = await apiRequest<ChecksResponse>(url)
+
+  // If the API request fails, return an empty array
+  if (!response) {
+    return []
+  }
+
+  // Return the checks array from the response
+  return response.value
+}

src/markdownRenderer.ts

@@ -1,5 +1,7 @@
+import { isNumber } from '@latest-version/orval-core'
 import {
   BrowserMetricSummary,
+  Check,
   ChecksMetricSummary,
   GrpcMetricSummary,
   HttpMetricSummary,
@@ -239,7 +241,8 @@ export function getBrowserMetricsMarkdown(
  * @returns Markdown string for checks metrics
  */
 export function getChecksMarkdown(
-  checksMetrics: ChecksMetricSummary | null
+  checksMetrics: ChecksMetricSummary | null,
+  checks: Check[] | null
 ): string[] {
   if (
     !checksMetrics ||
@@ -253,15 +256,42 @@ export function getChecksMarkdown(
   const markdownSections = []
 
   if (
-    checksMetrics.successes != null &&
+    isNumber(checksMetrics.successes) &&
     checksMetrics.successes < checksMetrics.total
   ) {
     markdownSections.push(
       `- ❌ **${formatNumber(checksMetrics.total - checksMetrics.successes)}** out of **${formatNumber(checksMetrics.total)}** checks were not successful.`
     )
+
+    // Add checks that failed
+    if (checks && checks.length > 0) {
+      // Aggregate checks by name
+      const checksByName: Record<
+        string,
+        { success_count: number; fail_count: number }
+      > = {}
+
+      // Group checks by name and aggregate metrics
+      checks.forEach((check) => {
+        const { name, metric_summary } = check
+        if (!checksByName[name]) {
+          checksByName[name] = { success_count: 0, fail_count: 0 }
+        }
+        checksByName[name].success_count += metric_summary.success_count
+        checksByName[name].fail_count += metric_summary.fail_count
+      })
+      // List failed checks (those with fail_count > 0)
+      Object.entries(checksByName)
+        .filter(([, metrics]) => metrics.fail_count > 0)
+        .forEach(([name, metrics]) => {
+          markdownSections.push(
+            `  - \`${name}\`: Failed **${formatNumber(metrics.fail_count)}**, out of **${formatNumber(metrics.success_count + metrics.fail_count)}** times.`
+          )
+        })
+    }
   } else {
     markdownSections.push(
-      `-✅ All **${formatNumber(checksMetrics.total)}** checks were successful.`
+      `- ✅ All **${formatNumber(checksMetrics.total)}** checks were successful.`
     )
   }
 
@@ -331,16 +361,17 @@ export function getTestRunStatusMarkdown(
  * @param metricsSummary The complete metrics summary object
  * @returns Markdown string for all metrics
  */
-export function generateMetricsSummary(
-  metricsSummary: MetricsSummary | null | undefined
+export function generateMarkdownSummary(
+  metricsSummary: MetricsSummary | null | undefined,
+  checks: Check[] | null
 ): string {
   if (!metricsSummary) return 'No metrics data available.'
 
   const markdownSections: string[] = []
 
   // Add checks summary
   markdownSections.push(
-    ...getChecksMarkdown(metricsSummary.checks_metric_summary)
+    ...getChecksMarkdown(metricsSummary.checks_metric_summary, checks)
   )
 
   // Add thresholds summary
@@ -368,7 +399,5 @@ export function generateMetricsSummary(
     ...getBrowserMetricsMarkdown(metricsSummary.browser_metric_summary)
   )
 
-  return markdownSections.length
-    ? markdownSections.join('\n')
-    : 'No metrics data available.'
+  return markdownSections.length ? markdownSections.join('\n') : ''
 }

src/types.ts

@@ -82,5 +82,29 @@ export type MetricsSummary = {
 
 export type TestRunSummary = {
   metrics_summary: MetricsSummary
-  test_run_status: number
+  run_status: number
+}
+
+/**
+ * Interface for the metric summary of a check
+ */
+export interface CheckMetricSummary {
+  fail_count: number
+  success_count: number
+  success_rate: number
+}
+
+/**
+ * Interface for a single check item
+ */
+export interface Check {
+  metric_summary: CheckMetricSummary
+  name: string
+}
+
+/**
+ * Interface for the checks API response
+ */
+export interface ChecksResponse {
+  value: Check[]
 }

test/k6helper.test.ts

@@ -1,14 +1,16 @@
 import { spawn } from 'child_process'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { apiRequest } from '../src/apiUtils'
 import {
   cleanScriptPath,
   executeRunK6Command,
   extractTestRunId,
+  fetchChecks,
   generateK6RunCommand,
   isCloudIntegrationEnabled,
   validateTestPaths,
 } from '../src/k6helper'
-import { generateMetricsSummary } from '../src/markdownRenderer'
+import { generateMarkdownSummary } from '../src/markdownRenderer'
 import { TestRunUrlsMap } from '../src/types'
 
 // Mock child_process.spawn
@@ -37,6 +39,11 @@ vi.mock('@actions/core', () => ({
   setFailed: vi.fn(),
 }))
 
+// Mock apiRequest to return specific responses for different tests
+vi.mock('../src/apiUtils', () => ({
+  apiRequest: vi.fn(),
+}))
+
 describe('cleanScriptPath', () => {
   const originalEnv = process.env
 
@@ -241,13 +248,17 @@ describe('extractTestRunId', () => {
   })
 })
 
-describe('generateMetricsSummary', () => {
+describe('generateMarkdownSummary', () => {
   it('should return a default message for null metrics', () => {
-    expect(generateMetricsSummary(null)).toBe('No metrics data available.')
+    expect(generateMarkdownSummary(null, null)).toBe(
+      'No metrics data available.'
+    )
   })
 
   it('should return a default message for undefined metrics', () => {
-    expect(generateMetricsSummary(undefined)).toBe('No metrics data available.')
+    expect(generateMarkdownSummary(undefined, null)).toBe(
+      'No metrics data available.'
+    )
   })
 
   it('should return a default message for empty metrics', () => {
@@ -260,7 +271,7 @@ describe('generateMetricsSummary', () => {
       browser_metric_summary: null,
     }
 
-    expect(generateMetricsSummary(emptyMetrics)).toBe(
+    expect(generateMarkdownSummary(emptyMetrics, null)).toBe(
       'No metrics data available.'
     )
   })
@@ -278,8 +289,71 @@ describe('generateMetricsSummary', () => {
       browser_metric_summary: null,
     }
 
-    const result = generateMetricsSummary(metrics)
+    const result = generateMarkdownSummary(metrics, null)
 
     expect(result).toContain('checks were not successful')
   })
 })
+
+describe('fetchChecks', () => {
+  beforeEach(() => {
+    vi.resetAllMocks()
+  })
+
+  it('should return checks array when API request succeeds', async () => {
+    // Mock response from the API
+    const mockChecksResponse = {
+      '@count': 3,
+      value: [
+        {
+          group_id: null,
+          id: 'c43e6629-af3b-59c4-a047-531dca123851',
+          metric_summary: {
+            fail_count: 10,
+            success_count: 0,
+            success_rate: 0.0,
+          },
+          name: 'black friday is present',
+          scenario_id: 'efeb799c-a053-56bf-9a26-c59d78537bc0',
+        },
+        {
+          group_id: null,
+          id: 'f9c6591a-3195-525f-a1e8-c39173a65056',
+          metric_summary: {
+            fail_count: 0,
+            success_count: 10,
+            success_rate: 1.0,
+          },
+          name: 'Connected successfully',
+          scenario_id: 'efeb799c-a053-56bf-9a26-c59d78537bc0',
+        },
+      ],
+    }
+
+    // Mock the apiRequest function to return our mock response
+    vi.mocked(apiRequest).mockResolvedValueOnce(mockChecksResponse)
+
+    // Call the function
+    const result = await fetchChecks('1234')
+
+    // Verify the result
+    expect(result).toEqual(mockChecksResponse.value)
+    expect(apiRequest).toHaveBeenCalledWith(
+      expect.stringContaining('/test_runs(1234)/checks')
+    )
+  })
+
+  it('should return empty array when API request fails', async () => {
+    // Mock the apiRequest function to return undefined (API failure)
+    vi.mocked(apiRequest).mockResolvedValueOnce(undefined)
+
+    // Call the function
+    const result = await fetchChecks('1234')
+
+    // Verify the result
+    expect(result).toEqual([])
+    expect(apiRequest).toHaveBeenCalledWith(
+      expect.stringContaining('/test_runs(1234)/checks')
+    )
+  })
+})