Skip to content

Commit 4c81af3

Browse files
clampleclample
committed
feat: add support for test retries [sc-20570]
1 parent 7e9363c commit 4c81af3

27 files changed

+340
-124
lines changed

Diff for: ‎examples/boilerplate-project/__checks__/api.check.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ new ApiCheck('books-api-check-1', {
1111
followRedirects: true,
1212
skipSSL: false,
1313
assertions: [
14-
AssertionBuilder.statusCode().equals(200),
14+
AssertionBuilder.statusCode().equals(201),
1515
AssertionBuilder.jsonBody('$[0].id').isNotNull(),
1616
],
1717
},

Diff for: ‎examples/boilerplate-project/__checks__/homepage.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { test, expect } from '@playwright/test'
55

66
test('webshop homepage', async ({ page }) => {
77
const response = await page.goto('https://danube-web.shop')
8-
expect(response?.status()).toBeLessThan(400)
8+
expect(response?.status()).toBeLessThan(100)
99
await expect(page).toHaveTitle(/Danube WebShop/)
1010
await page.screenshot({ path: 'homepage.jpg' })
1111
})

Diff for: ‎examples/boilerplate-project/checkly.config.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ const config = defineConfig({
2323
/** The Checkly Runtime identifier, determining npm packages and the Node.js version available at runtime.
2424
* See https://www.checklyhq.com/docs/cli/npm-packages/
2525
*/
26-
runtimeId: '2024.02',
26+
runtimeId: '2023.09',
2727
/* A glob pattern that matches the Checks inside your repo, see https://www.checklyhq.com/docs/cli/using-check-test-match/ */
2828
checkMatch: '**/__checks__/**/*.check.ts',
2929
/* Global configuration option for Playwright-powered checks. See https://docs/browser-checks/playwright-test/#global-configuration */

Diff for: ‎package-lock.json

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎packages/cli/e2e/__tests__/fixtures/retry-project/checkly.config.ts

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { defineConfig } from 'checkly'
2+
3+
const config = defineConfig({
4+
projectName: 'Test Project',
5+
logicalId: 'test-project',
6+
repoUrl: 'https://github.com/checkly/checkly-cli',
7+
checks: {
8+
locations: ['us-east-1', 'eu-west-1'],
9+
tags: ['mac'],
10+
runtimeId: '2023.09',
11+
checkMatch: '**/*.check.ts',
12+
browserChecks: {
13+
// using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
14+
testMatch: '**/__checks__/*.test.ts',
15+
},
16+
},
17+
cli: {
18+
runLocation: 'us-east-1',
19+
},
20+
})
21+
22+
export default config

Diff for: ‎packages/cli/e2e/__tests__/fixtures/retry-project/src/__checks__/group.check.ts

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { CheckGroup, BrowserCheck } from 'checkly/constructs'
2+
3+
const group = new CheckGroup('check-group-1', {
4+
name: 'Group',
5+
activated: true,
6+
muted: false,
7+
locations: ['us-east-1', 'eu-west-1'],
8+
tags: ['mac', 'group'],
9+
environmentVariables: [],
10+
apiCheckDefaults: {},
11+
alertChannels: [],
12+
browserChecks: {
13+
// using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
14+
testMatch: '**/*.test.ts',
15+
},
16+
})
17+
18+
const browserCheck = new BrowserCheck('group-browser-check-1', {
19+
name: 'Check with group',
20+
activated: false,
21+
groupId: group.ref(),
22+
code: {
23+
content: 'throw new Error("Failing Check Result")',
24+
},
25+
})

Diff for: ‎packages/cli/e2e/__tests__/test.spec.ts

+13
Original file line numberDiff line numberDiff line change
@@ -191,4 +191,17 @@ describe('test', () => {
191191
fs.rmSync(snapshotDir, { recursive: true })
192192
}
193193
})
194+
195+
it('Should execute retries', async () => {
196+
const result = await runChecklyCli({
197+
args: ['test', '--retries=3'],
198+
apiKey: config.get('apiKey'),
199+
accountId: config.get('accountId'),
200+
directory: path.join(__dirname, 'fixtures', 'retry-project'),
201+
timeout: 120000, // 2 minutes
202+
})
203+
// The failing check result will have "Failing Check Result" in the output.
204+
// We expect the check to be run 4 times.
205+
expect(result.stdout.match(/Failing Check Result/g)).toHaveLength(4)
206+
})
194207
})

Diff for: ‎packages/cli/e2e/run-checkly.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ export function runChecklyCli (options: {
3333
CHECKLY_API_KEY: apiKey,
3434
CHECKLY_ACCOUNT_ID: accountId,
3535
CHECKLY_ENV: process.env.CHECKLY_ENV,
36-
CHECKLY_CLI_VERSION: cliVersion,
36+
// We need the CLI to report 4.8.0 or greater in order for the backend to use the new MQTT topic format.
37+
// Once 4.8.0 has been released, we can remove the 4.8.0 fallback here.
38+
CHECKLY_CLI_VERSION: cliVersion ?? '4.8.0',
3739
CHECKLY_E2E_PROMPTS_INJECTIONS: promptsInjection?.length ? JSON.stringify(promptsInjection) : undefined,
3840
...env,
3941
},

Diff for: ‎packages/cli/src/commands/test.ts

+37-9
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ import {
88
Events,
99
RunLocation,
1010
PrivateRunLocation,
11-
CheckRunId,
11+
SequenceId,
1212
DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
1313
} from '../services/abstract-check-runner'
1414
import TestRunner from '../services/test-runner'
1515
import { loadChecklyConfig } from '../services/checkly-config-loader'
1616
import { filterByFileNamePattern, filterByCheckNamePattern, filterByTags } from '../services/test-filters'
1717
import type { Runtime } from '../rest/runtimes'
1818
import { AuthCommand } from './authCommand'
19-
import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, Session } from '../constructs'
19+
import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, RetryStrategyBuilder, Session } from '../constructs'
2020
import type { Region } from '..'
2121
import { splitConfigFilePath, getGitInformation, getCiInformation, getEnvs } from '../services/util'
2222
import { createReporters, ReporterType } from '../reporters/reporter'
@@ -26,6 +26,7 @@ import { printLn, formatCheckTitle, CheckStatus } from '../reporters/util'
2626
import { uploadSnapshots } from '../services/snapshot-service'
2727

2828
const DEFAULT_REGION = 'eu-central-1'
29+
const MAX_RETRIES = 3
2930

3031
export default class Test extends AuthCommand {
3132
static coreCommand = true
@@ -100,6 +101,9 @@ export default class Test extends AuthCommand {
100101
description: 'Update any snapshots using the actual result of this test run.',
101102
default: false,
102103
}),
104+
retries: Flags.integer({
105+
description: 'How many times to retry a failing test run.',
106+
}),
103107
}
104108

105109
static args = {
@@ -132,6 +136,7 @@ export default class Test extends AuthCommand {
132136
record: shouldRecord,
133137
'test-session-name': testSessionName,
134138
'update-snapshots': updateSnapshots,
139+
retries,
135140
} = flags
136141
const filePatterns = argv as string[]
137142

@@ -228,6 +233,7 @@ export default class Test extends AuthCommand {
228233
const reporters = createReporters(reporterTypes, location, verbose)
229234
const repoInfo = getGitInformation(project.repoUrl)
230235
const ciInfo = getCiInformation()
236+
const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
231237

232238
const runner = new TestRunner(
233239
config.getAccountId(),
@@ -241,34 +247,43 @@ export default class Test extends AuthCommand {
241247
ciInfo.environment,
242248
updateSnapshots,
243249
configDirectory,
250+
testRetryStrategy,
244251
)
245252

246253
runner.on(Events.RUN_STARTED,
247-
(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
254+
(checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
248255
reporters.forEach(r => r.onBegin(checks, testSessionId)),
249256
)
250257

251-
runner.on(Events.CHECK_INPROGRESS, (check: any, checkRunId: CheckRunId) => {
252-
reporters.forEach(r => r.onCheckInProgress(check, checkRunId))
258+
runner.on(Events.CHECK_INPROGRESS, (check: any, sequenceId: SequenceId) => {
259+
reporters.forEach(r => r.onCheckInProgress(check, sequenceId))
253260
})
254261

255262
runner.on(Events.MAX_SCHEDULING_DELAY_EXCEEDED, () => {
256263
reporters.forEach(r => r.onSchedulingDelayExceeded())
257264
})
258265

259-
runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, check, result, links?: TestResultsShortLinks) => {
266+
runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
267+
reporters.forEach(r => r.onCheckAttemptResult(sequenceId, {
268+
logicalId: check.logicalId,
269+
sourceFile: check.getSourceFile(),
270+
...result,
271+
}, links))
272+
})
273+
274+
runner.on(Events.CHECK_SUCCESSFUL, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
260275
if (result.hasFailures) {
261276
process.exitCode = 1
262277
}
263278

264-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
279+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
265280
logicalId: check.logicalId,
266281
sourceFile: check.getSourceFile(),
267282
...result,
268283
}, links))
269284
})
270-
runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
271-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
285+
runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
286+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
272287
...check,
273288
logicalId: check.logicalId,
274289
sourceFile: check.getSourceFile(),
@@ -337,6 +352,19 @@ export default class Test extends AuthCommand {
337352
}
338353
}
339354

355+
prepareTestRetryStrategy (retries?: number, configRetries?: number) {
356+
const numRetries = retries ?? configRetries ?? 0
357+
if (numRetries > MAX_RETRIES) {
358+
printLn(`Defaulting to the maximum of ${MAX_RETRIES} retries.`)
359+
}
360+
return numRetries
361+
? RetryStrategyBuilder.fixedStrategy({
362+
maxRetries: Math.min(numRetries, MAX_RETRIES),
363+
baseBackoffSeconds: 0,
364+
})
365+
: null
366+
}
367+
340368
private listChecks (checks: Array<Check>) {
341369
// Sort and print the checks in a way that's consistent with AbstractListReporter
342370
const sortedCheckFiles = [...new Set(checks.map((check) => check.getSourceFile()))].sort()

Diff for: ‎packages/cli/src/commands/trigger.ts

+31-7
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,15 @@ import { loadChecklyConfig } from '../services/checkly-config-loader'
77
import { splitConfigFilePath, getEnvs, getGitInformation, getCiInformation } from '../services/util'
88
import type { Region } from '..'
99
import TriggerRunner, { NoMatchingChecksError } from '../services/trigger-runner'
10-
import { RunLocation, Events, PrivateRunLocation, CheckRunId } from '../services/abstract-check-runner'
10+
import { RunLocation, Events, PrivateRunLocation, SequenceId } from '../services/abstract-check-runner'
1111
import config from '../services/config'
1212
import { createReporters, ReporterType } from '../reporters/reporter'
13+
import { printLn } from '../reporters/util'
1314
import { TestResultsShortLinks } from '../rest/test-sessions'
14-
import { Session } from '../constructs'
15+
import { Session, RetryStrategyBuilder } from '../constructs'
1516

1617
const DEFAULT_REGION = 'eu-central-1'
18+
const MAX_RETRIES = 3
1719

1820
export default class Trigger extends AuthCommand {
1921
static coreCommand = true
@@ -74,6 +76,9 @@ export default class Trigger extends AuthCommand {
7476
char: 'n',
7577
description: 'A name to use when storing results in Checkly with --record.',
7678
}),
79+
retries: Flags.integer({
80+
description: 'How many times to retry a check run.',
81+
}),
7782
}
7883

7984
async run (): Promise<void> {
@@ -90,6 +95,7 @@ export default class Trigger extends AuthCommand {
9095
env,
9196
'env-file': envFile,
9297
'test-session-name': testSessionName,
98+
retries,
9399
} = flags
94100
const envVars = await getEnvs(envFile, env)
95101
const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
@@ -106,6 +112,7 @@ export default class Trigger extends AuthCommand {
106112
const verbose = this.prepareVerboseFlag(verboseFlag, checklyConfig?.cli?.verbose)
107113
const reporterTypes = this.prepareReportersTypes(reporterFlag as ReporterType, checklyConfig?.cli?.reporters)
108114
const reporters = createReporters(reporterTypes, location, verbose)
115+
const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
109116

110117
const repoInfo = getGitInformation()
111118
const ciInfo = getCiInformation()
@@ -121,19 +128,23 @@ export default class Trigger extends AuthCommand {
121128
repoInfo,
122129
ciInfo.environment,
123130
testSessionName,
131+
testRetryStrategy,
124132
)
125133
// TODO: This is essentially the same for `checkly test`. Maybe reuse code.
126134
runner.on(Events.RUN_STARTED,
127-
(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
135+
(checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
128136
reporters.forEach(r => r.onBegin(checks, testSessionId)))
129-
runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, _, result, links?: TestResultsShortLinks) => {
137+
runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
138+
reporters.forEach(r => r.onCheckAttemptResult(sequenceId, result, links))
139+
})
140+
runner.on(Events.CHECK_SUCCESSFUL, (sequenceId: SequenceId, _, result, links?: TestResultsShortLinks) => {
130141
if (result.hasFailures) {
131142
process.exitCode = 1
132143
}
133-
reporters.forEach(r => r.onCheckEnd(checkRunId, result, links))
144+
reporters.forEach(r => r.onCheckEnd(sequenceId, result, links))
134145
})
135-
runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
136-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
146+
runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
147+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
137148
...check,
138149
hasFailures: true,
139150
runError: message,
@@ -206,4 +217,17 @@ export default class Trigger extends AuthCommand {
206217
}
207218
return reporterFlag ? [reporterFlag] : cliReporters
208219
}
220+
221+
prepareTestRetryStrategy (retries?: number, configRetries?: number) {
222+
const numRetries = retries ?? configRetries ?? 0
223+
if (numRetries > MAX_RETRIES) {
224+
printLn(`Defaulting to the maximum of ${MAX_RETRIES} retries.`)
225+
}
226+
return numRetries
227+
? RetryStrategyBuilder.fixedStrategy({
228+
maxRetries: Math.min(numRetries, MAX_RETRIES),
229+
baseBackoffSeconds: 0,
230+
})
231+
: null
232+
}
209233
}

Diff for: ‎packages/cli/src/reporters/__tests__/__snapshots__/json-builder.spec.ts.snap

+12-6
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ exports[`JsonBuilder renders JSON markdown output with assets & links: json-with
1313
"durationMilliseconds": 6522,
1414
"filename": "src/__checks__/folder/browser.check.ts",
1515
"link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/702961fd-7e2c-45f0-97be-1aa9eabd4d82",
16-
"runError": "Run error"
16+
"runError": "Run error",
17+
"retries": 0
1718
},
1819
{
1920
"result": "Pass",
@@ -22,7 +23,8 @@ exports[`JsonBuilder renders JSON markdown output with assets & links: json-with
2223
"durationMilliseconds": 1234,
2324
"filename": "src/some-other-folder/api.check.ts",
2425
"link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/1c0be612-a5ec-432e-ac1c-837d2f70c010",
25-
"runError": "Run error"
26+
"runError": "Run error",
27+
"retries": 0
2628
}
2729
]
2830
}"
@@ -40,7 +42,8 @@ exports[`JsonBuilder renders basic JSON output with no assets & links: json-basi
4042
"durationMilliseconds": 6522,
4143
"filename": "src/__checks__/folder/browser.check.ts",
4244
"link": null,
43-
"runError": null
45+
"runError": null,
46+
"retries": 0
4447
},
4548
{
4649
"result": "Pass",
@@ -49,7 +52,8 @@ exports[`JsonBuilder renders basic JSON output with no assets & links: json-basi
4952
"durationMilliseconds": 1234,
5053
"filename": "src/some-other-folder/api.check.ts",
5154
"link": null,
52-
"runError": null
55+
"runError": null,
56+
"retries": 0
5357
}
5458
]
5559
}"
@@ -67,7 +71,8 @@ exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] =
6771
"durationMilliseconds": 6522,
6872
"filename": "src/__checks__/folder/browser.check.ts",
6973
"link": null,
70-
"runError": "Run error"
74+
"runError": "Run error",
75+
"retries": 0
7176
},
7277
{
7378
"result": "Pass",
@@ -76,7 +81,8 @@ exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] =
7681
"durationMilliseconds": 1234,
7782
"filename": "src/some-other-folder/api.check.ts",
7883
"link": null,
79-
"runError": "Run error"
84+
"runError": "Run error",
85+
"retries": 0
8086
}
8187
]
8288
}"

Diff for: ‎packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ export const apiCheckResult = {
55
sourceInfo: {
66
checkRunId: '4f20dfa7-8c66-4a15-8c43-5dc24f6206c6',
77
checkRunSuiteId: '6390a87e-89c7-4295-b6f8-b23e87922ef3',
8+
sequenceId: '72c5d10f-fc68-4361-a779-8543575336ae',
89
ephemeral: true,
910
},
1011
checkRunId: '1c0be612-a5ec-432e-ac1c-837d2f70c010',

0 commit comments

Comments
 (0)