Skip to content

Commit c663617

Browse files
clampleclample
committed
feat: add support for test retries [sc-20570]
1 parent dea54f0 commit c663617

File tree

15 files changed

+196
-106
lines changed

15 files changed

+196
-106
lines changed

Diff for: ‎examples/boilerplate-project/checkly.config.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ const config = defineConfig({
2323
/** The Checkly Runtime identifier, determining npm packages and the Node.js version available at runtime.
2424
* See https://www.checklyhq.com/docs/cli/npm-packages/
2525
*/
26-
runtimeId: '2024.02',
26+
runtimeId: '2023.09',
2727
/* A glob pattern that matches the Checks inside your repo, see https://www.checklyhq.com/docs/cli/using-check-test-match/ */
2828
checkMatch: '**/__checks__/**/*.check.ts',
2929
/* Global configuration option for Playwright-powered checks. See https://docs/browser-checks/playwright-test/#global-configuration */

Diff for: ‎package-lock.json

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎packages/cli/src/commands/test.ts

+29-9
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ import {
88
Events,
99
RunLocation,
1010
PrivateRunLocation,
11-
CheckRunId,
11+
SequenceId,
1212
DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
1313
} from '../services/abstract-check-runner'
1414
import TestRunner from '../services/test-runner'
1515
import { loadChecklyConfig } from '../services/checkly-config-loader'
1616
import { filterByFileNamePattern, filterByCheckNamePattern, filterByTags } from '../services/test-filters'
1717
import type { Runtime } from '../rest/runtimes'
1818
import { AuthCommand } from './authCommand'
19-
import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, Session } from '../constructs'
19+
import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, RetryStrategyBuilder, Session } from '../constructs'
2020
import type { Region } from '..'
2121
import { splitConfigFilePath, getGitInformation, getCiInformation, getEnvs } from '../services/util'
2222
import { createReporters, ReporterType } from '../reporters/reporter'
@@ -100,6 +100,10 @@ export default class Test extends AuthCommand {
100100
description: 'Update any snapshots using the actual result of this test run.',
101101
default: false,
102102
}),
103+
retries: Flags.integer({
104+
default: 0,
105+
description: 'How many times to retry a failing test run.',
106+
}),
103107
}
104108

105109
static args = {
@@ -132,6 +136,7 @@ export default class Test extends AuthCommand {
132136
record: shouldRecord,
133137
'test-session-name': testSessionName,
134138
'update-snapshots': updateSnapshots,
139+
retries,
135140
} = flags
136141
const filePatterns = argv as string[]
137142

@@ -228,6 +233,12 @@ export default class Test extends AuthCommand {
228233
const reporters = createReporters(reporterTypes, location, verbose)
229234
const repoInfo = getGitInformation(project.repoUrl)
230235
const ciInfo = getCiInformation()
236+
const retryStrategy = retries
237+
? RetryStrategyBuilder.fixedStrategy({
238+
maxRetries: retries,
239+
baseBackoffSeconds: 0,
240+
})
241+
: null
231242

232243
const runner = new TestRunner(
233244
config.getAccountId(),
@@ -241,34 +252,43 @@ export default class Test extends AuthCommand {
241252
ciInfo.environment,
242253
updateSnapshots,
243254
configDirectory,
255+
retryStrategy,
244256
)
245257

246258
runner.on(Events.RUN_STARTED,
247-
(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
259+
(checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
248260
reporters.forEach(r => r.onBegin(checks, testSessionId)),
249261
)
250262

251-
runner.on(Events.CHECK_INPROGRESS, (check: any, checkRunId: CheckRunId) => {
252-
reporters.forEach(r => r.onCheckInProgress(check, checkRunId))
263+
runner.on(Events.CHECK_INPROGRESS, (check: any, sequenceId: SequenceId) => {
264+
reporters.forEach(r => r.onCheckInProgress(check, sequenceId))
253265
})
254266

255267
runner.on(Events.MAX_SCHEDULING_DELAY_EXCEEDED, () => {
256268
reporters.forEach(r => r.onSchedulingDelayExceeded())
257269
})
258270

259-
runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, check, result, links?: TestResultsShortLinks) => {
271+
runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
272+
reporters.forEach(r => r.onCheckAttemptResult(sequenceId, {
273+
logicalId: check.logicalId,
274+
sourceFile: check.getSourceFile(),
275+
...result,
276+
}, links))
277+
})
278+
279+
runner.on(Events.CHECK_SUCCESSFUL, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
260280
if (result.hasFailures) {
261281
process.exitCode = 1
262282
}
263283

264-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
284+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
265285
logicalId: check.logicalId,
266286
sourceFile: check.getSourceFile(),
267287
...result,
268288
}, links))
269289
})
270-
runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
271-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
290+
runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
291+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
272292
...check,
273293
logicalId: check.logicalId,
274294
sourceFile: check.getSourceFile(),

Diff for: ‎packages/cli/src/commands/trigger.ts

+22-7
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ import { loadChecklyConfig } from '../services/checkly-config-loader'
77
import { splitConfigFilePath, getEnvs, getGitInformation, getCiInformation } from '../services/util'
88
import type { Region } from '..'
99
import TriggerRunner, { NoMatchingChecksError } from '../services/trigger-runner'
10-
import { RunLocation, Events, PrivateRunLocation, CheckRunId } from '../services/abstract-check-runner'
10+
import { RunLocation, Events, PrivateRunLocation, SequenceId } from '../services/abstract-check-runner'
1111
import config from '../services/config'
1212
import { createReporters, ReporterType } from '../reporters/reporter'
1313
import { TestResultsShortLinks } from '../rest/test-sessions'
14-
import { Session } from '../constructs'
14+
import { Session, RetryStrategyBuilder } from '../constructs'
1515

1616
const DEFAULT_REGION = 'eu-central-1'
1717

@@ -74,6 +74,10 @@ export default class Trigger extends AuthCommand {
7474
char: 'n',
7575
description: 'A name to use when storing results in Checkly with --record.',
7676
}),
77+
retries: Flags.integer({
78+
default: 0,
79+
description: 'How many times to retry a check run.',
80+
}),
7781
}
7882

7983
async run (): Promise<void> {
@@ -90,6 +94,7 @@ export default class Trigger extends AuthCommand {
9094
env,
9195
'env-file': envFile,
9296
'test-session-name': testSessionName,
97+
retries,
9398
} = flags
9499
const envVars = await getEnvs(envFile, env)
95100
const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
@@ -109,6 +114,12 @@ export default class Trigger extends AuthCommand {
109114

110115
const repoInfo = getGitInformation()
111116
const ciInfo = getCiInformation()
117+
const retryStrategy = retries
118+
? RetryStrategyBuilder.fixedStrategy({
119+
maxRetries: retries,
120+
baseBackoffSeconds: 0,
121+
})
122+
: null
112123

113124
const runner = new TriggerRunner(
114125
config.getAccountId(),
@@ -121,19 +132,23 @@ export default class Trigger extends AuthCommand {
121132
repoInfo,
122133
ciInfo.environment,
123134
testSessionName,
135+
retryStrategy,
124136
)
125137
// TODO: This is essentially the same for `checkly test`. Maybe reuse code.
126138
runner.on(Events.RUN_STARTED,
127-
(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
139+
(checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
128140
reporters.forEach(r => r.onBegin(checks, testSessionId)))
129-
runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, _, result, links?: TestResultsShortLinks) => {
141+
runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
142+
reporters.forEach(r => r.onCheckAttemptResult(sequenceId, result, links))
143+
})
144+
runner.on(Events.CHECK_SUCCESSFUL, (sequenceId: SequenceId, _, result, links?: TestResultsShortLinks) => {
130145
if (result.hasFailures) {
131146
process.exitCode = 1
132147
}
133-
reporters.forEach(r => r.onCheckEnd(checkRunId, result, links))
148+
reporters.forEach(r => r.onCheckEnd(sequenceId, result, links))
134149
})
135-
runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
136-
reporters.forEach(r => r.onCheckEnd(checkRunId, {
150+
runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
151+
reporters.forEach(r => r.onCheckEnd(sequenceId, {
137152
...check,
138153
hasFailures: true,
139154
runError: message,

Diff for: ‎packages/cli/src/reporters/abstract-list.ts

+21-14
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
11
import chalk from 'chalk'
22
import indentString from 'indent-string'
33

4+
import { TestResultsShortLinks } from '../rest/test-sessions'
45
import { Reporter } from './reporter'
56
import { CheckStatus, formatCheckTitle, getTestSessionUrl, printLn } from './util'
6-
import type { CheckRunId, RunLocation } from '../services/abstract-check-runner'
7+
import type { SequenceId, RunLocation } from '../services/abstract-check-runner'
78
import { Check } from '../constructs/check'
89
import { testSessions } from '../rest/api'
910

1011
// Map from file -> checkRunId -> check+result.
1112
// This lets us print a structured list of the checks.
1213
// Map remembers the original insertion order, so each time we print the summary will be consistent.
1314
// Note that in the case of `checkly trigger`, the file will be `undefined`!
14-
export type checkFilesMap = Map<string|undefined, Map<CheckRunId, {
15+
export type checkFilesMap = Map<string|undefined, Map<SequenceId, {
1516
check?: Check,
1617
result?: any,
1718
titleString: string,
18-
testResultId?: string,
19-
checkStatus?: CheckStatus
19+
checkStatus?: CheckStatus,
20+
links?: TestResultsShortLinks,
2021
}>>
2122

2223
export default abstract class AbstractListReporter implements Reporter {
@@ -36,27 +37,26 @@ export default abstract class AbstractListReporter implements Reporter {
3637
this.verbose = verbose
3738
}
3839

39-
onBegin (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string): void {
40+
onBegin (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId?: string): void {
4041
this.testSessionId = testSessionId
4142
this.numChecks = checks.length
4243
// Sort the check files and checks alphabetically. This makes sure that there's a consistent order between runs.
4344
// For `checkly trigger`, getSourceFile() is not defined so we use optional chaining.
4445
const sortedCheckFiles = [...new Set(checks.map(({ check }) => check.getSourceFile?.()))].sort()
4546
const sortedChecks = checks.sort(({ check: a }, { check: b }) => a.name.localeCompare(b.name))
4647
this.checkFilesMap = new Map(sortedCheckFiles.map((file) => [file, new Map()]))
47-
sortedChecks.forEach(({ check, testResultId, checkRunId }) => {
48+
sortedChecks.forEach(({ check, sequenceId }) => {
4849
const fileMap = this.checkFilesMap!.get(check.getSourceFile?.())!
49-
fileMap.set(checkRunId, {
50+
fileMap.set(sequenceId, {
5051
check,
5152
titleString: formatCheckTitle(CheckStatus.SCHEDULING, check),
5253
checkStatus: CheckStatus.SCHEDULING,
53-
testResultId,
5454
})
5555
})
5656
}
5757

58-
onCheckInProgress (check: any, checkRunId: CheckRunId) {
59-
const checkFile = this.checkFilesMap!.get(check.getSourceFile?.())!.get(checkRunId)!
58+
onCheckInProgress (check: any, sequenceId: SequenceId) {
59+
const checkFile = this.checkFilesMap!.get(check.getSourceFile?.())!.get(sequenceId)!
6060
checkFile.titleString = formatCheckTitle(CheckStatus.RUNNING, check)
6161
checkFile.checkStatus = CheckStatus.RUNNING
6262
}
@@ -67,11 +67,18 @@ export default abstract class AbstractListReporter implements Reporter {
6767
this._isSchedulingDelayExceeded = true
6868
}
6969

70-
onCheckEnd (checkRunId: CheckRunId, checkResult: any) {
71-
const checkStatus = this.checkFilesMap!.get(checkResult.sourceFile)!.get(checkRunId)!
70+
onCheckAttemptResult (sequenceId: string, checkResult: any, links?: TestResultsShortLinks | undefined): void {
71+
const checkStatus = this.checkFilesMap!.get(checkResult.sourceFile)!.get(sequenceId)!
72+
checkResult.checkStatus = CheckStatus.RETRIED
73+
checkStatus.titleString = formatCheckTitle(CheckStatus.RETRIED, checkResult)
74+
}
75+
76+
onCheckEnd (sequenceId: SequenceId, checkResult: any, links?: TestResultsShortLinks) {
77+
const checkStatus = this.checkFilesMap!.get(checkResult.sourceFile)!.get(sequenceId)!
7278
checkStatus.result = checkResult
73-
const status = checkResult.hasFailures ? CheckStatus.FAILED : CheckStatus.SUCCESSFUL
74-
checkStatus.titleString = formatCheckTitle(status, checkResult, {
79+
checkStatus.links = links
80+
checkStatus.checkStatus = checkResult.hasFailures ? CheckStatus.FAILED : CheckStatus.SUCCESSFUL
81+
checkStatus.titleString = formatCheckTitle(checkStatus.checkStatus, checkResult, {
7582
includeSourceFile: false,
7683
})
7784
}

Diff for: ‎packages/cli/src/reporters/ci.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ import indentString from 'indent-string'
22

33
import AbstractListReporter from './abstract-list'
44
import { formatCheckTitle, formatCheckResult, CheckStatus, printLn } from './util'
5-
import { CheckRunId } from '../services/abstract-check-runner'
5+
import { SequenceId } from '../services/abstract-check-runner'
66

77
export default class CiReporter extends AbstractListReporter {
8-
onBegin (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string) {
8+
onBegin (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId?: string) {
99
super.onBegin(checks, testSessionId)
1010
printLn(`Running ${this.numChecks} checks in ${this._runLocationString()}:`, 2, 1)
1111
this._printSummary({ skipCheckCount: true })
@@ -17,8 +17,8 @@ export default class CiReporter extends AbstractListReporter {
1717
this._printTestSessionsUrl()
1818
}
1919

20-
onCheckEnd (checkRunId: CheckRunId, checkResult: any) {
21-
super.onCheckEnd(checkRunId, checkResult)
20+
onCheckEnd (sequenceId: SequenceId, checkResult: any) {
21+
super.onCheckEnd(sequenceId, checkResult)
2222
printLn(formatCheckTitle(checkResult.hasFailures ? CheckStatus.FAILED : CheckStatus.SUCCESSFUL, checkResult))
2323

2424
if (this.verbose || checkResult.hasFailures) {

Diff for: ‎packages/cli/src/reporters/dot.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import chalk from 'chalk'
22
import AbstractListReporter from './abstract-list'
3-
import { CheckRunId } from '../services/abstract-check-runner'
3+
import { SequenceId } from '../services/abstract-check-runner'
44
import { print, printLn } from './util'
55

66
export default class DotReporter extends AbstractListReporter {
7-
onBegin (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string) {
7+
onBegin (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId?: string) {
88
super.onBegin(checks, testSessionId)
99
printLn(`Running ${this.numChecks} checks in ${this._runLocationString()}.`, 2, 1)
1010
}
@@ -14,8 +14,8 @@ export default class DotReporter extends AbstractListReporter {
1414
this._printTestSessionsUrl()
1515
}
1616

17-
onCheckEnd (checkRunId: CheckRunId, checkResult: any) {
18-
super.onCheckEnd(checkRunId, checkResult)
17+
onCheckEnd (sequenceId: SequenceId, checkResult: any) {
18+
super.onCheckEnd(sequenceId, checkResult)
1919
if (checkResult.hasFailures) {
2020
print(`${chalk.red('F')}`)
2121
} else {

Diff for: ‎packages/cli/src/reporters/github.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import * as fs from 'fs'
22
import * as path from 'path'
33

44
import AbstractListReporter, { checkFilesMap } from './abstract-list'
5-
import { CheckRunId } from '../services/abstract-check-runner'
5+
import { SequenceId } from '../services/abstract-check-runner'
66
import { formatDuration, printLn, getTestSessionUrl } from './util'
77

88
const outputFile = './checkly-github-report.md'
@@ -63,7 +63,7 @@ export class GithubMdBuilder {
6363
}
6464

6565
for (const [_, checkMap] of this.checkFilesMap.entries()) {
66-
for (const [_, { result, testResultId }] of checkMap.entries()) {
66+
for (const [_, { result, links }] of checkMap.entries()) {
6767
const tableRow: Array<string> = [
6868
`${result.hasFailures ? '❌ Fail' : '✅ Pass'}`,
6969
`${result.name}`,
@@ -72,8 +72,8 @@ export class GithubMdBuilder {
7272
`${formatDuration(result.responseTime)} `,
7373
].filter(nonNullable)
7474

75-
if (this.testSessionId && testResultId) {
76-
const linkColumn = `[Full test report](${getTestSessionUrl(this.testSessionId)}/results/${testResultId})`
75+
if (links?.testResultLink) {
76+
const linkColumn = `[Full test report](${links?.testResultLink})`
7777
tableRow.push(linkColumn)
7878
}
7979

@@ -96,7 +96,7 @@ export class GithubMdBuilder {
9696
}
9797

9898
export default class GithubReporter extends AbstractListReporter {
99-
onBegin (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string) {
99+
onBegin (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId?: string) {
100100
super.onBegin(checks, testSessionId)
101101
printLn(`Running ${this.numChecks} checks in ${this._runLocationString()}.`, 2, 1)
102102
}

0 commit comments

Comments
 (0)