Skip to content

Commit fc44052

Browse files
vitalicsvitalics
authored
move test step metrics into onTestStepEnd hook (#32)
* move test step metrics into `onTestStepEnd` hook * update readme file
1 parent 65ad6cd commit fc44052

File tree

4 files changed

+68
-60
lines changed

4 files changed

+68
-60
lines changed

‎.changeset/fuzzy-camels-kick.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"playwright-prometheus-remote-write-reporter": patch
3+
---
4+
5+
- Move test step collection metrics into `onTestStepEnd` hook
6+
- Update readme about metrics name

‎README.md‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,16 @@ bun a playwright-prometheus-remote-write-reporter # bun
3030
In your `playwright.config.ts` add next lines:
3131

3232
```ts
33-
import { defineConfig } from '@playwright/test'
34-
import { type PrometheusOptions } from 'playwright-prometheus-remote-write-reporter'
33+
import { defineConfig } from "@playwright/test";
34+
import { type PrometheusOptions } from "playwright-prometheus-remote-write-reporter";
3535
3636
export default defineConfig({
3737
// ...
3838
reporter: [
3939
[
4040
"playwright-prometheus-remote-write-reporter",
4141
{
42-
serverUrl: 'http://localhost:9090/api/v1/write' // same url as declared in precondition
42+
serverUrl: "http://localhost:9090/api/v1/write", // same url as declared in precondition
4343
} satisfies PrometheusOptions, // for autocomplete
4444
],
4545
],
@@ -77,20 +77,31 @@ this metrics below sends periodically and you may found when they sends
7777
| project | playwright project object. E.g. chromium, firefox | onExit |
7878
| test | test object | onTestEnd |
7979
| test_attachment | test attachment information | onTestEnd |
80+
| tests_total_attachment_size | size (in bytes) about attachments across whole run | onTestEnd |
81+
| test_attachment_count | count of attachments across whole run | onTestEnd |
8082
| test_attachment_size | attachment size in bytes | onTestEnd |
81-
| test_annnotation | annotations for 1 test | onTestEnd |
83+
| test_annotation | annotations for 1 test | onTestEnd |
84+
| test_step_total_count | count of test steps across whole run | onTestEnd |
85+
| test_step_total_duration | duration(in ms) how long test steps has been executed | onTestEnd |
86+
| test_annotation_count | count of annotations across all tests | onTestEnd |
8287
| test_error | test errors information | onTestEnd |
8388
| test_duration | test duration in milliseconds | onTestEnd |
8489
| test_retry_count | count of retries for 1 test | onTestEnd |
90+
| timed_out_tests_count | count of tests with `timedOut` status | onTestEnd |
91+
| test_step_total_error | Count of errors in all test steps | onTestEnd |
92+
| test_step_duration | duration of test step | onTestStepEnd |
93+
| test_step_error_count | Count of errors in test steps | onTestStepEnd |
94+
| test_step | Individual test steps information | onTestStepEnd |
8595
| tests_attachment_total_size | total attachment size in bytes for all tests | onExit |
8696
| tests_total_duration | time for all tests | onExit |
8797
| tests_total_count | total count of all tests | onExit |
88-
| tests_skip_count | count of all skipped tests | onExit |
89-
| tests_pass_count | count of all passed tests | onExit |
90-
| tests_fail_count | count of all failed tests | onExit |
91-
| tests_timed_out_count | count of all timedOut tests | onExit |
98+
| passed_count | count of all passed tests | onExit |
99+
| timed_out_tests_count | count of all tests with `timedOut` status | onExit |
100+
| skipped_tests_count | count of all skipped tests | onExit |
101+
| passed_count | count of all tests with `passed` status | onExit |
102+
| failed_count | count of all failed tests | onExit |
92103
| tests_attachment_total_count | count of attachments across all tests | onExit |
93-
| error_count | count of errors | onError |
104+
| errors_count | count of errors across all tests | onError |
94105
| stdout | stdout for test. Reporter logs have label: `internal="true"` | onStdOut |
95106
| stderr | stdout for test. Reporter logs have label: `internal="true"` | onStdErr |
96107

@@ -141,7 +152,7 @@ const countOfUrlCalls = new Counter(
141152
// only name is required
142153
name: "url_open", // will automatically appends prefix
143154
},
144-
0
155+
0,
145156
); // starts from 0
146157
147158
test("simple counter test", async ({ page }) => {

‎result‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎src/index.ts‎

Lines changed: 41 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ export default class PrometheusReporter implements Reporter {
131131
name: "test_step_total_error",
132132
description: `Total errors in test steps`,
133133
});
134-
private readonly test_step = new Counter({
134+
private test_step = new Counter({
135135
name: "test_step",
136136
description: `Individual test steps`,
137137
});
@@ -168,7 +168,7 @@ export default class PrometheusReporter implements Reporter {
168168
name: "error_count",
169169
});
170170
private test_errors = new Counter({
171-
name: "test_error",
171+
name: "test_errors",
172172
});
173173
private tests_total_attachment = new Counter({
174174
name: "tests_attachment_total_count",
@@ -398,22 +398,55 @@ export default defineConfig({
398398
}
399399

400400
async onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
401-
this.test_step.labels({
402-
path: this.location(step),
401+
const location = this.location(step);
402+
const labels: Record<string, string> = {
403403
category: step.category,
404+
path: location,
404405
testId: test.id,
406+
duration: String(step.duration),
407+
startTime: String(step.startTime),
408+
titlePath: step.titlePath().join("->"),
409+
stepsCount: String(step.steps.length),
405410
testTitle: test.title,
406-
stepTitle: step.title,
407411
stepInnerCount: String(step.steps.length),
408-
startTime: String(step.startTime),
409-
});
412+
parallelIndex: String(result.parallelIndex),
413+
retryCount: String(result.retry),
414+
status: String(result.status),
415+
};
416+
if (step.error) {
417+
const errorMessage =
418+
step.error.message?.replace(/\r?\n|\r/g, " ") ??
419+
// [TODO] trim message?
420+
JSON.stringify(step.error.message);
421+
labels.errorMessage = errorMessage;
422+
labels.errorSnippet = String(step.error.snippet ?? "<unknown snippet>");
423+
labels.errorStack = String(step.error.stack ?? "<empty stack>");
424+
labels.errorValue = String(step.error.value ?? "<unknown value>");
425+
labels.errorPath = this.location(step.error);
426+
427+
this.test_step_total_error.labels(labels).inc();
428+
this.test_step_error_count.labels(labels).inc();
429+
}
430+
431+
this.test_step_total_duration.inc(step.duration);
432+
this.test_step_total_count.inc();
433+
this.test_step_duration.labels(labels).inc(step.duration);
434+
435+
this.test_step.labels(labels);
436+
410437
this.test_step_total_duration
411438
.labels({
412439
testId: test.id,
413440
testTitle: test.title,
414441
})
415442
.inc(step.duration);
416443
this.updateNodejsStats();
444+
await this.send(
445+
[this.test_step_duration, this.test_step_error_count, this.test_step].map(
446+
(m) => this.mapTimeseries(m),
447+
),
448+
);
449+
this.test_step.reset();
417450
}
418451

419452
async onTestEnd(test: TestCase, result: TestResult) {
@@ -472,47 +505,6 @@ export default defineConfig({
472505
const testDuration = this.test_duration.labels(labels).set(result.duration);
473506
const testRetries = this.test_retry_count.labels(labels).inc(result.retry);
474507

475-
for (const step of result.steps) {
476-
this.test_step_duration.reset();
477-
this.test_step_error_count.reset();
478-
479-
const location = this.location(step);
480-
const labels: Record<string, string> = {
481-
category: step.category,
482-
path: location,
483-
testId: test.id,
484-
duration: String(step.duration),
485-
startTime: String(step.startTime),
486-
titlePath: step.titlePath().join("->"),
487-
stepsCount: String(step.steps.length),
488-
};
489-
490-
if (step.error) {
491-
const errorMessage =
492-
step.error.message?.replace(/\r?\n|\r/g, " ") ??
493-
// [TODO] trim message?
494-
JSON.stringify(step.error.message);
495-
labels.errorMessage = errorMessage;
496-
labels.errorSnippet = String(step.error.snippet ?? "<unknown snippet>");
497-
labels.errorStack = String(step.error.stack ?? "<empty stack>");
498-
labels.errorValue = String(step.error.value ?? "<unknown value>");
499-
labels.errorPath = this.location(step.error);
500-
501-
this.test_step_total_error.labels(labels).inc();
502-
this.test_step_error_count.labels(labels).inc();
503-
}
504-
505-
this.test_step_total_duration.inc(step.duration);
506-
this.test_step_total_count.inc();
507-
this.test_step_duration.labels(labels).inc(step.duration);
508-
509-
await this.send(
510-
[this.test_step_duration, this.test_step_error_count].map((m) =>
511-
this.mapTimeseries(m),
512-
),
513-
);
514-
}
515-
516508
await this.send([
517509
this.mapTimeseries(this.test_step),
518510
this.mapTimeseries(this.test_step_total_duration),
@@ -526,7 +518,7 @@ export default defineConfig({
526518
this.mapTimeseries(this.test_step_total_count),
527519
]);
528520

529-
this.test_step.reset();
521+
this.test_step = this.test_step.reset();
530522
this.test_annotation_count.reset();
531523
this.test = this.test.reset();
532524
this.test_duration = this.test_duration.reset();

0 commit comments

Comments
 (0)