Better async and sync substep support

resources/benchmark-runner.mjs

@@ -377,7 +377,7 @@ export class BenchmarkRunner {
     }
 
     async _prepareAllSuites() {
-        this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN };
+        this._measuredValues = { steps: {}, total: 0, mean: NaN, geomean: NaN, score: NaN };
         await this._wakeLock.request();
 
         const prepareStartLabel = "runner-prepare-start";
@@ -449,8 +449,8 @@ export class BenchmarkRunner {
         if (this._client?.didRunSuites) {
             let product = 1;
             const values = [];
-            for (const suiteName in this._measuredValues.tests) {
-                const suiteTotal = this._measuredValues.tests[suiteName].total;
+            for (const suiteName in this._measuredValues.steps) {
+                const suiteTotal = this._measuredValues.steps[suiteName].total;
                 product *= suiteTotal;
                 values.push(suiteTotal);
             }
@@ -482,15 +482,15 @@ export class BenchmarkRunner {
                 metric.add(results.total ?? results);
                 if (metric.parent !== parent)
                     parent.addChild(metric);
-                if (results.tests)
-                    collectSubMetrics(`${metric.name}${Metric.separator}`, results.tests, metric);
+                if (results.steps)
+                    collectSubMetrics(`${metric.name}${Metric.separator}`, results.steps, metric);
             }
         };
         const initializeMetrics = this._metrics === null;
         if (initializeMetrics)
             this._metrics = { __proto__: null };
 
-        const iterationResults = this._measuredValues.tests;
+        const iterationResults = this._measuredValues.steps;
         collectSubMetrics("", iterationResults);
 
         if (initializeMetrics) {

resources/litert-js/src/index.mjs

@@ -1,5 +1,4 @@
-import { BenchmarkConnector } from "speedometer-utils/benchmark.mjs";
-import { AsyncBenchmarkStep, AsyncBenchmarkSuite } from "speedometer-utils/benchmark.mjs";
+import { BenchmarkConnector, AsyncBenchmarkStep, AsyncBenchmarkSuite } from "speedometer-utils/benchmark.mjs";
 import { forceLayout } from "speedometer-utils/helpers.mjs";
 import * as tf from '@tensorflow/tfjs';
 import { loadAndCompile, loadLiteRt, Tensor } from '@litertjs/core';
@@ -384,8 +383,8 @@ export async function initializeBenchmark(modelType) {
              forceLayout();
              await benchmark.run();
              forceLayout();
-         }),
-     ], true),
+         }, { measureAsync: false }),
+     ]),
  };
 
  const benchmarkConnector = new BenchmarkConnector(suites, appName, appVersion);

resources/shared/benchmark.mjs

@@ -1,5 +1,5 @@
 /* eslint-disable no-case-declarations */
-import { TestRunner, AsyncTestRunner} from "./test-runner.mjs";
+import { StepRunner, AsyncStepRunner } from "./step-runner.mjs";
 import { Params } from "./params.mjs";
 
 /**
@@ -8,22 +8,23 @@ import { Params } from "./params.mjs";
  * A single test step, with a common interface to interact with.
  */
 export class BenchmarkStep {
-    constructor(name, run) {
+    constructor(name, run, { measureAsync = true } = {}) {
         this.name = name;
         this.run = run;
+        this.measureAsync = measureAsync;
     }
 
-    async runAndRecord(params, suite, test, callback) {
-        const testRunner = new TestRunner(null, null, params, suite, test, callback);
-        const result = await testRunner.runTest();
+    async runAndRecord(params, suite, callback) {
+        const testRunner = new StepRunner(null, null, params, suite, this, callback);
+        const result = await testRunner.runStep();
         return result;
     }
 }
 
 export class AsyncBenchmarkStep extends BenchmarkStep {
-    async runAndRecord(params, suite, test, callback) {
-        const testRunner = new AsyncTestRunner(null, null, params, suite, test, callback);
-        const result = await testRunner.runTest();
+    async runAndRecord(params, suite, callback) {
+        const testRunner = new AsyncStepRunner(null, null, params, suite, this, callback);
+        const result = await testRunner.runStep();
         return result;
     }
 }
@@ -34,25 +35,30 @@ export class AsyncBenchmarkStep extends BenchmarkStep {
  * A single test suite that contains one or more test steps.
  */
 export class BenchmarkSuite {
-    constructor(name, tests, type = "sync") {
+    constructor(name, steps, type = "sync") {
         this.name = name;
-        this.tests = tests;
+        this.steps = steps;
         this.type = type;
     }
 
-    record(_test, syncTime, asyncTime) {
-        const total = syncTime + asyncTime;
+    record(step, syncTime, asyncTime) {
+        let total = syncTime;
+        const tests = {};
+        if (step.measureAsync) {
+            total += asyncTime;
+            tests.Async = asyncTime;
+            test.Sync = syncTime;
+        }
         const results = {
-            tests: { Sync: syncTime, Async: asyncTime },
+            tests,
             total: total,
         };
-
         return results;
     }
 
     async runAndRecord(params, onProgress) {
         const measuredValues = {
-            tests: {},
+            steps: {},
             prepare: 0,
             total: 0,
         };
@@ -61,18 +67,18 @@ export class BenchmarkSuite {
 
         performance.mark(suiteStartLabel);
 
-        for (const test of this.tests) {
-            const result = await test.runAndRecord(params, this, test, this.record);
-            measuredValues.tests[test.name] = result;
+        for (const step of this.steps) {
+            const result = await step.runAndRecord(params, this, this.record);
+            measuredValues.steps[step.name] = result;
             measuredValues.total += result.total;
-            onProgress?.(test.name);
+            onProgress?.(step.name);
         }
 
         performance.mark(suiteEndLabel);
         performance.measure(`suite-${this.name}`, suiteStartLabel, suiteEndLabel);
 
         return {
-            type: "suite-tests-complete",
+            type: "suite-steps-complete",
             status: "success",
             result: measuredValues,
             suiteName: this.name,

resources/shared/test-invoker.mjs → resources/shared/step-invoker.mjs

@@ -1,4 +1,4 @@
-class TestInvoker {
+class StepInvoker {
     constructor(syncCallback, asyncCallback, reportCallback, params) {
         this._syncCallback = syncCallback;
         this._asyncCallback = asyncCallback;
@@ -7,7 +7,7 @@ class TestInvoker {
     }
 }
 
-class BaseRAFTestInvoker extends TestInvoker {
+class BaseRAFStepInvoker extends StepInvoker {
     start() {
         return new Promise((resolve) => {
             if (this._params.waitBeforeSync)
@@ -18,7 +18,7 @@ class BaseRAFTestInvoker extends TestInvoker {
     }
 }
 
-class RAFTestInvoker extends BaseRAFTestInvoker {
+class RAFStepInvoker extends BaseRAFStepInvoker {
     _scheduleCallbacks(resolve) {
         requestAnimationFrame(() => this._syncCallback());
         requestAnimationFrame(() => {
@@ -33,7 +33,7 @@ class RAFTestInvoker extends BaseRAFTestInvoker {
     }
 }
 
-class AsyncRAFTestInvoker extends BaseRAFTestInvoker {
+class AsyncRAFStepInvoker extends BaseRAFStepInvoker {
     static mc = new MessageChannel();
     _scheduleCallbacks(resolve) {
         let gotTimer = false;
@@ -64,7 +64,7 @@ class AsyncRAFTestInvoker extends BaseRAFTestInvoker {
                 tryTriggerAsyncCallback();
             });
 
-            AsyncRAFTestInvoker.mc.port1.addEventListener(
+            AsyncRAFStepInvoker.mc.port1.addEventListener(
                 "message",
                 async function () {
                     await Promise.resolve();
@@ -73,14 +73,14 @@ class AsyncRAFTestInvoker extends BaseRAFTestInvoker {
                 },
                 { once: true }
             );
-            AsyncRAFTestInvoker.mc.port1.start();
-            AsyncRAFTestInvoker.mc.port2.postMessage("speedometer");
+            AsyncRAFStepInvoker.mc.port1.start();
+            AsyncRAFStepInvoker.mc.port2.postMessage("speedometer");
         });
     }
 }
 
-export const TEST_INVOKER_LOOKUP = Object.freeze({
+export const STEP_INVOKER_LOOKUP = Object.freeze({
     __proto__: null,
-    raf: RAFTestInvoker,
-    async: AsyncRAFTestInvoker,
+    raf: RAFStepInvoker,
+    async: AsyncRAFStepInvoker,
 });

resources/shared/test-runner.mjs → resources/shared/step-runner.mjs

@@ -1,17 +1,17 @@
-import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
+import { STEP_INVOKER_LOOKUP } from "./step-invoker.mjs";
 
-export class TestRunner {
+export class StepRunner {
     #frame;
     #page;
     #params;
     #suite;
-    #test;
+    #step;
     #callback;
     #type;
 
-    constructor(frame, page, params, suite, test, callback, type) {
+    constructor(frame, page, params, suite, step, callback, type) {
         this.#suite = suite;
-        this.#test = test;
+        this.#step = step;
         this.#params = params;
         this.#callback = callback;
         this.#page = page;
@@ -23,21 +23,21 @@ export class TestRunner {
         return this.#page;
     }
 
-    get test() {
-        return this.#test;
+    get step() {
+        return this.#step;
     }
 
     _runSyncStep(test, page) {
         test.run(page);
     }
 
-    async runTest() {
+    async runStep() {
         // Prepare all mark labels outside the measuring loop.
         const suiteName = this.#suite.name;
-        const testName = this.#test.name;
-        const syncStartLabel = `${suiteName}.${testName}-start`;
-        const syncEndLabel = `${suiteName}.${testName}-sync-end`;
-        const asyncEndLabel = `${suiteName}.${testName}-async-end`;
+        const stepName = this.#step.name;
+        const syncStartLabel = `${suiteName}.${stepName}-start`;
+        const syncEndLabel = `${suiteName}.${stepName}-sync-end`;
+        const asyncEndLabel = `${suiteName}.${stepName}-async-end`;
 
         let syncTime;
         let asyncStartTime;
@@ -56,9 +56,9 @@ export class TestRunner {
             const syncStartTime = performance.now();
 
             if (this.#type === "async")
-                await this._runSyncStep(this.test, this.page);
+                await this._runSyncStep(this.step, this.page);
             else
-                this._runSyncStep(this.test, this.page);
+                this._runSyncStep(this.step, this.page);
 
             const mark = performance.mark(syncEndLabel);
             const syncEndTime = mark.startTime;
@@ -80,13 +80,13 @@ export class TestRunner {
 
             if (this.#params.warmupBeforeSync)
                 performance.measure("warmup", "warmup-start", "warmup-end");
-            performance.measure(`${suiteName}.${testName}-sync`, syncStartLabel, syncEndLabel);
-            performance.measure(`${suiteName}.${testName}-async`, syncEndLabel, asyncEndLabel);
+            performance.measure(`${suiteName}.${stepName}-sync`, syncStartLabel, syncEndLabel);
+            performance.measure(`${suiteName}.${stepName}-async`, syncEndLabel, asyncEndLabel);
         };
 
-        const report = () => this.#callback(this.#test, syncTime, asyncTime);
+        const report = () => this.#callback(this.#step, syncTime, asyncTime);
         const invokerType = this.invokerType;
-        const invokerClass = TEST_INVOKER_LOOKUP[invokerType];
+        const invokerClass = STEP_INVOKER_LOOKUP[invokerType];
         const invoker = new invokerClass(runSync, measureAsync, report, this.#params);
 
         return invoker.start();
@@ -99,23 +99,23 @@ export class TestRunner {
     }
 }
 
-export class AsyncTestRunner extends TestRunner {
-    constructor(frame, page, params, suite, test, callback, type) {
-        super(frame, page, params, suite, test, callback, type = "async");
+export class AsyncStepRunner extends StepRunner {
+    constructor(frame, page, params, suite, step, callback, type) {
+        super(frame, page, params, suite, step, callback, type = "async");
     }
 
     async _runSyncStep(test, page) {
         await test.run(page);
     }
 }
 
-export class RemoteTestRunner extends TestRunner {
+export class RemoteStepRunner extends StepRunner {
 }
 
 
-export const TEST_RUNNER_LOOKUP = Object.freeze({
+export const STEP_RUNNER_LOOKUP = Object.freeze({
     __proto__: null,
-    default: TestRunner,
-    async: AsyncTestRunner,
-    remote: RemoteTestRunner,
+    default: StepRunner,
+    async: AsyncStepRunner,
+    remote: RemoteStepRunner,
 });