Add github action for testing (#65)

The full test suite is slow, so we only run wasm tests and 2 gpu tests (1 from tflite and transformers).

Enabled for chrome only for now. 

Fixes #60

Author: brendandahl

.github/workflows/test.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+    push:
+        branches: ["main"]
+    pull_request:
+        branches: ["main"]
+
+    # Allows you to run this workflow manually from the Actions tab
+    workflow_dispatch:
+
+jobs:
+    build:
+        name: Build
+        runs-on: macos-latest
+        strategy:
+            matrix:
+                # TODO enable safari and firefox
+                browser: [chrome]
+        steps:
+            - name: Install Firefox
+              if: ${{ matrix.browser == 'firefox' }}
+              run: brew install --cask firefox
+
+            - name: Checkout Branch
+              uses: actions/checkout@v4
+
+            - name: Setup Node
+              uses: actions/setup-node@v4
+              with:
+                  node-version-file: "package.json"
+                  cache: "npm"
+
+            - name: Install Node Packages
+              run: npm ci
+
+            - name: Download Models and Build Benchmarks
+              run: npm run build
+
+            - name: Run Unit Tests
+              run: |
+                  echo "Running in $BROWSER"
+                  npm run test:${{ matrix.browser }}
+
+            - name: Run end2end Tests
+              run: |
+                  echo "Running in $BROWSER"
+                  npm run test-e2e:${{ matrix.browser }}

resources/default-tests.mjs

@@ -22,7 +22,7 @@ export const defaultSuites = [
     {
         name: "Sentence-Similarity-webgpu",
         url: "resources/transformers-js/dist/sentence-similarity-gpu.html",
-        tags: ["default", "sentence-similarity", "webgpu", "transformers-js"],
+        tags: ["default", "sentence-similarity", "webgpu", "transformers-js", "gpu-test-suite"],
         type: "remote",
     },
     {
@@ -118,7 +118,7 @@ export const defaultSuites = [
     {
         name: "Image-Classification-LiteRT.js-webgpu",
         url: "resources/litert-js/dist/image-classification-gpu.html",
-        tags: ["default", "image-classification", "webgpu", "litert-js"],
+        tags: ["default", "image-classification", "webgpu", "litert-js", "gpu-test-suite"],
         type: "remote",
     },
     {

tests/run-end2end.mjs

@@ -9,9 +9,23 @@ This script runs end2end tests by invoking the benchmark via the main
 Speedometer page in /index.html.
 `.trim();
 
+const ONE_MINUTE_IN_MS = 60000;
+
 const { driver, PORT, stop } = await testSetup(HELP);
 
-const suites = benchmarkConfigurator.suites;
+// Running all of the benchmarks is very slow (especially when the GPU is emulated). To run the
+// tests faster we run all of the Wasm benchmarks, and only a few GPU tests to cover most of
+// the common code. To run all benchmarks, enable this.
+const RUN_FULL_SUITE = false;
+let tags = 'wasm,gpu-test-suite';
+let suites = benchmarkConfigurator.suites.filter(suite => suite.tags.some((tag) => tag === 'wasm' || tag === 'gpu-test-suite'));
+let timeout = 5 * ONE_MINUTE_IN_MS;
+
+if (RUN_FULL_SUITE) {
+    tags = 'all';
+    suites = benchmarkConfigurator.suites;
+    timeout = 20 * ONE_MINUTE_IN_MS;
+}
 
 async function testPage(url) {
     console.log(`Testing: ${url}`);
@@ -72,7 +86,7 @@ function validateMetric(name, metric) {
 
 async function testIterations() {
     const iterationCount = 2;
-    const metrics = await testPage(`index.html?iterationCount=${iterationCount}`);
+    const metrics = await testPage(`index.html?iterationCount=${iterationCount}&tags=${tags}`);
     suites.forEach((suite) => {
         if (suite.enabled) {
             const metric = metrics[suite.name];
@@ -87,7 +101,7 @@ async function testIterations() {
 }
 
 async function testAll() {
-    const metrics = await testPage("index.html?iterationCount=1&tags=all");
+    const metrics = await testPage(`index.html?iterationCount=1&tags=${tags}`);
     suites.forEach((suite) => {
         assert(suite.name in metrics);
         const metric = metrics[suite.name];
@@ -98,10 +112,10 @@ async function testAll() {
 }
 
 async function testDeveloperMode() {
-    const params = ["developerMode", "iterationCount=1", "warmupBeforeSync=2", "waitBeforeSync=2", "shuffleSeed=123", "suites=Perf-Dashboard"];
+    const params = ["developerMode", "iterationCount=1", "warmupBeforeSync=2", "waitBeforeSync=2", "shuffleSeed=123", "suites=Image-Classification-LiteRT.js-wasm"];
     const metrics = await testPage(`index.html?${params.join("&")}`);
     suites.forEach((suite) => {
-        if (suite.name === "Perf-Dashboard") {
+        if (suite.name === "Image-Classification-LiteRT.js-wasm") {
             const metric = metrics[suite.name];
             assert(metric.values.length === 1);
         } else {
@@ -112,7 +126,7 @@ async function testDeveloperMode() {
 
 async function test() {
     try {
-        await driver.manage().setTimeouts({ script: 60000 });
+        await driver.manage().setTimeouts({ script: timeout });
         await testIterations();
         await testAll();
         await testDeveloperMode();