feat: experimental WebGPU renderer (#652)

Summary:

Experimental WebGPU renderer. Currently only supports the image layer.
This PR replaces the renderer in first 3 examples to use the WebGPU
renderer instead of WebGL.

Author: shlomnissan

examples/ca_wave_dynamics/main.ts

@@ -71,7 +71,8 @@ const timePointOverlay = {
   },
 };
 
-new Idetik({
+const idetik = await Idetik.create({
+  renderer: "webgpu-experimental",
   canvas: document.querySelector<HTMLCanvasElement>("#canvas")!,
   viewports: [
     {
@@ -82,7 +83,9 @@ new Idetik({
   ],
   overlays: [timePointOverlay],
   showStats: true,
-}).start();
+});
+
+idetik.start();
 
 const controls = {
   showWireframes: imageLayer.debugMode,

examples/chunk_streaming/main.ts

@@ -75,7 +75,8 @@ const scaleBar = new ScaleBar({
   unit: "μm",
 });
 
-new Idetik({
+const idetik = await Idetik.create({
+  renderer: "webgpu-experimental",
   canvas: document.querySelector<HTMLCanvasElement>("#canvas")!,
   viewports: [
     {
@@ -86,7 +87,9 @@ new Idetik({
   ],
   overlays: [timePointOverlay, scaleBar],
   showStats: true,
-}).start();
+});
+
+idetik.start();
 
 const controls = {
   showWireframes: imageLayer.debugMode,

examples/image2d_from_omezarr4d_hcs/main.ts

@@ -42,17 +42,21 @@ wellPaths.forEach((path) => {
 const policy = createNoPrefetchPolicy();
 
 const camera = new OrthographicCamera(0, xSize * xScale, 0, ySize * yScale);
-const app = new Idetik({
+
+const idetik = await Idetik.create({
+  renderer: "webgpu-experimental",
   canvas: document.querySelector<HTMLCanvasElement>("canvas")!,
   viewports: [
     {
       camera,
       cameraControls: new PanZoomControls(camera),
     },
   ],
-}).start();
+});
+
+idetik.start();
 
-const viewport = app.viewports[0];
+const viewport = idetik.viewports[0];
 const imageSelector = document.querySelector("#image") as HTMLSelectElement;
 
 const onImageChange = async () => {

generate-examples-manifest.js

@@ -1,4 +1,3 @@
-/* eslint-env node */
 /* global process */
 
 import { readdirSync, existsSync, readFileSync, writeFileSync } from 'node:fs';

package-lock.json

@@ -47,6 +47,7 @@
   "dependencies": {
     "gl-matrix": "^3.4.3",
     "zarrita": "^0.7.1",
+    "webgpu-utils": "^2.1.0",
     "zod": "^3.24.1"
   },
   "devDependencies": {
@@ -58,11 +59,12 @@
     "@typescript-eslint/parser": "^8.1.0",
     "@vitest/browser": "^3.0.9",
     "@vitest/coverage-istanbul": "^3.0.9",
+    "@webgpu/types": "^0.1.69",
     "conventional-changelog-conventionalcommits": "^9.1.0",
     "eslint": "^9.9.0",
     "globals": "^15.9.0",
-    "json-schema-to-zod": "^2.6.1",
     "jsdom": "^24.1.1",
+    "json-schema-to-zod": "^2.6.1",
     "lil-gui": "^0.20.0",
     "playwright": "1.56.1",
     "prettier": "^3.3.2",

package.json

@@ -6,7 +6,8 @@ import { EventContext } from "./event_dispatcher";
 import { Viewport } from "./viewport";
 
 export type LayerState = "initialized" | "loading" | "ready";
-export type blendMode =
+export type BlendMode =
+  | "none"
   | "normal"
   | "additive"
   | "subtractive"
@@ -21,7 +22,7 @@ type StateChangeCallback = (
 export interface LayerOptions {
   transparent?: boolean;
   opacity?: number;
-  blendMode?: blendMode;
+  blendMode?: BlendMode;
 }
 
 export type RenderContext = {
@@ -37,7 +38,7 @@ export abstract class Layer {
 
   public transparent: boolean;
   private opacity_: number;
-  public blendMode: blendMode;
+  public blendMode: BlendMode;
 
   constructor({
     transparent = false,

src/core/layer.ts

@@ -23,6 +23,8 @@ export abstract class Renderer {
     this.updateRendererSize();
   }
 
+  public beginFrame(): void {}
+
   public abstract render(viewport: Viewport): void;
 
   public updateSize(): void {

src/core/renderer.ts

@@ -1,4 +1,5 @@
 import { WebGLRenderer } from "./renderers/webgl_renderer";
+import { createWebGPURenderer } from "./renderers/webgpu/webgpu_renderer";
 import { Logger } from "./utilities/logger";
 import { ChunkManager } from "./core/chunk_manager";
 import { Renderer } from "./core/renderer";
@@ -87,10 +88,20 @@ export class Idetik {
    *
    * @throws {Error} If viewports have duplicate IDs or shared elements
    */
-  constructor(params: IdetikParams) {
+  static async create(
+    params: IdetikParams & { renderer?: "webgl" | "webgpu-experimental" }
+  ): Promise<Idetik> {
+    const renderer =
+      params.renderer === "webgpu-experimental"
+        ? await createWebGPURenderer(params.canvas)
+        : new WebGLRenderer(params.canvas);
+    return new Idetik(params, renderer);
+  }
+
+  constructor(params: IdetikParams, renderer?: Renderer) {
     this.canvas = params.canvas;
 
-    this.renderer_ = new WebGLRenderer(this.canvas);
+    this.renderer_ = renderer ?? new WebGLRenderer(this.canvas);
     this.chunkManager_ = new ChunkManager();
     this.context_ = {
       chunkManager: this.chunkManager_,
@@ -114,6 +125,7 @@ export class Idetik {
     }
     this.sizeObserver_ = new PixelSizeObserver(sizeDependents, () => {
       this.renderer_.updateSize();
+      this.renderer_.beginFrame();
       for (const viewport of this.viewports_) {
         viewport.updateSize();
         this.renderer_.render(viewport);
@@ -219,6 +231,7 @@ export class Idetik {
 
     this.lastTimestamp_ = timestamp;
 
+    this.renderer_.beginFrame();
     for (const viewport of this.viewports_) {
       viewport.cameraControls?.onUpdate(dt);
       this.renderer_.render(viewport);

src/idetik.ts

@@ -53,6 +53,32 @@ export function bufferToDataType(
   throw new Error("Unsupported buffer type.");
 }
 
+export function textureChannelCount(texture: Texture) {
+  switch (texture.dataFormat) {
+    case "scalar":
+      return 1;
+    case "rgb":
+      return 3;
+    case "rgba":
+      return 4;
+  }
+}
+
+export function textureBytesPerChannel(texture: Texture) {
+  switch (texture.dataType) {
+    case "byte":
+    case "unsigned_byte":
+      return 1;
+    case "short":
+    case "unsigned_short":
+      return 2;
+    case "int":
+    case "unsigned_int":
+    case "float":
+      return 4;
+  }
+}
+
 export function textureDefaultValueRange(texture: Texture): [number, number] {
   if (texture.dataFormat === "rgb" || texture.dataFormat === "rgba") {
     return [0, 1];