fix(volume_rendering): Volume rendering fixes for max projection - picking, blending, and performance (#584)

* fix: ensure picking value from max projection is < 1

* fix: better blending of multiple VR layers with max enabled

* fix: clarify comment

* fix: userIntensity always overwrites default

* fix: no longer double draw opaque layers

* chore: remove log to check fix

* refactor: remove non-functioning continue statement

* feat: emit a single ID for picking in VR layer

* refactor: rename uniform to uPickId

* feat: register VR layer to get pick ID

* refactor: remove unused demo code around pick ID

* refactor: clarify role of copy shaders in perspective panel

* fix: don't copy over the max projection picking buffer for each max layer
Do one copy if there are any max layers instead

* refactor: clarify buffer names

* fix: VR tool from Python

Author: seankmartin

python/neuroglancer/__init__.py

@@ -49,7 +49,7 @@
     PlaceEllipsoidTool,  # noqa: F401
     BlendTool,  # noqa: F401
     OpacityTool,  # noqa: F401
-    VolumeRenderingModeTool,  # noqa: F401
+    VolumeRenderingTool,  # noqa: F401
     VolumeRenderingGainTool,  # noqa: F401
     VolumeRenderingDepthSamplesTool,  # noqa: F401
     CrossSectionRenderScaleTool,  # noqa: F401

python/neuroglancer/viewer_state.py

@@ -165,9 +165,9 @@ class OpacityTool(Tool):
 
 
 @export_tool
-class VolumeRenderingModeTool(Tool):
+class VolumeRenderingTool(Tool):
     __slots__ = ()
-    TOOL_TYPE = "volumeRenderingMode"
+    TOOL_TYPE = "volumeRendering"
 
 
 @export_tool

src/perspective_view/panel.ts

@@ -153,21 +153,26 @@ export function perspectivePanelEmitOIT(builder: ShaderBuilder) {
 }
 
 export function maxProjectionEmit(builder: ShaderBuilder) {
-  builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
-  builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1);
-  builder.addOutputBuffer("highp vec4", "v4f_fragData2", 2);
+  builder.addOutputBuffer("vec4", "out_color", 0);
+  builder.addOutputBuffer("highp vec4", "out_z", 1);
+  builder.addOutputBuffer("highp vec4", "out_intensity", 2);
+  builder.addOutputBuffer("highp vec4", "out_pickId", 3);
   builder.addFragmentCode(`
-void emit(vec4 color, float depth, float pick) {
-  v4f_fragData0 = color;
-  v4f_fragData1 = vec4(1.0 - depth, 1.0 - depth, 1.0 - depth, 1.0);
-  v4f_fragData2 = vec4(pick, pick, pick, 1.0);
+void emit(vec4 color, float depth, float intensity, highp uint pickId) {
+  float pickIdFloat = float(pickId);
+  float bufferDepth = 1.0 - depth;
+  out_color = color;
+  out_z = vec4(bufferDepth, bufferDepth, bufferDepth, 1.0);
+  out_intensity = vec4(intensity, intensity, intensity, 1.0);
+  out_pickId = vec4(pickIdFloat, pickIdFloat, pickIdFloat, 1.0);
 }`);
 }
 
 const tempVec3 = vec3.create();
 const tempVec4 = vec4.create();
 const tempMat4 = mat4.create();
 
+// Copy the OIT values to the main color buffer
 function defineTransparencyCopyShader(builder: ShaderBuilder) {
   builder.addOutputBuffer("vec4", "v4f_fragColor", null);
   builder.setFragmentMain(`
@@ -180,31 +185,45 @@ v4f_fragColor = vec4(accum.rgb / accum.a, revealage);
 `);
 }
 
+// Copy the max projection color to the OIT buffer
 function defineMaxProjectionColorCopyShader(builder: ShaderBuilder) {
-  builder.addOutputBuffer("vec4", "v4f_fragColor", null);
+  builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
+  builder.addOutputBuffer("vec4", "v4f_fragData1", 1);
+  builder.addFragmentCode(glsl_perspectivePanelEmitOIT);
   builder.setFragmentMain(`
-v4f_fragColor = getValue0();
+vec4 color = getValue0();
+float bufferDepth = getValue1().r;
+float weight = computeOITWeight(color.a, 1.0 - bufferDepth);
+vec4 accum = color * weight;
+float revealage = color.a;
+
+emitAccumAndRevealage(accum, revealage, 0u);
 `);
 }
 
+// Copy the max projection depth and pick values to the main buffer
 function defineMaxProjectionPickCopyShader(builder: ShaderBuilder) {
-  builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
-  builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1);
-  builder.addOutputBuffer("highp vec4", "v4f_fragData2", 2);
+  builder.addOutputBuffer("vec4", "out_color", 0);
+  builder.addOutputBuffer("highp vec4", "out_z", 1);
+  builder.addOutputBuffer("highp vec4", "out_pickId", 2);
   builder.setFragmentMain(`
-v4f_fragData0 = vec4(0.0);
-v4f_fragData1 = getValue0();
-v4f_fragData2 = getValue1();
+out_color = vec4(0.0);
+out_z = getValue0();
+out_pickId = getValue1();
 `);
 }
 
+// Copy the max projection depth and picking to the max projection pick buffer.
+// Note that the depth is set as the intensity value from the render layer.
+// This is to combine max projection picking data via depth testing
+// on the maximum intensity value of the data.
 function defineMaxProjectionToPickCopyShader(builder: ShaderBuilder) {
-  builder.addOutputBuffer("highp vec4", "v4f_fragData0", 0);
-  builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1);
+  builder.addOutputBuffer("highp vec4", "out_z", 0);
+  builder.addOutputBuffer("highp vec4", "out_pickId", 1);
   builder.setFragmentMain(`
-v4f_fragData0 = getValue0();
-v4f_fragData1 = getValue1();
-gl_FragDepth = v4f_fragData1.r;
+out_z = getValue0();
+out_pickId = getValue2();
+gl_FragDepth = getValue1().r;
 `);
 }
 
@@ -306,13 +325,13 @@ export class PerspectivePanel extends RenderedDataPanel {
     OffscreenCopyHelper.get(this.gl, defineTransparencyCopyShader, 2),
   );
   protected maxProjectionColorCopyHelper = this.registerDisposer(
-    OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 1),
+    OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 2),
   );
   protected maxProjectionPickCopyHelper = this.registerDisposer(
     OffscreenCopyHelper.get(this.gl, defineMaxProjectionPickCopyShader, 2),
   );
   protected maxProjectionToPickCopyHelper = this.registerDisposer(
-    OffscreenCopyHelper.get(this.gl, defineMaxProjectionToPickCopyShader, 2),
+    OffscreenCopyHelper.get(this.gl, defineMaxProjectionToPickCopyShader, 3),
   );
 
   private sharedObject: PerspectiveViewState;
@@ -730,6 +749,12 @@ export class PerspectivePanel extends RenderedDataPanel {
                 WebGL2RenderingContext.RED,
                 WebGL2RenderingContext.FLOAT,
               ),
+              new TextureBuffer(
+                this.gl,
+                WebGL2RenderingContext.R32F,
+                WebGL2RenderingContext.RED,
+                WebGL2RenderingContext.FLOAT,
+              ),
             ],
             depthBuffer: new DepthStencilRenderbuffer(this.gl),
           }),
@@ -1004,6 +1029,7 @@ export class PerspectivePanel extends RenderedDataPanel {
           renderContext.depthBufferTexture =
             this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture;
         }
+        // Draw max projection layers
         if (
           renderLayer.isVolumeRendering &&
           isProjectionLayer(renderLayer as VolumeRenderingRenderLayer)
@@ -1021,21 +1047,26 @@ export class PerspectivePanel extends RenderedDataPanel {
           bindMaxProjectionPickingBuffer();
           this.maxProjectionToPickCopyHelper.draw(
             this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
-            this.maxProjectionConfiguration.colorBuffers[2 /*pick*/].texture,
+            this.maxProjectionConfiguration.colorBuffers[2 /*intensity*/]
+              .texture,
+            this.maxProjectionConfiguration.colorBuffers[3 /*pick*/].texture,
           );
 
-          // Copy max projection color result to color only buffer
+          // Copy max projection color result to the transparent buffer with OIT
           // Depth testing off to combine max layers into one color via blend
-          this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR);
+          renderContext.bindFramebuffer();
           gl.depthMask(false);
           gl.disable(WebGL2RenderingContext.DEPTH_TEST);
           gl.enable(WebGL2RenderingContext.BLEND);
-          gl.blendFunc(
+          gl.blendFuncSeparate(
+            WebGL2RenderingContext.ONE,
             WebGL2RenderingContext.ONE,
+            WebGL2RenderingContext.ZERO,
             WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
           );
           this.maxProjectionColorCopyHelper.draw(
             this.maxProjectionConfiguration.colorBuffers[0 /*color*/].texture,
+            this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
           );
 
           // Reset the max projection buffer
@@ -1052,19 +1083,15 @@ export class PerspectivePanel extends RenderedDataPanel {
           gl.clearDepth(1.0);
           gl.clearColor(0.0, 0.0, 0.0, 1.0);
           gl.depthMask(false);
-          gl.blendFuncSeparate(
-            WebGL2RenderingContext.ONE,
-            WebGL2RenderingContext.ONE,
-            WebGL2RenderingContext.ZERO,
-            WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
-          );
           gl.enable(WebGL2RenderingContext.DEPTH_TEST);
           gl.depthFunc(WebGL2RenderingContext.LESS);
           renderContext.emitter = perspectivePanelEmitOIT;
           renderContext.bindFramebuffer();
-          continue;
         }
-        renderLayer.draw(renderContext, attachment);
+        // Draw regular transparent layers
+        else if (renderLayer.isTransparent) {
+          renderLayer.draw(renderContext, attachment);
+        }
       }
       // Copy transparent rendering result back to primary buffer.
       gl.disable(WebGL2RenderingContext.DEPTH_TEST);
@@ -1109,27 +1136,21 @@ export class PerspectivePanel extends RenderedDataPanel {
         /*dppass=*/ WebGL2RenderingContext.REPLACE,
       );
       gl.stencilMask(2);
+      if (hasMaxProjection) {
+        this.maxProjectionPickCopyHelper.draw(
+          this.maxProjectionPickConfiguration.colorBuffers[0].texture /*depth*/,
+          this.maxProjectionPickConfiguration.colorBuffers[1].texture /*pick*/,
+        );
+      }
       for (const [renderLayer, attachment] of visibleLayers) {
-        if (!renderLayer.isTransparent || !renderLayer.transparentPickEnabled) {
+        if (
+          !renderLayer.isTransparent ||
+          !renderLayer.transparentPickEnabled ||
+          renderLayer.isVolumeRendering
+        ) {
+          // Skip non-transparent layers and transparent layers with transparentPickEnabled=false.
+          // Volume rendering layers are handled separately and are combined in a pick buffer
           continue;
-        }
-        // For max projection layers, can copy over the pick buffer directly.
-        if (renderLayer.isVolumeRendering) {
-          if (isProjectionLayer(renderLayer as VolumeRenderingRenderLayer)) {
-            this.maxProjectionPickCopyHelper.draw(
-              this.maxProjectionPickConfiguration.colorBuffers[0]
-                .texture /*depth*/,
-              this.maxProjectionPickConfiguration.colorBuffers[1]
-                .texture /*pick*/,
-            );
-          }
-          // Draw picking for non min/max volume rendering layers
-          else {
-            // Currently volume rendering layers have no picking support
-            // Outside of min/max mode
-            continue;
-          }
-          // other transparent layers are drawn as usual
         } else {
           renderLayer.draw(renderContext, attachment);
         }

src/volume_rendering/volume_render_layer.ts

@@ -254,16 +254,18 @@ void emitIntensity(float value) {
 float savedDepth = 0.0;
 float savedIntensity = 0.0;
 vec4 newColor = vec4(0.0);
+float userEmittedIntensity = -100.0;
 `);
             glsl_emitIntensity = `
 float convertIntensity(float value) {
   return clamp(${glsl_intensityConversion}, 0.0, 1.0);
 }
 void emitIntensity(float value) {
-  defaultMaxProjectionIntensity = value;
+  userEmittedIntensity = value;
 }
 float getIntensity() {
-  return convertIntensity(defaultMaxProjectionIntensity);
+  float intensity = userEmittedIntensity > -100.0 ? userEmittedIntensity : defaultMaxProjectionIntensity;
+  return convertIntensity(intensity);
 }
 `;
             glsl_rgbaEmit = `
@@ -281,8 +283,9 @@ void emitRGBA(vec4 rgba) {
   savedIntensity = intensityChanged ? newIntensity : savedIntensity; 
   savedDepth = intensityChanged ? depthAtRayPosition : savedDepth;
   outputColor = intensityChanged ? newColor : outputColor;
-  emit(outputColor, savedDepth, savedIntensity);
+  emit(outputColor, savedDepth, savedIntensity, uPickId);
   defaultMaxProjectionIntensity = 0.0;
+  userEmittedIntensity = -100.0;
 `;
           }
           emitter(builder);
@@ -308,6 +311,7 @@ void emitRGBA(vec4 rgba) {
 
           builder.addUniform("highp float", "uBrightnessFactor");
           builder.addUniform("highp float", "uGain");
+          builder.addUniform("highp uint", "uPickId");
           builder.addVarying("highp vec4", "vNormalizedPosition");
           builder.addTextureSampler(
             "sampler2D",
@@ -365,7 +369,7 @@ vec2 computeUVFromClipSpace(vec4 clipSpacePosition) {
 `;
             if (isProjectionMode(shaderParametersState.mode)) {
               glsl_emitWireframe = `
-  emit(outputColor, 1.0, uChunkNumber);
+  emit(outputColor, 1.0, uChunkNumber, uPickId);
             `;
             }
             builder.setFragmentMainFunction(`
@@ -621,6 +625,9 @@ void main() {
     gl.enable(WebGL2RenderingContext.CULL_FACE);
     gl.cullFace(WebGL2RenderingContext.FRONT);
 
+    const pickId = isProjectionMode(this.mode.value)
+      ? renderContext.pickIDs.register(this)
+      : 0;
     forEachVisibleVolumeRenderingChunk(
       renderContext.projectionParameters,
       this.localPosition.value,
@@ -798,6 +805,7 @@ void main() {
           }
           newSource = false;
           gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition);
+          gl.uniform1ui(shader.uniform("uPickId"), pickId);
           drawBoxes(gl, 1, 1);
           ++presentCount;
         } else {