Skip to content

Commit 3272b99

Browse files
chrisjchrisj
committed
wip segmentation mesh shader
1 parent 9c5fdd4 commit 3272b99

File tree

9 files changed

+182
-33
lines changed

9 files changed

+182
-33
lines changed

‎src/datasource/graphene/backend.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ async function decodeDracoFragmentChunk(
121121
response: Uint8Array,
122122
) {
123123
const rawMesh = await decodeDraco(response);
124+
// console.log("decodeDracoFragmentChunk", chunk.key, chunk.fragmentId);
124125
assignMeshFragmentData(chunk, rawMesh);
125126
}
126127

‎src/layer/annotation/index.ts‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -857,17 +857,19 @@ class RenderingOptionsTab extends Tab {
857857
),
858858
);
859859

860+
ShaderControls;
861+
860862
element.appendChild(this.codeWidget.element);
861-
element.appendChild(
862-
this.registerDisposer(
863-
new ShaderControls(
864-
layer.annotationDisplayState.shaderControls,
865-
this.layer.manager.root.display,
866-
this.layer,
867-
{ visibility: this.visibility },
868-
),
869-
).element,
870-
);
863+
// element.appendChild(
864+
// this.registerDisposer(
865+
// new ShaderControls(
866+
// layer.annotationDisplayState.shaderControls,
867+
// this.layer.manager.root.display,
868+
// this.layer,
869+
// { visibility: this.visibility },
870+
// ),
871+
// ).element,
872+
// );
871873

872874
element.appendChild(
873875
addLayerControlToOptionsTab(

‎src/layer/segmentation/index.ts‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,9 +129,13 @@ import {
129129
verifyString,
130130
} from "#src/util/json.js";
131131
import { Signal } from "#src/util/signal.js";
132-
import { makeWatchableShaderError } from "#src/webgl/dynamic_shader.js";
132+
import {
133+
makeTrackableFragmentMain,
134+
makeWatchableShaderError,
135+
} from "#src/webgl/dynamic_shader.js";
133136
import type { DependentViewContext } from "#src/widget/dependent_view_widget.js";
134137
import { registerLayerShaderControlsTool } from "#src/widget/shader_controls.js";
138+
import { ShaderControlState } from "#src/webgl/shader_ui_controls.js";
135139

136140
const MAX_LAYER_BAR_UI_INDICATOR_COLORS = 6;
137141

@@ -410,6 +414,12 @@ class LinkedSegmentationGroupState<
410414
}
411415
}
412416

417+
const DEFAULT_FRAGMENT_MAIN = `
418+
void main() {
419+
emit(vColor, uPickID);
420+
}
421+
`;
422+
413423
class SegmentationUserLayerDisplayState implements SegmentationDisplayState {
414424
constructor(public layer: SegmentationUserLayer) {
415425
// Even though `SegmentationUserLayer` assigns this to its `displayState` property, redundantly
@@ -536,6 +546,8 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState {
536546
ignoreNullVisibleSet = new TrackableBoolean(true, true);
537547
skeletonRenderingOptions = new SkeletonRenderingOptions();
538548
shaderError = makeWatchableShaderError();
549+
fragmentMain = makeTrackableFragmentMain(DEFAULT_FRAGMENT_MAIN);
550+
shaderControlState = new ShaderControlState(this.fragmentMain);
539551
renderScaleHistogram = new RenderScaleHistogram();
540552
renderScaleTarget = trackableRenderScaleTarget(1);
541553
selectSegment: (id: bigint, pin: boolean | "toggle") => void;
@@ -689,6 +701,9 @@ export class SegmentationUserLayer extends Base {
689701
this.displayState.linkedSegmentationGroup.changed.add(() =>
690702
this.updateDataSubsourceActivations(),
691703
);
704+
this.displayState.fragmentMain.changed.add(
705+
this.specificationChanged.dispatch,
706+
);
692707
this.tabs.add("rendering", {
693708
label: "Render",
694709
order: -100,
@@ -996,6 +1011,9 @@ export class SegmentationUserLayer extends Base {
9961011
this.displayState.ignoreNullVisibleSet.restoreState(
9971012
specification[json_keys.IGNORE_NULL_VISIBLE_SET_JSON_KEY],
9981013
);
1014+
this.displayState.fragmentMain.restoreState(
1015+
specification[json_keys.SHADER_JSON_KEY],
1016+
);
9991017

10001018
const { skeletonRenderingOptions } = this.displayState;
10011019
skeletonRenderingOptions.restoreState(
@@ -1066,6 +1084,7 @@ export class SegmentationUserLayer extends Base {
10661084
this.displayState.renderScaleTarget.toJSON();
10671085
x[json_keys.CROSS_SECTION_RENDER_SCALE_JSON_KEY] =
10681086
this.sliceViewRenderScaleTarget.toJSON();
1087+
x[json_keys.SHADER_JSON_KEY] = this.displayState.fragmentMain.toJSON();
10691088

10701089
const { linkedSegmentationGroup, linkedSegmentationColorGroup } =
10711090
this.displayState;
@@ -1401,6 +1420,10 @@ registerLayerTypeDetector((subsource) => {
14011420
return undefined;
14021421
});
14031422

1423+
registerLayerShaderControlsTool(SegmentationUserLayer, (layer) => ({
1424+
shaderControlState: layer.displayState.shaderControlState,
1425+
}));
1426+
14041427
registerLayerShaderControlsTool(
14051428
SegmentationUserLayer,
14061429
(layer) => ({

‎src/layer/segmentation/json_keys.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export const COLOR_SEED_JSON_KEY = "colorSeed";
1414
export const SEGMENT_STATED_COLORS_JSON_KEY = "segmentColors";
1515
export const MESH_RENDER_SCALE_JSON_KEY = "meshRenderScale";
1616
export const CROSS_SECTION_RENDER_SCALE_JSON_KEY = "crossSectionRenderScale";
17+
export const SHADER_JSON_KEY = "shader";
1718
export const SKELETON_RENDERING_JSON_KEY = "skeletonRendering";
1819
export const SKELETON_SHADER_JSON_KEY = "skeletonShader";
1920
export const SKELETON_CODE_VISIBLE_KEY = "codeVisible";

‎src/mesh/backend.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -802,6 +802,7 @@ function convertMeshData(
802802
} else {
803803
encodedVertexPositions = data.vertexPositions as Float32Array;
804804
}
805+
// TODO, add attributes here? Or can we change a uniform for every fragment draw request?
805806
return {
806807
vertexPositions: encodedVertexPositions,
807808
vertexNormals: encodedNormals,

‎src/mesh/frontend.ts‎

Lines changed: 102 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ import type {
4343
import { PerspectiveViewRenderLayer } from "#src/perspective_view/render_layer.js";
4444
import type { ThreeDimensionalRenderLayerAttachmentState } from "#src/renderlayer.js";
4545
import { update3dRenderLayerAttachment } from "#src/renderlayer.js";
46+
import {
47+
SegmentColorHash,
48+
SegmentColorShaderManager,
49+
} from "#src/segment_color.js";
4650
import {
4751
forEachVisibleSegment,
4852
getObjectKey,
@@ -56,7 +60,7 @@ import {
5660
import type { WatchableValueInterface } from "#src/trackable_value.js";
5761
import { makeCachedDerivedWatchableValue } from "#src/trackable_value.js";
5862
import type { Borrowed, RefCounted } from "#src/util/disposable.js";
59-
import type { vec4 } from "#src/util/geom.js";
63+
import { vec4 } from "#src/util/geom.js";
6064
import {
6165
getFrustrumPlanes,
6266
mat3,
@@ -68,14 +72,27 @@ import {
6872
import * as matrix from "#src/util/matrix.js";
6973
import { GLBuffer } from "#src/webgl/buffer.js";
7074
import type { GL } from "#src/webgl/context.js";
71-
import { parameterizedEmitterDependentShaderGetter } from "#src/webgl/dynamic_shader.js";
75+
import {
76+
parameterizedEmitterDependentShaderGetter,
77+
shaderCodeWithLineDirective,
78+
WatchableShaderError,
79+
} from "#src/webgl/dynamic_shader.js";
7280
import type { ShaderBuilder, ShaderProgram } from "#src/webgl/shader.js";
81+
import {
82+
addControlsToBuilder,
83+
setControlsInShader,
84+
ShaderControlState,
85+
} from "#src/webgl/shader_ui_controls.js";
7386
import type { RPC } from "#src/worker_rpc.js";
7487
import { registerSharedObjectOwner } from "#src/worker_rpc.js";
7588

7689
const tempMat4 = mat4.create();
7790
const tempMat3 = mat3.create();
7891

92+
const tempVec3 = vec3.create();
93+
94+
const colorHash = SegmentColorHash.getDefault();
95+
7996
// To validate the octrees and to determine the multiscale fragment responsible for each framebuffer
8097
// location, set `DEBUG_MULTISCALE_FRAGMENTS=true` and also set `DEBUG_PICKING=true` in
8198
// `src/object_picking.ts`.
@@ -269,6 +286,14 @@ export class MeshShaderManager {
269286
gl.uniform4fv(shader.uniform("uColor"), color);
270287
}
271288

289+
setID(gl: GL, shader: ShaderProgram, id: bigint) {
290+
gl.uniform2ui(
291+
shader.uniform(`uID`),
292+
Number(id & 0xffffffffn),
293+
Number(id >> 32n),
294+
);
295+
}
296+
272297
setPickID(gl: GL, shader: ShaderProgram, pickID: number) {
273298
gl.uniform1ui(shader.uniform("uPickID"), pickID);
274299
}
@@ -303,6 +328,7 @@ export class MeshShaderManager {
303328
indexBegin: number,
304329
indexEnd: number,
305330
) {
331+
// drawFragmentHelper
306332
this.vertexPositionHandler.bind(gl, shader, fragmentChunk);
307333
const { meshData } = fragmentChunk;
308334
fragmentChunk.normalBuffer.bindToVertexAttrib(
@@ -328,6 +354,18 @@ export class MeshShaderManager {
328354
drawFragment(gl: GL, shader: ShaderProgram, fragmentChunk: FragmentChunk) {
329355
const { meshData } = fragmentChunk;
330356
const { indices } = meshData;
357+
const id = (meshData as any).id;
358+
const bigId = BigInt(id);
359+
tempVec3;
360+
colorHash;
361+
// const color = colorHash.compute(tempVec3, bigId);
362+
// console.log('color', id, color);
363+
// this.setColor(
364+
// gl,
365+
// shader,
366+
// vec4.fromValues(color[0], color[1], color[2], 1.0),
367+
// );
368+
this.setID(gl, shader, bigId);
331369
this.drawFragmentHelper(gl, shader, fragmentChunk, 0, indices.length);
332370
}
333371

@@ -349,27 +387,33 @@ export class MeshShaderManager {
349387
}
350388

351389
makeGetter(layer: RefCounted & { gl: GL; displayState: MeshDisplayState }) {
352-
const silhouetteRenderingEnabled = layer.registerDisposer(
353-
makeCachedDerivedWatchableValue(
354-
(x) => x > 0,
355-
[layer.displayState.silhouetteRendering],
356-
),
357-
);
390+
makeCachedDerivedWatchableValue;
391+
// const silhouetteRenderingEnabled = layer.registerDisposer(
392+
// makeCachedDerivedWatchableValue(
393+
// (x) => x > 0,
394+
// [layer.displayState.silhouetteRendering],
395+
// ),
396+
// );
397+
358398
return parameterizedEmitterDependentShaderGetter(layer, layer.gl, {
359399
memoizeKey: `mesh/MeshShaderManager/${this.fragmentRelativeVertices}/${this.vertexPositionFormat}`,
360-
parameters: silhouetteRenderingEnabled,
361-
defineShader: (builder, silhouetteRenderingEnabled) => {
400+
parameters: layer.displayState.shaderControlState.builderState,
401+
shaderError: layer.displayState.shaderError,
402+
defineShader: (builder, shaderBuilderState) => {
403+
addControlsToBuilder(shaderBuilderState, builder);
362404
this.vertexPositionHandler.defineShader(builder);
363405
builder.addAttribute("highp vec2", "aVertexNormal");
364406
builder.addVarying("highp vec4", "vColor");
407+
builder.addVarying("highp float", "vLightingFactor");
365408
builder.addUniform("highp vec4", "uLightDirection");
366409
builder.addUniform("highp vec4", "uColor");
367410
builder.addUniform("highp mat3", "uNormalMatrix");
368411
builder.addUniform("highp mat4", "uModelViewProjection");
369412
builder.addUniform("highp uint", "uPickID");
370-
if (silhouetteRenderingEnabled) {
371-
builder.addUniform("highp float", "uSilhouettePower");
372-
}
413+
builder.addUniform("highp uvec2", "uID");
414+
// if (silhouetteRenderingEnabled) {
415+
// builder.addUniform("highp float", "uSilhouettePower");
416+
// }
373417
if (this.fragmentRelativeVertices) {
374418
builder.addUniform("highp vec3", "uFragmentOrigin");
375419
builder.addUniform("highp vec3", "uFragmentShape");
@@ -394,21 +438,30 @@ vec3 normal = normalize(uNormalMatrix * (normalMultiplier * origNormal));
394438
float absCosAngle = abs(dot(normal, uLightDirection.xyz));
395439
float lightingFactor = absCosAngle + uLightDirection.w;
396440
vColor = vec4(lightingFactor * uColor.rgb, uColor.a);
441+
vLightingFactor = lightingFactor;
397442
`;
398-
if (silhouetteRenderingEnabled) {
399-
vertexMain += `
400-
vColor *= pow(1.0 - absCosAngle, uSilhouettePower);
401-
`;
402-
}
443+
// if (silhouetteRenderingEnabled) {
444+
// vertexMain += `
445+
// vColor *= pow(1.0 - absCosAngle, uSilhouettePower);
446+
// `;
447+
// }
403448
builder.setVertexMain(vertexMain);
404-
builder.setFragmentMain("emit(vColor, uPickID);");
449+
450+
const shaderManager = new SegmentColorShaderManager("getColor");
451+
shaderManager.defineShader(builder);
452+
builder.setFragmentMainFunction(
453+
shaderCodeWithLineDirective(shaderBuilderState.parseResult.code),
454+
);
405455
},
406456
});
407457
}
408458
}
409459

410460
export interface MeshDisplayState extends SegmentationDisplayState3D {
411461
silhouetteRendering: WatchableValueInterface<number>;
462+
463+
shaderControlState: ShaderControlState;
464+
shaderError: WatchableShaderError;
412465
}
413466

414467
export class MeshLayer extends PerspectiveViewRenderLayer<ThreeDimensionalRenderLayerAttachmentState> {
@@ -432,6 +485,22 @@ export class MeshLayer extends PerspectiveViewRenderLayer<ThreeDimensionalRender
432485
this.registerDisposer(
433486
displayState.silhouetteRendering.changed.add(this.redrawNeeded.dispatch),
434487
);
488+
this.registerDisposer(
489+
displayState.shaderControlState.changed.add(this.redrawNeeded.dispatch),
490+
);
491+
this.registerDisposer(
492+
displayState.shaderControlState.parseResult.changed.add(
493+
this.redrawNeeded.dispatch,
494+
),
495+
);
496+
497+
console.log("or are we not multiscale?");
498+
499+
this.registerDisposer(
500+
displayState.shaderControlState.changed.add(() => {
501+
console.log("shader control state changed!");
502+
}),
503+
);
435504

436505
const sharedObject = (this.backend = this.registerDisposer(
437506
new SegmentationLayerSharedObject(
@@ -490,11 +559,20 @@ export class MeshLayer extends PerspectiveViewRenderLayer<ThreeDimensionalRender
490559
if (modelMatrix === undefined) {
491560
return;
492561
}
493-
const { shader } = this.getShader(renderContext.emitter);
562+
// console.log("get shader!", this.displayState);
563+
const { shader, parameters } = this.getShader(renderContext.emitter);
494564
if (shader === null) return;
495565
shader.bind();
496566
meshShaderManager.beginLayer(gl, shader, renderContext, this.displayState);
497567
meshShaderManager.beginModel(gl, shader, renderContext, modelMatrix);
568+
setControlsInShader;
569+
this.displayState.silhouetteRendering;
570+
setControlsInShader(
571+
gl,
572+
shader,
573+
this.displayState.shaderControlState,
574+
parameters.parseResult.controls,
575+
);
498576

499577
const manifestChunks = this.source.chunks;
500578

@@ -774,6 +852,8 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer<ThreeDimensi
774852
displayState.silhouetteRendering.changed.add(this.redrawNeeded.dispatch),
775853
);
776854

855+
console.log("are we multiscale?");
856+
777857
const sharedObject = (this.backend = this.registerDisposer(
778858
new SegmentationLayerSharedObject(
779859
chunkManager,
@@ -829,7 +909,8 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer<ThreeDimensi
829909
attachment,
830910
);
831911
if (modelMatrix === undefined) return;
832-
const { shader } = this.getShader(renderContext.emitter);
912+
const { shader, parameters } = this.getShader(renderContext.emitter);
913+
parameters;
833914
if (shader === null) return;
834915
shader.bind();
835916
meshShaderManager.beginLayer(gl, shader, renderContext, this.displayState);

‎src/skeleton/frontend.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ import type { GL } from "#src/webgl/context.js";
6262
import type { WatchableShaderError } from "#src/webgl/dynamic_shader.js";
6363
import {
6464
makeTrackableFragmentMain,
65+
makeWatchableShaderError,
6566
parameterizedEmitterDependentShaderGetter,
6667
shaderCodeWithLineDirective,
6768
} from "#src/webgl/dynamic_shader.js";
@@ -428,6 +429,7 @@ export class SkeletonRenderingOptions implements Trackable {
428429
}
429430

430431
shader = makeTrackableFragmentMain(DEFAULT_FRAGMENT_MAIN);
432+
shaderError = makeWatchableShaderError();
431433
shaderControlState = new ShaderControlState(this.shader);
432434
params2d: ViewSpecificSkeletonRenderingOptions = {
433435
mode: new TrackableSkeletonRenderMode(SkeletonRenderMode.LINES_AND_POINTS),

0 commit comments

Comments
 (0)