Skip to content

Commit fa8afce

Browse files
chrisjchrisj
committed
wip segment properties in shader
1 parent 3272b99 commit fa8afce

File tree

9 files changed

+228
-24
lines changed

9 files changed

+228
-24
lines changed

‎src/layer/annotation/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -854,6 +854,7 @@ class RenderingOptionsTab extends Tab {
854854
href: "https://github.com/google/neuroglancer/blob/master/src/annotation/rendering.md",
855855
},
856856
"neuroglancer-annotation-dropdown-shader-top-row",
857+
"anno",
857858
),
858859
);
859860

‎src/layer/image/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -551,6 +551,7 @@ class RenderingOptionsTab extends Tab {
551551
href: "https://github.com/google/neuroglancer/blob/master/src/sliceview/image_layer_rendering.md",
552552
},
553553
"neuroglancer-image-dropdown-top-row",
554+
"fin",
554555
),
555556
);
556557
element.appendChild(

‎src/layer/segmentation/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ class LinkedSegmentationGroupState<
416416

417417
const DEFAULT_FRAGMENT_MAIN = `
418418
void main() {
419-
emit(vColor, uPickID);
419+
setColor(defaultColor());
420420
}
421421
`;
422422

‎src/layer/segmentation/style.css‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
.neuroglancer-segmentation-rendering-tab .neuroglancer-shader-code-widget {
2424
height: 6em;
25+
resize: vertical;
2526
}
2627

2728
.neuroglancer-segmentation-dropdown-skeleton-shader-header {

‎src/layer/single_mesh/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ class DisplayOptionsTab extends Tab {
222222
href: "https://github.com/google/neuroglancer/blob/master/src/sliceview/image_layer_rendering.md",
223223
},
224224
"neuroglancer-single-mesh-dropdown-top-row",
225+
"foop",
225226
),
226227
);
227228
element.appendChild(this.attributeWidget.element);

‎src/mesh/frontend.ts‎

Lines changed: 209 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,10 @@ import {
5757
registerRedrawWhenSegmentationDisplayState3DChanged,
5858
SegmentationLayerSharedObject,
5959
} from "#src/segmentation_display_state/frontend.js";
60+
import { PreprocessedSegmentPropertyMap } from "#src/segmentation_display_state/property_map.js";
6061
import type { WatchableValueInterface } from "#src/trackable_value.js";
6162
import { makeCachedDerivedWatchableValue } from "#src/trackable_value.js";
63+
import { DataType } from "#src/util/data_type.js";
6264
import type { Borrowed, RefCounted } from "#src/util/disposable.js";
6365
import { vec4 } from "#src/util/geom.js";
6466
import {
@@ -387,19 +389,62 @@ export class MeshShaderManager {
387389
}
388390

389391
makeGetter(layer: RefCounted & { gl: GL; displayState: MeshDisplayState }) {
390-
makeCachedDerivedWatchableValue;
391-
// const silhouetteRenderingEnabled = layer.registerDisposer(
392-
// makeCachedDerivedWatchableValue(
393-
// (x) => x > 0,
394-
// [layer.displayState.silhouetteRendering],
395-
// ),
396-
// );
392+
const parameters = layer.registerDisposer(
393+
makeCachedDerivedWatchableValue(
394+
(a, b, c) => {
395+
console.log("update CDWV");
396+
return {
397+
silhouetteRenderingEnabled: a > 0,
398+
shaderBuilderState: b,
399+
segmentProperties: c,
400+
};
401+
},
402+
[
403+
layer.displayState.silhouetteRendering,
404+
layer.displayState.shaderControlState.builderState,
405+
layer.displayState.segmentationGroupState.value.segmentPropertyMap,
406+
],
407+
),
408+
);
409+
410+
parameters.changed.add(() => {
411+
console.log("parameters changed");
412+
});
413+
414+
layer.displayState.segmentationGroupState.value.segmentPropertyMap.changed.add(
415+
() => {
416+
console.log("segment property map changed");
417+
},
418+
);
419+
420+
function getShaderOutputType(ioType: DataType): string {
421+
switch (ioType) {
422+
case DataType.UINT8:
423+
case DataType.UINT16:
424+
case DataType.UINT32:
425+
return "uint";
426+
case DataType.INT8:
427+
case DataType.INT16:
428+
case DataType.INT32:
429+
return "int";
430+
case DataType.FLOAT32:
431+
return "float";
432+
case DataType.UINT64:
433+
return "uvec2";
434+
}
435+
}
397436

398437
return parameterizedEmitterDependentShaderGetter(layer, layer.gl, {
399438
memoizeKey: `mesh/MeshShaderManager/${this.fragmentRelativeVertices}/${this.vertexPositionFormat}`,
400-
parameters: layer.displayState.shaderControlState.builderState,
439+
parameters,
440+
encodeParameters: (p) => {
441+
return `${p.silhouetteRenderingEnabled}/${p.shaderBuilderState.parseResult.code}/${p.segmentProperties?.numericalProperties.map((np) => np.id).join(",")}`;
442+
},
401443
shaderError: layer.displayState.shaderError,
402-
defineShader: (builder, shaderBuilderState) => {
444+
defineShader: (
445+
builder,
446+
{ silhouetteRenderingEnabled, shaderBuilderState, segmentProperties },
447+
) => {
403448
addControlsToBuilder(shaderBuilderState, builder);
404449
this.vertexPositionHandler.defineShader(builder);
405450
builder.addAttribute("highp vec2", "aVertexNormal");
@@ -411,14 +456,60 @@ export class MeshShaderManager {
411456
builder.addUniform("highp mat4", "uModelViewProjection");
412457
builder.addUniform("highp uint", "uPickID");
413458
builder.addUniform("highp uvec2", "uID");
414-
// if (silhouetteRenderingEnabled) {
415-
// builder.addUniform("highp float", "uSilhouettePower");
416-
// }
459+
460+
// TODO define segment property uniforms here
461+
// console.log("segmentProperties", segmentProperties);
462+
463+
const shaderCodeWithPropertyPreprocessing = (code: string) => {
464+
if (!segmentProperties) return code;
465+
const { numericalProperties, tags } = segmentProperties;
466+
numericalProperties;
467+
if (tags) {
468+
for (const [i, tag] of tags.tags.entries()) {
469+
code = code.replaceAll(`tag("${tag}")`, `uTag${i} == 1u`);
470+
}
471+
}
472+
return code;
473+
};
474+
475+
if (segmentProperties) {
476+
const { numericalProperties, tags } = segmentProperties;
477+
for (const property of numericalProperties) {
478+
console.log("numeric", property);
479+
// TODO maybe just number it instead of using id
480+
builder.addUniform(
481+
`highp ${getShaderOutputType(property.dataType)}`,
482+
property.id.replaceAll(" ", "_"),
483+
);
484+
}
485+
if (tags) {
486+
console.log("tags", tags);
487+
488+
for (const [i, tag] of tags.tags.entries()) {
489+
console.log("tag", tag, tag.replaceAll(" ", "_"));
490+
builder.addUniform("highp uint", `uTag${i}`);
491+
builder.addVertexCode(
492+
`\n#define TAG_${tag.replaceAll(" ", "_").replaceAll("-", "_").toUpperCase()} uTag${i}\n`,
493+
);
494+
// gl.uniform1ui(shader.uniform(`tag${tagIndices.charCodeAt(i)}`), 1);
495+
}
496+
}
497+
}
498+
499+
if (silhouetteRenderingEnabled) {
500+
builder.addUniform("highp float", "uSilhouettePower");
501+
}
417502
if (this.fragmentRelativeVertices) {
418503
builder.addUniform("highp vec3", "uFragmentOrigin");
419504
builder.addUniform("highp vec3", "uFragmentShape");
420505
}
421506
builder.addVertexCode(glsl_decodeNormalOctahedronSnorm8);
507+
508+
builder.addVertexCode(`
509+
vec4 defaultColor() { return uColor; }
510+
void setColor(vec4 color) {
511+
vColor = color;
512+
}`);
422513
let vertexMain = "";
423514
if (this.fragmentRelativeVertices) {
424515
vertexMain += `
@@ -437,26 +528,103 @@ vec3 origNormal = decodeNormalOctahedronSnorm8(aVertexNormal);
437528
vec3 normal = normalize(uNormalMatrix * (normalMultiplier * origNormal));
438529
float absCosAngle = abs(dot(normal, uLightDirection.xyz));
439530
float lightingFactor = absCosAngle + uLightDirection.w;
440-
vColor = vec4(lightingFactor * uColor.rgb, uColor.a);
531+
vColor = uColor;
532+
userMain();
533+
vColor = vec4(lightingFactor * vColor.rgb, vColor.a);
441534
vLightingFactor = lightingFactor;
442535
`;
443-
// if (silhouetteRenderingEnabled) {
444-
// vertexMain += `
445-
// vColor *= pow(1.0 - absCosAngle, uSilhouettePower);
446-
// `;
447-
// }
536+
if (silhouetteRenderingEnabled) {
537+
vertexMain += `
538+
vColor *= pow(1.0 - absCosAngle, uSilhouettePower);
539+
`;
540+
}
448541
builder.setVertexMain(vertexMain);
449542

450543
const shaderManager = new SegmentColorShaderManager("getColor");
451544
shaderManager.defineShader(builder);
452-
builder.setFragmentMainFunction(
453-
shaderCodeWithLineDirective(shaderBuilderState.parseResult.code),
545+
builder.setFragmentMain("emit(vColor, uPickID);");
546+
shaderCodeWithLineDirective;
547+
// builder.setFragmentMainFunction(
548+
// shaderCodeWithLineDirective(shaderBuilderState.parseResult.code),
549+
// );
550+
551+
builder.addVertexCode(
552+
"\n#define main userMain\n" +
553+
shaderCodeWithLineDirective(shaderCodeWithPropertyPreprocessing(shaderBuilderState.parseResult.code)) +
554+
"\n#undef main\n",
454555
);
455556
},
456557
});
457558
}
458559
}
459560

561+
const addPropertyUniforms = (
562+
gl: GL,
563+
shader: ShaderProgram,
564+
segmentPropertyMap: PreprocessedSegmentPropertyMap,
565+
id: bigint,
566+
) => {
567+
const index = segmentPropertyMap.getSegmentInlineIndex(id);
568+
// const {tags} = segmentPropertyMap;
569+
// console.log("index", index, objectId, key);
570+
// console.log(
571+
// "segmentPropertyMap.numerical",
572+
// segmentPropertyMap.numericalProperties,
573+
// );
574+
// console.log("segmentPropertyMap.numerical", segmentPropertyMap.tags);
575+
576+
if (index !== -1) {
577+
const { labels, tags: tagsProperty } = segmentPropertyMap;
578+
labels;
579+
// let label = "";
580+
// if (labels !== undefined) {
581+
// label = labels.values[index];
582+
// }
583+
if (tagsProperty !== undefined) {
584+
const { values, tags } = tagsProperty;
585+
const tagIndices = values[index];
586+
for (let i = 0; i < tags.length; ++i) {
587+
gl.uniform1ui(shader.uniform(`uTag${i}`), 0);
588+
}
589+
for (let i = 0, length = tagIndices.length; i < length; ++i) {
590+
const tagIdx = tagIndices.charCodeAt(i);
591+
console.log("enable tag", tagIdx);
592+
gl.uniform1ui(shader.uniform(`uTag${tagIdx}`), 1);
593+
}
594+
}
595+
596+
function getShaderUniformValueSetter(gl: GL, ioType: DataType) {
597+
switch (ioType) {
598+
case DataType.UINT8:
599+
case DataType.UINT16:
600+
case DataType.UINT32:
601+
return gl.uniform1ui;
602+
case DataType.INT8:
603+
case DataType.INT16:
604+
case DataType.INT32:
605+
return gl.uniform1i;
606+
case DataType.FLOAT32:
607+
return gl.uniform1f;
608+
case DataType.UINT64:
609+
throw new Error("nope");
610+
// return gl.uniform
611+
}
612+
}
613+
// if (label.length === 0) return undefined;
614+
615+
// if (tags) {
616+
// console.log('tags', tags);
617+
// // for (const tag of tags) {
618+
// }
619+
getShaderUniformValueSetter;
620+
// for (const [name, channel] of Object.entries(value)) {
621+
// const setter = getShaderUniformValueSetter(gl, channel.dataType);
622+
// if (setter) {
623+
// setter.call(gl, shader.uniforms[name], channel.value);
624+
// }
625+
// }
626+
}
627+
};
460628
export interface MeshDisplayState extends SegmentationDisplayState3D {
461629
silhouetteRendering: WatchableValueInterface<number>;
462630

@@ -560,8 +728,10 @@ export class MeshLayer extends PerspectiveViewRenderLayer<ThreeDimensionalRender
560728
return;
561729
}
562730
// console.log("get shader!", this.displayState);
731+
console.log("get shader reg!");
563732
const { shader, parameters } = this.getShader(renderContext.emitter);
564733
if (shader === null) return;
734+
console.log("shader", shader.vertexSource);
565735
shader.bind();
566736
meshShaderManager.beginLayer(gl, shader, renderContext, this.displayState);
567737
meshShaderManager.beginModel(gl, shader, renderContext, modelMatrix);
@@ -571,7 +741,7 @@ export class MeshLayer extends PerspectiveViewRenderLayer<ThreeDimensionalRender
571741
gl,
572742
shader,
573743
this.displayState.shaderControlState,
574-
parameters.parseResult.controls,
744+
parameters.shaderBuilderState.parseResult.controls,
575745
);
576746

577747
const manifestChunks = this.source.chunks;
@@ -600,6 +770,14 @@ export class MeshLayer extends PerspectiveViewRenderLayer<ThreeDimensionalRender
600770
}
601771
totalChunks += manifestChunk.fragmentIds.length;
602772

773+
const {
774+
segmentPropertyMap: { value: segmentPropertyMap },
775+
} = displayState.segmentationGroupState.value;
776+
if (segmentPropertyMap) {
777+
console.log('key/objectId', key, objectId);
778+
addPropertyUniforms(gl, shader, segmentPropertyMap, objectId);
779+
}
780+
603781
for (const fragmentId of manifestChunk.fragmentIds) {
604782
const { key: fragmentKey } = this.source.getFragmentKey(
605783
key,
@@ -909,6 +1087,7 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer<ThreeDimensi
9091087
attachment,
9101088
);
9111089
if (modelMatrix === undefined) return;
1090+
console.log("get shader multiscale!");
9121091
const { shader, parameters } = this.getShader(renderContext.emitter);
9131092
parameters;
9141093
if (shader === null) return;
@@ -981,6 +1160,15 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer<ThreeDimensi
9811160
if (renderContext.emitPickID) {
9821161
meshShaderManager.setPickID(gl, shader, pickIndex!);
9831162
}
1163+
1164+
const {
1165+
segmentPropertyMap: { value: segmentPropertyMap },
1166+
} = displayState.segmentationGroupState.value;
1167+
if (segmentPropertyMap) {
1168+
console.log('key/objectId', key, objectId);
1169+
addPropertyUniforms(gl, shader, segmentPropertyMap, objectId);
1170+
}
1171+
9841172
getMultiscaleChunksToDraw(
9851173
manifest,
9861174
modelViewProjection,

‎src/segmentation_display_state/property_map.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ export class PreprocessedSegmentPropertyMap {
208208
}
209209

210210
getSegmentLabel(id: bigint): string | undefined {
211+
// console.log("get segment label!", id);
211212
const index = this.getSegmentInlineIndex(id);
212213
if (index === -1) return undefined;
213214
const { labels, tags: tagsProperty } = this;
@@ -228,6 +229,7 @@ export class PreprocessedSegmentPropertyMap {
228229
}
229230
}
230231
if (label.length === 0) return undefined;
232+
// console.log("label", label);
231233
return label;
232234
}
233235
}

0 commit comments

Comments
 (0)