Some OpenLime feature suggestion, bugfixes, and discussion

[jo-chemla]

Thanks for getting back on the release thread here. First of all, thanks for this amazing OpenLime utility, really great playing with multi-layer RTI, deepzoom, BRDF and relight layers as well as lens, shaders, filters, annotations, lightsphere etc.

Here are some question after playing a bit with the library during the past few days - feel free to tell me to open a dedicated issue/discussion thread for some points in case it makes more sense. Sorry for the lengthy read.

1. RelightLab convert to *.relight: RelightLab can convert an existing *.rti/*.ptm file to relight directory (info.json + plane_k.jpg images), but not to *.relight format, correct? If so, it would be great for the convert function to also allow exporting .relight file. In the meantime, looks like the ony way to extract normals from an existing *.rti project is to either:

   • create a new project with input images and recompute the relight file,
   • use the cnr-isti-vclab/relight-cli,
   • or export fullres snapshot from eg RTIViewer

2. layer.visible = false should hide layers: From my understanding, layers which have property visible: false should not appear in the layers sidepanel from UIBasic. Or is there a reason for displaying them in the tree view sidepanel? Probably this line can be prefixed with a check on layer.visible, or better, the for loop can start with a continue statement if layer.visible is false

 for (let [id, layer] of Object.entries(this.viewer.canvas.layers)) {
+  if (!layer.visible) continue
   let modes = []
   // ...

3. new tests to index.js: After spending some time parsing through the source, I've added tests (mostly shaders) that were previously not existing to src/index.js . Do you want me to submit a PR for these? Also, it could be useful to add a note in the doc pointing to this index.js file which is very useful while doc gets more complete. These are the code blocks that might be useful: testLightSphere, testEdge, testOtherFilters, testAnisotropic, testVector

4. ShaderCombinerBlend: Would it make sense to add a blend/mix mode to LayerCombiner, that would be parametrized by a 0-1 coefficient. I've added such a shader in a folded code section below, with claude helping for the openlime Shader wrapper. I'm probably messing with layer.modes and layer.shader, so I had to overwrite layer.getMode, getModes, setMode to switch between shaders in the layers sidepanel.

ShaderCombinerBlend

  function addBlendLayer() {
    
    let layer0 = new Layer({
      type: 'image',
      url: './RTI_panneau_1_test2_7360/plane_0.jpg',
      layout: 'image',
      zindex: 0,
      transform: { x: 0, y: 0, z: 1, a: 0 },
      visible: false
    });
    
    let layer1 = new Layer({
      type: 'image',
      url: './RTI_panneau_1_test2_7360/plane_1.jpg',
      layout: 'image',
      zindex: 0,
      transform: { x: 0, y: 0, z: 1, a: 0 },
      visible: false
    });
      
    let blendLayerCombiner = new LayerCombiner({
      layers: [layer0, layer1]
    });

    // Create the existing shaders
    let shaderDiff = new ShaderCombiner();
    shaderDiff.mode = 'diff';
    
    let shaderMean = new ShaderCombiner();
    shaderMean.mode = 'mean';


    // Create the blend shader with initial weight
    let shaderBlend = new ShaderCombinerBlend({
      blendWeight: 0.5  // Start at 50/50
    });

    // Add all shaders
    blendLayerCombiner.shaders = {
      'standarddiff': shaderDiff, 
      'standardmean': shaderMean,
      'blend': shaderBlend  // Add blend shader
    };

    // shaderBlend.modes = ['blend', 'standarddiff', 'standardmean']
    // shaderBlend.mode = 'blend'
    
    // Start with blend mode
    blendLayerCombiner.setShader('blend'); 


  // Override getModes() to return the shader names
  blendLayerCombiner.getModes = function() {
    return Object.keys(this.shaders);
  };

  // Override setMode() to switch shaders instead
  blendLayerCombiner.setMode = function(shaderName) {
    this.setShader(shaderName);
  };

  // Override getMode() to return current shader name
  blendLayerCombiner.getMode = function() {
    for (let [name, shader] of Object.entries(this.shaders)) {
      if (shader === this.shader) return name;
    }
    return 'blend';
  };


    limeViewer.canvas.addLayer('blend-layer0', layer0);
    limeViewer.canvas.addLayer('blend-layer1', layer1);
    limeViewer.canvas.addLayer('blendLayerCombiner', blendLayerCombiner);
    window.blendLayerCombiner = blendLayerCombiner

  }



/**
 * A shader for blending two layers with adjustable weight.
 * Works with the standard LayerCombiner without any modifications.
 * 
 * @extends Shader
 * @example
 * ```javascript
 * // Use with existing LayerCombiner
 * const combiner = new OpenLIME.LayerCombiner({
 *   layers: [layer1, layer2]
 * });
 * 
 * const blendShader = new ShaderCombinerBlend({
 *   blendWeight: 0.5
 * });
 * 
 * combiner.shaders = { 'blend': blendShader };
 * combiner.setShader('blend');
 * 
 * // Change blend weight directly through shader
 * blendShader.setUniform('uBlendWeight', 0.7);
 * combiner.emit('update');
 * 
 * // Or add UI control for it
 * uiBasic.addUniformUI(
 *   combiner, 
 *   'uBlendWeight', 
 *   'Blend', 
 *   'slider',
 *   0, 100,    // display range
 *   0.0, 1.0,  // actual uniform range
 *   100        // steps
 * );
 * ```
 */
class ShaderCombinerBlend extends Shader {
    constructor(options = {}) {
        super(options);
        
        const blendWeight = options.blendWeight !== undefined ? options.blendWeight : 0.5;
        
        // Match the sampler names from ShaderCombiner
        this.samplers = [
            { id: 0, name: 'source1', type: 'vec3' },
            { id: 1, name: 'source2', type: 'vec3' }
        ];
        
        this.uniforms = {
            uBlendWeight: { type: 'float', needsUpdate: true, size: 1, value: blendWeight }
        };
    }

    /**
     * Gets fragment shader source code.
     * Implements texture blending with adjustable weight.
     * @param {WebGLRenderingContext} gl - WebGL context
     * @returns {string} Fragment shader source code
     * @private
     */
    fragShaderSrc(gl) {
        return `
in vec2 v_texcoord;
uniform float uBlendWeight;


vec4 data() {
    vec4 c1 = texture(source1, v_texcoord);
    vec4 c2 = texture(source2, v_texcoord);
    
    // Blend between the two layers
    // uBlendWeight = 0.0 -> 100% layer 0 (c1)
    // uBlendWeight = 1.0 -> 100% layer 1 (c2)
    vec4 color = mix(c1, c2, uBlendWeight);
    
    return color;
}
`;
    }

    /**
     * Gets vertex shader source code.
     * Provides basic vertex transformation and texture coordinate passing.
     * @param {WebGLRenderingContext} gl - WebGL context
     * @returns {string} Vertex shader source code
     * @private
     */
    vertShaderSrc(gl) {
        return `#version 300 es
in vec4 a_position;
in vec2 a_texcoord;
out vec2 v_texcoord;

void main() {
    gl_Position = a_position;
    v_texcoord = a_texcoord;
}`;
    }
}

5. BRDF: If no ks/gloss maps are provided, it would be useful to allow controlling the constant specular/gloss of the whole surface. One can create fake white/gray textures, but it's not the most interactive way to control the overall glossiness of the surface.

6. addUniformUI vec/array support I wanted to edit brightness and gamma from the BRDF shader, so I thought it would be useful for ui.addUniformUI slider controls to support array/vec uniforms like brdf uBrightnessGamma or ShaderFilterGrayscale.weights. Might be a too specific use-case, but I edited slightly

   openlime/src/UIBasic.js

   Lines 1106 to 1139 in 4f34630

   	else if (uiType === 'slider') {
   	// Calculate step size based on uiNStepDisplayed
   	const stepSize = uiNStepDisplayed > 0 ?
   	((uiMaxDisplayed - uiMinDisplayed) / uiNStepDisplayed).toFixed(6) :
   	'any';
   	// Create slider with value display
   	controlWrapper.innerHTML = `
   	<div class="openlime-uniform-slider-container">
   	<input type="range" class="openlime-uniform-slider"
   	min="${uiMinDisplayed}" max="${uiMaxDisplayed}"
   	step="${stepSize}" value="${displayValue}">
   	<span class="openlime-uniform-slider-value">${displayValue.toFixed(2)}</span>
   	</div>
   	`;
   	// Add event listener
   	const slider = controlWrapper.querySelector('.openlime-uniform-slider');
   	const valueDisplay = controlWrapper.querySelector('.openlime-uniform-slider-value');
   	slider.addEventListener('input', (e) => {
   	const displayedValue = parseFloat(e.target.value);
   	// Update value display
   	valueDisplay.textContent = displayedValue.toFixed(2);
   	// Map to uniform range
   	const uniformValue = mapToUniform(displayedValue);
   	this.updateUniformValue(layer, originalUniformName, uniformValue, filter);
   	// Update other controls for the same uniform
   	this.updateRelatedControls(layerEntry, originalUniformName, uniformValue, controlId);
   	});

   to add one-slider per array-elements-count into controlWrapper.innerHTML, see below folded code block.
   I then realized the uBrightnessGamma is actually not used because applyGamma is set to false, so I tweaked the below line for the sliders to work:

brdfLayer.shader.innerCode += '; \n applyGamma = true;\n'

addUniformUI vec/array support

else if (uiType === 'slider') {
	// Calculate step size based on uiNStepDisplayed
	const stepSize = uiNStepDisplayed > 0 ?
		((uiMaxDisplayed - uiMinDisplayed) / uiNStepDisplayed).toFixed(6) :
		'any';

	const isArray = Array.isArray(currentValue);
	const arrayLength = isArray ? currentValue.length : 1;
	
	// Create slider with value display
	let slidersHTML = '';
	for (let i = 0; i < arrayLength; i++) {
		const elementValue = isArray ? currentValue[i] : currentValue;
		const elementDisplayValue = mapToDisplay(elementValue);
		
		slidersHTML += `
			<div class="openlime-uniform-slider-container" data-array-index="${i}">
				${arrayLength > 1 ? `<span class="openlime-uniform-slider-label">[${i}]</span>` : ''}
				<input type="range" class="openlime-uniform-slider" 
					min="${uiMinDisplayed}" max="${uiMaxDisplayed}" 
					step="${stepSize}" value="${elementDisplayValue}">
				<span class="openlime-uniform-slider-value">${elementDisplayValue.toFixed(2)}</span>
			</div>
		`;
	}
	
	controlWrapper.innerHTML = slidersHTML;

	// Add event listeners
	const sliders = controlWrapper.querySelectorAll('.openlime-uniform-slider');
	const valueDisplays = controlWrapper.querySelectorAll('.openlime-uniform-slider-value');

	sliders.forEach((slider, idx) => {
		slider.addEventListener('input', (e) => {
			const displayedValue = parseFloat(e.target.value);
			valueDisplays[idx].textContent = displayedValue.toFixed(2);
			let uniformValue;
			if (isArray) {
				uniformValue = Array.from(valueDisplays).map(valEl => 
					mapToUniform(parseFloat(valEl.textContent))
				);
			} else {
				uniformValue = mapToUniform(displayedValue);
			}
			ui.updateUniformValue(layer, originalUniformName, uniformValue, filter);
			ui.updateRelatedControls(layerEntry, originalUniformName, uniformValue, controlId);
		});
	});
}

7. Layer and LayerCombiner always displays all modes, despite specifying a sub-selection of shaders in combinerTest. But again, I might be messing up with modes and shaders.

combiner.shaders = {'standard-diff': shaderDiff, 'standard-mean': shaderMean };
combiner.setShader('standard-diff'); 

[jo-chemla]

If the suggested changes make sense to you, I can quickly do one (or multple) PR addressing 2, 3, 4, 6, 7