[wip]: try to find acceptable perf/quality values

- add controls to tweak the values
- AI-coded, as it's up to deletion. Worth keeping for debugging purposes?

[skip ci]

Signed-off-by: Maksim Sukharev <antreesy.web@gmail.com>

Author: Antreesy

src/components/MediaSettings/VideoBackgroundEditor.vue

@@ -67,6 +67,24 @@
 			tabindex="-1"
 			aria-hidden="true"
 			@change="handleFileInput">
+		<div class="background-editor__debug-controls">
+			<label
+				v-for="control in debugControls"
+				:key="control.key"
+				class="background-editor__debug-control">
+				<span class="background-editor__debug-control-header">
+					<span>{{ control.label }}</span>
+					<span>{{ formatDebugConfigValue(control.key) }}</span>
+				</span>
+				<input
+					:min="control.min"
+					:max="control.max"
+					:step="control.step"
+					:value="debugConfigValues[control.key]"
+					type="range"
+					@input="handleDebugConfigInput(control.key, $event.target.value)">
+			</label>
+		</div>
 	</div>
 </template>
 
@@ -88,6 +106,7 @@ import { getDavClient } from '../../services/DavClient.ts'
 import { useActorStore } from '../../stores/actor.ts'
 import { useSettingsStore } from '../../stores/settings.ts'
 import { findUniquePath } from '../../utils/fileUpload.ts'
+import { VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES, virtualBackgroundDebugConfig, setVirtualBackgroundDebugConfigValue } from '../../utils/media/effects/virtual-background/runtimeConfig.js'
 
 const predefinedBackgroundLabels = {
 	'1_office': t('spreed', 'Select virtual office background'),
@@ -100,6 +119,24 @@ const predefinedBackgroundLabels = {
 	'8_space_station': t('spreed', 'Select virtual space station background'),
 }
 
+const virtualBackgroundDebugControlLabels = {
+	DEFAULT_BLUR_PASSES: t('spreed', 'Blur passes'),
+	SIGMA_SPACE: t('spreed', 'Sigma space'),
+	SIGMA_COLOR: t('spreed', 'Sigma color'),
+	SPARSITY_FACTOR: t('spreed', 'Sparsity factor'),
+	DEFAULT_FRAME_RATE: t('spreed', 'Default frame rate'),
+	MAX_SEGMENTATION_FRAME_RATE: t('spreed', 'Max segmentation frame rate'),
+}
+
+const virtualBackgroundDebugControlFractionDigits = {
+	DEFAULT_BLUR_PASSES: 0,
+	SIGMA_SPACE: 1,
+	SIGMA_COLOR: 2,
+	SPARSITY_FACTOR: 2,
+	DEFAULT_FRAME_RATE: 0,
+	MAX_SEGMENTATION_FRAME_RATE: 0,
+}
+
 export default {
 	name: 'VideoBackgroundEditor',
 
@@ -139,6 +176,7 @@ export default {
 	data() {
 		return {
 			selectedBackground: undefined,
+			debugConfigValues: { ...virtualBackgroundDebugConfig },
 		}
 	},
 
@@ -162,6 +200,15 @@ export default {
 		relativeBackgroundsFolderPath() {
 			return this.settingsStore.attachmentFolder + '/Backgrounds'
 		},
+
+		debugControls() {
+			return Object.entries(VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES).map(([key, range]) => ({
+				key,
+				label: virtualBackgroundDebugControlLabels[key],
+				fractionDigits: virtualBackgroundDebugControlFractionDigits[key],
+				...range,
+			}))
+		},
 	},
 
 	async mounted() {
@@ -268,6 +315,17 @@ export default {
 			this.handleSelectBackground(previewURL)
 		},
 
+		handleDebugConfigInput(key, value) {
+			this.debugConfigValues[key] = setVirtualBackgroundDebugConfigValue(key, value)
+		},
+
+		formatDebugConfigValue(key) {
+			const value = this.debugConfigValues[key]
+			const fractionDigits = virtualBackgroundDebugControlFractionDigits[key]
+
+			return fractionDigits > 0 ? value.toFixed(fractionDigits) : String(value)
+		},
+
 		loadBackground() {
 			// Set virtual background depending on browser storage's settings
 			if (BrowserStorage.getItem('virtualBackgroundEnabled') === 'true') {
@@ -305,6 +363,24 @@ export default {
 	max-height: calc(var(--background-button-height) * 3 + var(--default-grid-baseline) * 4);
 	overflow-y: auto;
 
+	&__debug-controls {
+		grid-column: 1 / -1;
+		display: grid;
+		gap: calc(var(--default-grid-baseline) * 2);
+	}
+
+	&__debug-control {
+		display: grid;
+		gap: var(--default-grid-baseline);
+	}
+
+	&__debug-control-header {
+		display: flex;
+		justify-content: space-between;
+		gap: calc(var(--default-grid-baseline) * 2);
+		font-size: 12px;
+	}
+
 	&__element {
 		border: none;
 		margin: 0 !important;

src/main.js

@@ -14,6 +14,7 @@ import store from './store/index.js'
 import pinia from './stores/pinia.ts'
 import { useSidebarStore } from './stores/sidebar.ts'
 import { NextcloudGlobalsVuePlugin } from './utils/NextcloudGlobalsVuePlugin.js'
+import { exposeVirtualBackgroundDebugConfig } from './utils/media/effects/virtual-background/runtimeConfig.js'
 
 import './init.js'
 // Leaflet icon patch
@@ -104,6 +105,7 @@ subscribe('viewer:sidebar:open', (node) => {
 if (!window.OCA.Talk) {
 	window.OCA.Talk = reactive({})
 }
+exposeVirtualBackgroundDebugConfig(window.OCA.Talk)
 OCA.Talk.instance = instance
 OCA.Talk.Settings = SettingsAPI
 

src/utils/media/effects/virtual-background/VideoStreamBackgroundEffect.js

@@ -13,13 +13,11 @@ import {
 	timerWorkerScript,
 } from './TimerWorker.js'
 import WebGLCompositor from './WebGLCompositor.js'
+import { DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG, virtualBackgroundDebugConfig } from './runtimeConfig.js'
 
 // Cache MediaPipe resources to avoid loading them multiple times.
 let _WasmFileset = null
 
-const DEFAULT_FRAME_RATE = 30
-const MAX_SEGMENTATION_FRAME_RATE = 25
-
 /**
  * Represents a modified MediaStream that applies virtual background effects
  * (blur, image, video, or video stream) using MediaPipe segmentation.
@@ -85,8 +83,9 @@ export default class VideoStreamBackgroundEffect {
 		this._inputVideoElement = document.createElement('video')
 		this._bgChanged = false
 		this._prevBgMode = null
-		this._frameRate = DEFAULT_FRAME_RATE
-		this._runInferenceInterval = 1000 / MAX_SEGMENTATION_FRAME_RATE
+		this._inputTrackFrameRate = null
+		this._frameRate = virtualBackgroundDebugConfig.DEFAULT_FRAME_RATE
+		this._runInferenceInterval = 1000 / Math.min(this._frameRate, virtualBackgroundDebugConfig.MAX_SEGMENTATION_FRAME_RATE)
 		this._lastInferenceTime = 0
 	}
 
@@ -197,10 +196,46 @@ export default class VideoStreamBackgroundEffect {
 	 * @param {number|string} frameRate - Input frame rate from track settings.
 	 * @return {void}
 	 */
-	_updateFrameRate(frameRate) {
+	_resolveFrameRate(frameRate) {
+		const configuredFrameRate = virtualBackgroundDebugConfig.DEFAULT_FRAME_RATE
 		const parsedFrameRate = parseInt(frameRate, 10)
-		this._frameRate = !Number.isNaN(parsedFrameRate) ? parsedFrameRate : DEFAULT_FRAME_RATE
-		this._runInferenceInterval = 1000 / Math.min(this._frameRate, MAX_SEGMENTATION_FRAME_RATE)
+
+		if (Number.isNaN(parsedFrameRate)) {
+			return configuredFrameRate
+		}
+
+		if (configuredFrameRate === DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG.DEFAULT_FRAME_RATE) {
+			return parsedFrameRate
+		}
+
+		return Math.min(parsedFrameRate, configuredFrameRate)
+	}
+
+	_syncFrameRateConfig() {
+		const nextFrameRate = this._resolveFrameRate(this._inputTrackFrameRate)
+		const nextRunInferenceInterval = 1000 / Math.min(nextFrameRate, virtualBackgroundDebugConfig.MAX_SEGMENTATION_FRAME_RATE)
+		const didFrameRateChange = this._frameRate !== nextFrameRate
+
+		this._frameRate = nextFrameRate
+		this._runInferenceInterval = nextRunInferenceInterval
+
+		if (!didFrameRateChange || !this._outputStream) {
+			return
+		}
+
+		const outputTrack = this._outputStream.getVideoTracks()[0]
+		if (!outputTrack) {
+			return
+		}
+
+		outputTrack.applyConstraints({ frameRate: this._frameRate }).catch((error) => {
+			console.error('Frame rate could not be adjusted in background effect', error)
+		})
+	}
+
+	_updateFrameRate(frameRate) {
+		this._inputTrackFrameRate = frameRate
+		this._syncFrameRateConfig()
 	}
 
 	/**
@@ -293,6 +328,8 @@ export default class VideoStreamBackgroundEffect {
 			this._frameId = this._lastFrameId
 		}
 
+		this._syncFrameRateConfig()
+
 		const now = performance.now()
 
 		// Run inference if ready and enough time passed since last _runInference() call
@@ -690,10 +727,6 @@ export default class VideoStreamBackgroundEffect {
 
 		this._updateFrameRate(frameRate)
 
-		this._outputStream.getVideoTracks()[0].applyConstraints({ frameRate: this._frameRate }).catch((error) => {
-			console.error('Frame rate could not be adjusted in background effect', error)
-		})
-
 		this._frameId = -1
 		this._lastFrameId = -1
 		this._lastInferenceTime = 0

src/utils/media/effects/virtual-background/WebGLCompositor.js

@@ -4,26 +4,7 @@
  */
 // @flow
 
-/**
- * Number of horizontal + vertical blur iterations applied.
- * Larger value - stronger blur effect. Low GPU impact.
- */
-const DEFAULT_BLUR_PASSES = 3
-/**
- * Spatial radius of the bilateral filter.
- * Larger value - smoother mask over a wider area, softer edges. High GPU impact.
- */
-const SIGMA_SPACE = 10
-/**
- * Range sensitivity of the bilateral filter.
- * Larger value: more smoothing across edges. Low GPU impact.
- */
-const SIGMA_COLOR = 0.15
-/**
- * Sampling stride scale for the bilateral kernel.
- * Larger value - faster processing, rougher mask refinement. High GPU impact.
- */
-const SPARSITY_FACTOR = 0.66
+import { virtualBackgroundDebugConfig } from './runtimeConfig.js'
 
 /**
  * WebGL-based compositor for background effects.
@@ -487,10 +468,13 @@ export default class WebGLCompositor {
 		// Calculate filter parameters
 		const texelWidth = 1 / width
 		const texelHeight = 1 / height
-		const step = Math.max(1, Math.sqrt(SIGMA_SPACE) * SPARSITY_FACTOR)
-		const radius = SIGMA_SPACE
+		const sigmaSpace = virtualBackgroundDebugConfig.SIGMA_SPACE
+		const sigmaColor = virtualBackgroundDebugConfig.SIGMA_COLOR
+		const sparsityFactor = virtualBackgroundDebugConfig.SPARSITY_FACTOR
+		const step = Math.max(1, Math.sqrt(sigmaSpace) * sparsityFactor)
+		const radius = sigmaSpace
 		const offset = step > 1 ? step * 0.5 : 0
-		const sigmaTexel = Math.max(texelWidth, texelHeight) * SIGMA_SPACE
+		const sigmaTexel = Math.max(texelWidth, texelHeight) * sigmaSpace
 
 		// Set uniforms
 		gl.uniform1i(gl.getUniformLocation(this.progBilateral, 'u_inputFrame'), 0)
@@ -500,7 +484,7 @@ export default class WebGLCompositor {
 		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_radius'), radius)
 		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_offset'), offset)
 		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_sigmaTexel'), sigmaTexel)
-		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_sigmaColor'), SIGMA_COLOR)
+		gl.uniform1f(gl.getUniformLocation(this.progBilateral, 'u_sigmaColor'), sigmaColor)
 
 		// Bind textures
 		gl.activeTexture(gl.TEXTURE0)
@@ -550,7 +534,9 @@ export default class WebGLCompositor {
 		gl.activeTexture(gl.TEXTURE1)
 		gl.bindTexture(gl.TEXTURE_2D, this.texMaskFiltered)
 
-		for (let i = 0; i < DEFAULT_BLUR_PASSES; i++) {
+		const blurPasses = virtualBackgroundDebugConfig.DEFAULT_BLUR_PASSES
+
+		for (let i = 0; i < blurPasses; i++) {
 			// Horizontal pass
 			gl.uniform2f(gl.getUniformLocation(this.progBlur, 'u_texelSize'), 0, texelHeight)
 			gl.bindFramebuffer(gl.FRAMEBUFFER, this.fboBlur1)

src/utils/media/effects/virtual-background/runtimeConfig.js

@@ -0,0 +1,87 @@
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+export const DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG = Object.freeze({
+	DEFAULT_BLUR_PASSES: 2,
+	SIGMA_SPACE: 9,
+	SIGMA_COLOR: 0.25,
+	SPARSITY_FACTOR: 0.95,
+	DEFAULT_FRAME_RATE: 30,
+	MAX_SEGMENTATION_FRAME_RATE: 25,
+})
+
+export const VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES = Object.freeze({
+	DEFAULT_BLUR_PASSES: Object.freeze({ min: 1, max: 6, step: 1 }),
+	SIGMA_SPACE: Object.freeze({ min: 1, max: 20, step: 0.5 }),
+	SIGMA_COLOR: Object.freeze({ min: 0.01, max: 1, step: 0.01 }),
+	SPARSITY_FACTOR: Object.freeze({ min: 0.1, max: 1, step: 0.01 }),
+	DEFAULT_FRAME_RATE: Object.freeze({ min: 1, max: 60, step: 1 }),
+	MAX_SEGMENTATION_FRAME_RATE: Object.freeze({ min: 1, max: 60, step: 1 }),
+})
+
+const integerKeys = new Set([
+	'DEFAULT_BLUR_PASSES',
+	'DEFAULT_FRAME_RATE',
+	'MAX_SEGMENTATION_FRAME_RATE',
+])
+
+const configKeys = Object.keys(DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG)
+
+export const virtualBackgroundDebugConfig = {
+	...DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG,
+}
+
+/**
+ * Sets a debug config value after parsing and clamping it to the allowed range.
+ *
+ * @param {string} key - Config key to update.
+ * @param {number|string} value - New value coming from the UI/global object.
+ * @return {number} The normalized value stored in the shared config.
+ */
+export function setVirtualBackgroundDebugConfigValue(key, value) {
+	if (!Object.prototype.hasOwnProperty.call(virtualBackgroundDebugConfig, key)) {
+		throw new Error(`Unknown virtual background debug config key: ${key}`)
+	}
+
+	const parsedValue = integerKeys.has(key)
+		? Number.parseInt(value, 10)
+		: Number.parseFloat(value)
+	const { min, max } = VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES[key]
+	const fallbackValue = DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG[key]
+	const normalizedValue = Number.isNaN(parsedValue)
+		? fallbackValue
+		: Math.min(max, Math.max(min, parsedValue))
+
+	virtualBackgroundDebugConfig[key] = normalizedValue
+
+	return normalizedValue
+}
+
+/**
+ * Exposes the shared debug config on the provided Talk object.
+ *
+ * @param {object} talkObject - window.OCA.Talk object.
+ * @return {object} The same object with debug config attached.
+ */
+export function exposeVirtualBackgroundDebugConfig(talkObject) {
+	for (const key of configKeys) {
+		Object.defineProperty(talkObject, key, {
+			configurable: true,
+			enumerable: true,
+			get() {
+				return virtualBackgroundDebugConfig[key]
+			},
+			set(value) {
+				setVirtualBackgroundDebugConfigValue(key, value)
+			},
+		})
+	}
+
+	talkObject.VIRTUAL_BACKGROUND_DEBUG_CONFIG = virtualBackgroundDebugConfig
+	talkObject.VIRTUAL_BACKGROUND_DEBUG_CONFIG_DEFAULTS = DEFAULT_VIRTUAL_BACKGROUND_DEBUG_CONFIG
+	talkObject.VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES = VIRTUAL_BACKGROUND_DEBUG_CONFIG_RANGES
+
+	return talkObject
+}