From d6048ceb272c5992c7cb7887b2d9df5a643a92a0 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:39:45 +0100 Subject: [PATCH 01/11] Extract Vector3f.toDoubleArray, add toDoubleArray for Vector2f and Vector4f too --- src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt index 2521eb61..d9a738f6 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt @@ -387,10 +387,6 @@ class EyeTracking( } - private fun Vector3f.toDoubleArray(): DoubleArray { - return this.toFloatArray().map { it.toDouble() }.toDoubleArray() - } - fun DoubleArray.toVector3f(): Vector3f { require(size == 3) { "DoubleArray must have exactly 3 elements" } return Vector3f(this[0].toFloat(), this[1].toFloat(), this[2].toFloat()) From c0356fdca678dc123fd2efc982ec490cf5e8eef9 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 13:41:06 +0100 Subject: [PATCH 02/11] Extract DoubleArray.toVector3f to new ListUtils collection --- src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt index d9a738f6..816508ca 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt @@ -11,6 +11,7 @@ import graphics.scenery.ui.Column import graphics.scenery.ui.ToggleButton import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.* +import graphics.scenery.utils.toVector3f import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull import net.imglib2.type.numeric.integer.UnsignedByteType @@ -387,11 +388,6 @@ class EyeTracking( } - fun DoubleArray.toVector3f(): Vector3f { - require(size == 3) { "DoubleArray must have exactly 3 elements" } - return Vector3f(this[0].toFloat(), this[1].toFloat(), this[2].toFloat()) - } - private fun getSpinesFromHedgehog(hedgehog: InstancedNode): List { return hedgehog.instances.mapNotNull { spine -> spine.metadata["spine"] as? SpineMetadata From 7c1ece52b81b7634a4a854d97b576988890a8fb6 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 13:50:56 +0100 Subject: [PATCH 03/11] Extract HedgehogAnalysis.localMaxima to ListUtils --- .../commands/analysis/HedgehogAnalysis.kt | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt index 4f5675c2..b5ca49d2 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt @@ -5,6 +5,7 @@ import org.joml.Matrix4f import org.joml.Quaternionf import graphics.scenery.utils.extensions.* import graphics.scenery.utils.lazyLogger +import graphics.scenery.utils.localMaxima import org.slf4j.LoggerFactory import java.io.File import kotlin.collections.iterator @@ -54,24 +55,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix avgConfidence /= totalSampleCount } - /** - * From a [list] of Floats, return both the index of local maxima, and their value, - * packaged nicely as a Pair - */ - fun localMaxima(list: List): List> { - return list.windowed(3, 1).mapIndexed { index, l -> - val left = l[0] - val center = l[1] - val right = l[2] - - // we have a match at center - if (left < center && center > right) { - index * 1 + 1 to center - } else { - null - } - }.filterNotNull() - } + /** Cell positions extracted from gaze analysis are collected in this data class together with other information * such as the volume [value] at this point, and the [previous] and [next] vertices. */ From 6ceb3f0d1757e567af4f8e7a2b65569539349b0d Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 13:58:15 +0100 Subject: [PATCH 04/11] Extract HedgehogAnalysis.stdDev to ListUtils --- .../kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt index b5ca49d2..f95f209f 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt @@ -6,6 +6,7 @@ import org.joml.Quaternionf import graphics.scenery.utils.extensions.* import graphics.scenery.utils.lazyLogger import graphics.scenery.utils.localMaxima +import graphics.scenery.utils.stdDev import org.slf4j.LoggerFactory import java.io.File import kotlin.collections.iterator @@ -88,7 +89,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } } - fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { val cross = forward.cross(this) @@ -233,7 +233,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // calculate average path lengths over all val beforeCount = shortestPath.size var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + var stdDevPathLength = shortestPath.map { it.distance() }.stdDev() logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) @@ -253,7 +253,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // recalculate statistics after offending vertex removal avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + stdDevPathLength = shortestPath.map { it.distance() }.stdDev().toFloat() //step5: remove some vertices according to zscoreThreshold // var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } From ee4ec28ead7255eec3885da9707f0e13ced2da2c Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 14:10:00 +0100 Subject: [PATCH 05/11] Remove unused (and potentially incorrect) Vector3f.toQuaternion from HedgehogAnalysis --- .../kotlin/sc/iview/commands/analysis/EyeTracking.kt | 3 ++- .../sc/iview/commands/analysis/HedgehogAnalysis.kt | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt index 816508ca..12e0f10f 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt @@ -11,6 +11,7 @@ import graphics.scenery.ui.Column import graphics.scenery.ui.ToggleButton import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.* +import graphics.scenery.utils.localMaxima import graphics.scenery.utils.toVector3f import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull @@ -473,7 +474,7 @@ class EyeTracking( val smoothed = analyzer.gaussSmoothing(samples, 4) val rayMax = smoothed.max() // take the first local maximum that is at least 20% of the global maximum to prevent spot creation in noisy areas - analyzer.localMaxima(smoothed).firstOrNull {it.second > 0.2 * rayMax}?.let { (index, sample) -> + localMaxima(smoothed).firstOrNull {it.second > 0.2 * rayMax}?.let { (index, sample) -> spotPos = samplePos[index] } } diff --git a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt index f95f209f..5017ec94 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt @@ -89,16 +89,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } } - - fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { - val cross = forward.cross(this) - val q = Quaternionf(cross.x(), cross.y(), cross.z(), this.dot(forward)) - - val x = sqrt((q.w + sqrt(q.x*q.x + q.y*q.y + q.z*q.z + q.w*q.w)) / 2.0f) - - return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) - } - data class VertexWithDistance(val vertex: SpineGraphVertex, val distance: Float) fun gaussSmoothing(samples: List, iterations: Int): List { From 47e3e0dbed49fdbc7a4eb77f3aa47fa116957edc Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 14:13:19 +0100 Subject: [PATCH 06/11] Extract HedgehogAnalysis.gaussSmoothing to ListUtils --- .../commands/analysis/HedgehogAnalysis.kt | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt index 5017ec94..ee55ce38 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt @@ -4,6 +4,7 @@ import org.joml.Vector3f import org.joml.Matrix4f import org.joml.Quaternionf import graphics.scenery.utils.extensions.* +import graphics.scenery.utils.gaussSmoothing import graphics.scenery.utils.lazyLogger import graphics.scenery.utils.localMaxima import graphics.scenery.utils.stdDev @@ -91,26 +92,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix data class VertexWithDistance(val vertex: SpineGraphVertex, val distance: Float) - fun gaussSmoothing(samples: List, iterations: Int): List { - var smoothed = samples.toList() - val kernel = listOf(0.25f, 0.5f, 0.25f) - for (i in 0 until iterations) { - val newSmoothed = ArrayList(smoothed.size) - // Handle the first element - newSmoothed.add(smoothed[0] * 0.75f + smoothed[1] * 0.25f) - // Apply smoothing to the middle elements - for (j in 1 until smoothed.size - 1) { - val value = kernel[0] * smoothed[j-1] + kernel[1] * smoothed[j] + kernel[2] * smoothed[j+1] - newSmoothed.add(value) - } - // Handle the last element - newSmoothed.add(smoothed[smoothed.size - 2] * 0.25f + smoothed[smoothed.size - 1] * 0.75f) - - smoothed = newSmoothed - } - return smoothed - } - fun run(): Track? { // Adapt thresholds based on data from the first spine From 3c24a474baf91ab5b0fba3df48405f0c21cd4210 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 14:24:53 +0100 Subject: [PATCH 07/11] Move MultiButtonManager and VR2HandNodeTransform from sciview to scenery --- .../commands/analysis/CellTrackingBase.kt | 4 +- .../sc/iview/commands/analysis/EyeTracking.kt | 3 +- .../controls/behaviours/MoveInstanceVR.kt | 1 + .../controls/behaviours/MultiButtonManager.kt | 85 --------- .../behaviours/VR2HandNodeTransform.kt | 172 ------------------ .../controls/behaviours/VRGrabTheWorld.kt | 1 + 6 files changed, 6 insertions(+), 260 deletions(-) delete mode 100644 src/main/kotlin/sc/iview/controls/behaviours/MultiButtonManager.kt delete mode 100644 src/main/kotlin/sc/iview/controls/behaviours/VR2HandNodeTransform.kt diff --git a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt index 9ac615af..d010651e 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt @@ -22,8 +22,8 @@ import org.scijava.ui.behaviour.DragBehaviour import sc.iview.SciView import sc.iview.commands.analysis.HedgehogAnalysis.SpineGraphVertex import sc.iview.controls.behaviours.MoveInstanceVR -import sc.iview.controls.behaviours.MultiButtonManager -import sc.iview.controls.behaviours.VR2HandNodeTransform +import graphics.scenery.controls.behaviours.MultiButtonManager +import graphics.scenery.controls.behaviours.VR2HandNodeTransform import sc.iview.controls.behaviours.VRGrabTheWorld import java.io.BufferedWriter import java.io.FileWriter diff --git a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt index 12e0f10f..d39edbaa 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt @@ -11,6 +11,7 @@ import graphics.scenery.ui.Column import graphics.scenery.ui.ToggleButton import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.* +import graphics.scenery.utils.gaussSmoothing import graphics.scenery.utils.localMaxima import graphics.scenery.utils.toVector3f import kotlinx.coroutines.runBlocking @@ -471,7 +472,7 @@ class EyeTracking( val (samples, samplePos) = sampleRayThroughVolume(origin, direction, volume) var spotPos: Vector3f? = null if (samples != null && samplePos != null) { - val smoothed = analyzer.gaussSmoothing(samples, 4) + val smoothed = gaussSmoothing(samples, 4) val rayMax = smoothed.max() // take the first local maximum that is at least 20% of the global maximum to prevent spot creation in noisy areas localMaxima(smoothed).firstOrNull {it.second > 0.2 * rayMax}?.let { (index, sample) -> diff --git a/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt b/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt index 049d75ba..87f68aae 100644 --- a/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt +++ b/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt @@ -4,6 +4,7 @@ import graphics.scenery.Scene import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.MultiButtonManager import org.joml.Vector3f import org.scijava.ui.behaviour.DragBehaviour import kotlin.collections.forEach diff --git a/src/main/kotlin/sc/iview/controls/behaviours/MultiButtonManager.kt b/src/main/kotlin/sc/iview/controls/behaviours/MultiButtonManager.kt deleted file mode 100644 index 488c87aa..00000000 --- a/src/main/kotlin/sc/iview/controls/behaviours/MultiButtonManager.kt +++ /dev/null @@ -1,85 +0,0 @@ -package sc.iview.controls.behaviours - -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackerRole -import graphics.scenery.utils.lazyLogger -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicBoolean - -/** Keep track of which VR buttons are currently being pressed. This is useful if you want to assign the same button - * to different behaviors with different combinations. This class helps with managing the button states. - * Buttons to track first need to be registered with [registerButtonConfig]. Call [pressButton] and [releaseButton] - * in your behavior init/end methods. You can check if both hands are in use with [isTwoHandedActive] or if a specific - * button is currently pressed with [isButtonPressed]. */ -class MultiButtonManager { - data class ButtonConfig ( - val button: OpenVRHMD.OpenVRButton, - val trackerRole: TrackerRole - ) - - val logger by lazyLogger() - - /** List of registered buttons, stored as [ButtonConfig] and whether the button is pressed right now. */ - private val buttons = ConcurrentHashMap() - private val twoHandedActive = AtomicBoolean(false) - - init { - buttons.forEach { (config, value) -> - buttons[config] = false - } - } - - /** Add a new button configuration that the manager will keep track of. */ - fun registerButtonConfig(button: OpenVRHMD.OpenVRButton, trackerRole: TrackerRole) { - logger.debug("Registered new button config: $button, $trackerRole") - buttons[ButtonConfig(button, trackerRole)] = false - } - - /** Add a button to the list of pressed buttons. */ - fun pressButton(button: OpenVRHMD.OpenVRButton, role: TrackerRole): Boolean { - val config = ButtonConfig(button, role) - if (!buttons.containsKey(config)) { return false } - buttons[config] = true - updateTwoHandedState() - return true - } - - /** Overload function that takes a button config instead of separate button and trackerrole inputs. */ - fun pressButton(buttonConfig: ButtonConfig): Boolean { - return pressButton(buttonConfig.button, buttonConfig.trackerRole) - } - - /** Remove a button from the list of pressed buttons. */ - fun releaseButton(button: OpenVRHMD.OpenVRButton, role: TrackerRole): Boolean { - val config = ButtonConfig(button, role) - if (!buttons.containsKey(config)) { return false } - buttons[config] = false - updateTwoHandedState() - return true - } - - /** Overload function that takes a button config instead of separate button and trackerrole inputs. */ - fun releaseButton(buttonConfig: ButtonConfig): Boolean { - return releaseButton(buttonConfig.button, buttonConfig.trackerRole) - } - - private fun updateTwoHandedState() { - // Check if any buttons are pressed on both hands - val leftPressed = buttons.any { it.key.trackerRole == TrackerRole.LeftHand && it.value } - val rightPressed = buttons.any { it.key.trackerRole == TrackerRole.RightHand && it.value } - twoHandedActive.set(leftPressed && rightPressed) - } - - /** Returns true when the same button is currently pressed on both VR controllers. */ - fun isTwoHandedActive(): Boolean = twoHandedActive.get() - - /** Check if a button is currently being pressed. */ - fun isButtonPressed(button: OpenVRHMD.OpenVRButton, role: TrackerRole): Boolean { - return buttons[ButtonConfig(button, role)] ?: false - } - - /** Retrieve a list of currently registered buttons. */ - fun getRegisteredButtons(): ConcurrentHashMap { - return buttons - } -} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/controls/behaviours/VR2HandNodeTransform.kt b/src/main/kotlin/sc/iview/controls/behaviours/VR2HandNodeTransform.kt deleted file mode 100644 index 2f3d1a03..00000000 --- a/src/main/kotlin/sc/iview/controls/behaviours/VR2HandNodeTransform.kt +++ /dev/null @@ -1,172 +0,0 @@ -package sc.iview.controls.behaviours - -import graphics.scenery.Node -import graphics.scenery.Scene -import graphics.scenery.attribute.spatial.Spatial -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.VRScale -import graphics.scenery.controls.behaviours.VRTwoHandDragBehavior -import graphics.scenery.controls.behaviours.VRTwoHandDragOffhand -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.plus -import graphics.scenery.utils.extensions.times -import org.joml.Quaternionf -import org.joml.Vector3f -import sc.iview.controls.behaviours.VRGrabTheWorld.Companion.createAndSet -import java.util.concurrent.CompletableFuture - - -/** Transform a target node [target] by pressing the same buttons defined in [createAndSet] on both VR controllers. - * The fastest way to attach the behavior is by using [createAndSet]. - * [onEndCallback] is an optional lambda that is executed once the behavior ends. - * @author Jan Tiemann - * @author Samuel Pantze */ -class VR2HandNodeTransform( - name: String, - controller: Spatial, - offhand: VRTwoHandDragOffhand, - val scene: Scene, - val scaleLocked: Boolean = false, - val rotationLocked: Boolean = false, - val positionLocked: Boolean = false, - val lockYaxis: Boolean = true, - val target: Node, - private val onStartCallback: (() -> Unit)? = null, - private val onDragCallback: (() -> Unit)? = null, - private val onEndCallback: (() -> Unit)? = null, - private val resetRotationBtnManager: MultiButtonManager? = null, - private val resetRotationButton: MultiButtonManager.ButtonConfig? = null, -) : VRTwoHandDragBehavior(name, controller, offhand) { - - /** To trigger the [onStartCallback] regardless of which order of buttons was used. */ - private var startCallbackTriggered = false - - override fun init(x: Int, y: Int) { - super.init(x, y) - // Find the button that doesn't lock the y Axis and indicate that it is now pressed - val transformBtn = - resetRotationBtnManager?.getRegisteredButtons() - ?.filter { it.key != resetRotationButton }?.map { it.key }?.firstOrNull() - if (transformBtn != null) { - resetRotationBtnManager?.pressButton(transformBtn) - } - if (bothPressed) { - onStartCallback?.invoke() - startCallbackTriggered = true - } - } - - override fun dragDelta( - currentPositionMain: Vector3f, - currentPositionOff: Vector3f, - lastPositionMain: Vector3f, - lastPositionOff: Vector3f - ) { - - // Test whether we now press both buttons but the startCallback wasn't triggered yet. - if (bothPressed && !startCallbackTriggered) { - onStartCallback?.invoke() - startCallbackTriggered = true - } - - val scaleDelta = - VRScale.getScaleDelta(currentPositionMain, currentPositionOff, lastPositionMain, lastPositionOff) - - val currentDirection = (currentPositionMain - currentPositionOff).normalize() - val lastDirection = (lastPositionMain - lastPositionOff).normalize() - if (lockYaxis) { - lastDirection.y = 0f - currentDirection.y = 0f - } - - // Rotation implementation: https://discussions.unity.com/t/two-hand-grabbing-of-objects-in-virtual-reality/219972 - - target.let { - if (!rotationLocked) { - it.ifSpatial { - val rotationDelta = Quaternionf().rotationTo(lastDirection, currentDirection) - if (resetRotationBtnManager?.isTwoHandedActive() == true) { - // Reset the rotation when the reset button was pressed too - rotation = Quaternionf() - } else { - // Rotate node with respect to the world space delta - rotation = Quaternionf(rotationDelta).mul(Quaternionf(rotation)) - } - } - } - if (!scaleLocked) { - target.ifSpatial { - scale *= scaleDelta - } - } - if (!positionLocked) { - val positionDelta = - (currentPositionMain + currentPositionOff) / 2f - (lastPositionMain + lastPositionOff) / 2f - target.ifSpatial { - position.add(positionDelta) - } - } - } - onDragCallback?.invoke() - } - - override fun end(x: Int, y: Int) { - super.end(x, y) - onEndCallback?.invoke() - // Find the button that doesn't lock the y Axis and indicate that it is now released - val transformBtn = resetRotationBtnManager?.getRegisteredButtons()?.filter { it.key != resetRotationButton }?.map {it.key}?.firstOrNull() - if (transformBtn != null) { - resetRotationBtnManager?.releaseButton(transformBtn) - } - // Reset this flag for the next event - startCallbackTriggered = false - } - - companion object { - /** - * Convenience method for adding scale behaviour - */ - fun createAndSet( - hmd: OpenVRHMD, - button: OpenVRHMD.OpenVRButton, - scene: Scene, - scaleLocked: Boolean = false, - rotationLocked: Boolean = false, - positionLocked: Boolean = false, - lockYaxis: Boolean = true, - target: Node, - onStartCallback: (() -> Unit)? = null, - onDragCallback: (() -> Unit)? = null, - onEndCallback: (() -> Unit)? = null, - resetRotationBtnManager: MultiButtonManager? = null, - resetRotationButton: MultiButtonManager.ButtonConfig? = null, - ): CompletableFuture { - @Suppress("UNCHECKED_CAST") return createAndSet( - hmd, button - ) { controller: Spatial, offhand: VRTwoHandDragOffhand -> - // Assign the yLock button and the right grab button to the button manager to handle multi-button events - resetRotationButton?.let { - resetRotationBtnManager?.registerButtonConfig(it.button, it.trackerRole) - } - resetRotationBtnManager?.registerButtonConfig(button, TrackerRole.RightHand) - VR2HandNodeTransform( - "Scaling", - controller, - offhand, - scene, - scaleLocked, - rotationLocked, - positionLocked, - lockYaxis, - target, - onStartCallback, - onDragCallback, - onEndCallback, - resetRotationBtnManager, - resetRotationButton - ) - } as CompletableFuture - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt b/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt index 347f25bc..cb1a2b95 100644 --- a/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt +++ b/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt @@ -6,6 +6,7 @@ import graphics.scenery.attribute.spatial.Spatial import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.MultiButtonManager import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.plusAssign import graphics.scenery.utils.extensions.times From aa959a51fba65f851ef8fe47d0c65b6d422e555c Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 30 Jan 2026 15:00:15 +0100 Subject: [PATCH 08/11] Rename all mastodon-sciview-bridge occurences to manvr3d --- .../kotlin/sc/iview/commands/analysis/CellTrackingBase.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt index d010651e..0274d75c 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt @@ -73,7 +73,7 @@ open class CellTrackingBase( /** Takes a list of [SpineGraphVertex] and its positions to create the corresponding track in Mastodon. * In the case of controller tracking, the points were already sent to Mastodon one by one via [singleLinkTrackedCallback] and the list is not needed. - * Set the first boolean to true if the coordinates are in world space. The bridge will convert them to Mastodon coords. + * Set the first boolean to true if the coordinates are in world space. The manvr3d will convert them to Mastodon coords. * The first Spot defines whether to start with an existing spot, so the lambda will use that as starting point. * The second spot defines whether we want to merge into this spot. */ var trackCreationCallback: (( @@ -84,7 +84,7 @@ open class CellTrackingBase( mergeSpot: Spot? ) -> Unit)? = null - /** Passes the current time point, a position and a radius to the bridge to either create a new spot + /** Passes the current time point, a position and a radius to manvr3d to either create a new spot * or to delete an existing spot if there is a spot selected. * The deleteBranch flag indicates whether we want to delete the whole branch or just a spot. * isVoxelCoords indicates whether the coordinates are in sciview or in mastodon space */ @@ -104,7 +104,7 @@ open class CellTrackingBase( /** Links a selected spot to the closest spot to handle merge events. */ var spotLinkCallback: (() -> Unit)? = null /** Generates a single link between a new position and the previously annotated one. - * Sends the position data to the bridge for intermediary keeping. The integer is the timepoint. + * Sends the position data to manvr3d for intermediary keeping. The integer is the timepoint. * The Float contains the cursor's radius in sciview space. * The boolean specifies whether the link preview should be rendered. */ var singleLinkTrackedCallback: ((pos: Vector3f, tp: Int, radius: Float, preview: Boolean) -> Unit)? = null From bdc51f232eb3ab1713b079baf23e1387ddf5967c Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 2 Feb 2026 10:06:19 +0100 Subject: [PATCH 09/11] Move VRGrabTheWorld from sciview to scenery --- .../commands/analysis/CellTrackingBase.kt | 2 +- .../controls/behaviours/VRGrabTheWorld.kt | 97 ------------------- 2 files changed, 1 insertion(+), 98 deletions(-) delete mode 100644 src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt diff --git a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt index 0274d75c..05b6c9dd 100644 --- a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt @@ -24,7 +24,7 @@ import sc.iview.commands.analysis.HedgehogAnalysis.SpineGraphVertex import sc.iview.controls.behaviours.MoveInstanceVR import graphics.scenery.controls.behaviours.MultiButtonManager import graphics.scenery.controls.behaviours.VR2HandNodeTransform -import sc.iview.controls.behaviours.VRGrabTheWorld +import graphics.scenery.controls.behaviours.VRGrabTheWorld import java.io.BufferedWriter import java.io.FileWriter import java.nio.file.Path diff --git a/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt b/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt deleted file mode 100644 index cb1a2b95..00000000 --- a/src/main/kotlin/sc/iview/controls/behaviours/VRGrabTheWorld.kt +++ /dev/null @@ -1,97 +0,0 @@ -package sc.iview.controls.behaviours - -import graphics.scenery.Node -import graphics.scenery.Scene -import graphics.scenery.attribute.spatial.Spatial -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.MultiButtonManager -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.plusAssign -import graphics.scenery.utils.extensions.times -import org.joml.Vector3f -import org.scijava.ui.behaviour.DragBehaviour - - -/** Move yourself (the scene camera) by pressing a VR button. - * The fastest way to attach the behavior is by using [createAndSet]. - * You can pass a [grabButtonmanager] to handle multiple assignments per button. - * @author Jan Tiemann - * @author Samuel Pantze */ -class VRGrabTheWorld ( - @Suppress("UNUSED_PARAMETER") name: String, - controllerHitbox: Node, - private val cam: Spatial, - private val grabButtonmanager: MultiButtonManager? = null, - val button: OpenVRHMD.OpenVRButton, - private val trackerRole: TrackerRole, - private val multiplier: Float -) : DragBehaviour { - - private var camDiff = Vector3f() - - private val controllerSpatial: Spatial = controllerHitbox.spatialOrNull() - ?: throw IllegalArgumentException("controller hitbox needs a spatial attribute") - - - override fun init(x: Int, y: Int) { - grabButtonmanager?.pressButton(button, trackerRole) - camDiff = controllerSpatial.worldPosition() - cam.position - } - - override fun drag(x: Int, y: Int) { - // Only drag when no other grab button is currently active - // to prevent simultaneous behaviors with two-handed gestures - if (grabButtonmanager?.isTwoHandedActive() != true) { - //grabbed world - val newCamDiff = controllerSpatial.worldPosition() - cam.position - val diffTranslation = camDiff - newCamDiff //reversed - cam.position += diffTranslation * multiplier - camDiff = newCamDiff - } - } - - override fun end(x: Int, y: Int) { - grabButtonmanager?.releaseButton(button, trackerRole) - } - - companion object { - - /** - * Convenience method for adding grab behaviour - */ - fun createAndSet( - scene: Scene, - hmd: OpenVRHMD, - buttons: List, - controllerSide: List, - buttonManager: MultiButtonManager? = null, - multiplier: Float = 1f - ) { - hmd.events.onDeviceConnect.add { _, device, _ -> - if (device.type == TrackedDeviceType.Controller) { - device.model?.let { controller -> - if (controllerSide.contains(device.role)) { - buttons.forEach { button -> - val name = "VRDrag:${hmd.trackingSystemName}:${device.role}:$button" - val grabBehaviour = VRGrabTheWorld( - name, - controller.children.first(), - scene.findObserver()!!.spatial(), - buttonManager, - button, - device.role, - multiplier - ) - buttonManager?.registerButtonConfig(button, device.role) - hmd.addBehaviour(name, grabBehaviour) - hmd.addKeyBinding(name, device.role, button) - } - } - } - } - } - } - } -} \ No newline at end of file From 932b59e6ce6284e56522cca3069b45d42406ae13 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:12:21 +0100 Subject: [PATCH 10/11] Move CelltrackingBase, Buttonmanager, HedgehogAnalysis, EyeTracking, Spinemetadata and MoveInstanceVR from sciview to manvr3d --- .../commands/analysis/CellTrackingBase.kt | 1122 ----------------- .../analysis/CellTrackingButtonMapper.kt | 164 --- .../sc/iview/commands/analysis/EyeTracking.kt | 509 -------- .../commands/analysis/HedgehogAnalysis.kt | 410 ------ .../iview/commands/analysis/SpineMetadata.kt | 23 - .../controls/behaviours/MoveInstanceVR.kt | 86 -- 6 files changed, 2314 deletions(-) delete mode 100644 src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt delete mode 100644 src/main/kotlin/sc/iview/commands/analysis/CellTrackingButtonMapper.kt delete mode 100644 src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt delete mode 100644 src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt delete mode 100644 src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt delete mode 100644 src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt diff --git a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt deleted file mode 100644 index 05b6c9dd..00000000 --- a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingBase.kt +++ /dev/null @@ -1,1122 +0,0 @@ -package sc.iview.commands.analysis - -import graphics.scenery.* -import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.* -import graphics.scenery.controls.behaviours.AnalogInputWrapper -import graphics.scenery.controls.behaviours.ConfirmableClickBehaviour -import graphics.scenery.controls.behaviours.VRTouch -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import graphics.scenery.ui.* -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.* -import graphics.scenery.utils.lazyLogger -import graphics.scenery.volumes.RAIVolume -import graphics.scenery.volumes.Volume -import org.joml.* -import org.mastodon.mamut.model.Spot -import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.ui.behaviour.DragBehaviour -import sc.iview.SciView -import sc.iview.commands.analysis.HedgehogAnalysis.SpineGraphVertex -import sc.iview.controls.behaviours.MoveInstanceVR -import graphics.scenery.controls.behaviours.MultiButtonManager -import graphics.scenery.controls.behaviours.VR2HandNodeTransform -import graphics.scenery.controls.behaviours.VRGrabTheWorld -import java.io.BufferedWriter -import java.io.FileWriter -import java.nio.file.Path -import java.util.ArrayList -import java.util.concurrent.atomic.AtomicInteger -import kotlin.concurrent.thread - -/** - * Base class for different VR cell tracking purposes. It includes functionality to add spines and edgehogs, - * as used by [EyeTracking], and registers controller bindings via [inputSetup]. It is possible to register observers - * that listen to timepoint changes with [registerObserver]. - * @param [sciview] The [SciView] instance to use - */ -open class CellTrackingBase( - open var sciview: SciView, - val resolutionScale: Float = 1f -) { - val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - - lateinit var hmd: OpenVRHMD - - val hedgehogs = Mesh() - val hedgehogIds = AtomicInteger(0) - lateinit var volume: Volume - - val referenceTarget = Icosphere(0.004f, 2) - - @Volatile var eyeTrackingActive = false - var playing = false - var direction = PlaybackDirection.Backward - var volumesPerSecond = 6f - var skipToNext = false - var skipToPrevious = false - - var volumeScaleFactor = 1.0f - - private lateinit var lightTetrahedron: List - - val volumeTimepointWidget = TextBoard() - - /** determines whether the volume and hedgehogs should keep listening for updates or not */ - var cellTrackingActive: Boolean = false - - /** Takes a list of [SpineGraphVertex] and its positions to create the corresponding track in Mastodon. - * In the case of controller tracking, the points were already sent to Mastodon one by one via [singleLinkTrackedCallback] and the list is not needed. - * Set the first boolean to true if the coordinates are in world space. The manvr3d will convert them to Mastodon coords. - * The first Spot defines whether to start with an existing spot, so the lambda will use that as starting point. - * The second spot defines whether we want to merge into this spot. */ - var trackCreationCallback: (( - List>?, - radius: Float, - isWorldSpace: Boolean, - startSpot: Spot?, - mergeSpot: Spot? - ) -> Unit)? = null - - /** Passes the current time point, a position and a radius to manvr3d to either create a new spot - * or to delete an existing spot if there is a spot selected. - * The deleteBranch flag indicates whether we want to delete the whole branch or just a spot. - * isVoxelCoords indicates whether the coordinates are in sciview or in mastodon space */ - var spotCreateDeleteCallback: (( - tp: Int, - sciviewPos: Vector3f, - radius: Float, - deleteBranch: Boolean, - isWorldSpace: Boolean - ) -> Unit)? = null - /** Select a spot based on the controller tip's position, current time point and a multiple of the radius - * in which a selection event is counted as valid. addOnly prevents deselection from clicking away. */ - var spotSelectCallback: ((sciviewPos: Vector3f, tp: Int, radiusFactor: Float, addOnly: Boolean) -> Pair)? = null - var spotMoveInitCallback: ((Vector3f) -> Unit)? = null - var spotMoveDragCallback: ((Vector3f) -> Unit)? = null - var spotMoveEndCallback: ((Vector3f) -> Unit)? = null - /** Links a selected spot to the closest spot to handle merge events. */ - var spotLinkCallback: (() -> Unit)? = null - /** Generates a single link between a new position and the previously annotated one. - * Sends the position data to manvr3d for intermediary keeping. The integer is the timepoint. - * The Float contains the cursor's radius in sciview space. - * The boolean specifies whether the link preview should be rendered. */ - var singleLinkTrackedCallback: ((pos: Vector3f, tp: Int, radius: Float, preview: Boolean) -> Unit)? = null - var toggleTrackingPreviewCallback: ((Boolean) -> Unit)? = null - var rebuildGeometryCallback: (() -> Unit)? = null - - var stageSpotsCallback: (() -> Unit)? = null - var predictSpotsCallback: ((all: Boolean) -> Unit)? = null - var trainSpotsCallback: (() -> Unit)? = null - var neighborLinkingCallback: (() -> Unit)? = null - // TODO add train flow functionality - var trainFlowCallback: (() -> Unit)? = null - /** Reverts to the point previously saved by Mastodon's undo recorder. Also handles redo events if undo is set to false. */ - var mastodonUndoRedoCallback: ((undo: Boolean) -> Unit)? = null - /** Returns a list of spots currently selected in Mastodon. Used to determine whether to scale the cursor or the spots. */ - var getSelectionCallback: (() -> List)? = null - /** Adjusts the radii of spots, both in sciview and Mastodon. */ - var scaleSpotsCallback: ((radius: Float, update: Boolean) -> Unit)? = null - /** Toggle the visibility of spots in the scene. */ - var setSpotVisCallback: ((Boolean) -> Unit)? = null - /** Toggle the visibility of tracks in the scene. */ - var setTrackVisCallback: ((Boolean) -> Unit)? = null - /** Toggle the visiblity of the volume in the scene while maintaining visibility of spots and links as child elements. */ - var setVolumeVisCallback: ((Boolean) -> Unit)? = null - /** Merges overlapping spots in a given timepoint. */ - var mergeOverlapsCallback: ((Int) -> Unit)? = null - /** Merges selected spots. */ - var mergeSelectedCallback: (() -> Unit)? = null - /** Deletes the whole graph. */ - var deleteGraphCallback: (() -> Unit)? = null - /** Deletes all annotations from this timepoint. */ - var deleteTimepointCallback: (() -> Unit)? = null - /** Recenter and set default scaling for volume, then center camera on volume. */ - var resetViewCallback: (() -> Unit)? = null - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - - enum class PlaybackDirection { Forward, Backward } - - enum class ElephantMode { StageSpots, TrainAll, PredictTP, PredictAll, NNLinking } - - var hedgehogVisibility = HedgehogVisibility.Hidden - var trackVisibility = true - var spotVisibility = true - - var leftVRController: TrackedDevice? = null - var rightVRController: TrackedDevice? = null - - var cursor = CursorTool - var leftElephantColumn: Column? = null - var generalMenu: Column? = null - var enableTrackingPreview = true - val leftMenuList = mutableListOf() - var leftMenuIndex = 0 - - val grabButtonManager = MultiButtonManager() - val resetRotationBtnManager = MultiButtonManager() - - val mapper = CellTrackingButtonMapper - - private val observers = mutableListOf() - - open fun run() { - sciview.toggleVRRendering(resolutionScale = resolutionScale) - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - - // Try to load the correct button mapping corresponding to the controller layout - val isProfileLoaded = mapper.loadProfile(hmd.manufacturer) - if (!isProfileLoaded) { - throw IllegalStateException("Could not load profile, headset type unknown!") - } - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial { - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) - } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - shell.name = "Shell" - sciview.addNode(shell) - - lightTetrahedron = Light.createLightTetrahedron( - Vector3f(0.0f, 0.0f, 0.0f), - spread = 5.0f, - radius = 15.0f, - intensity = 5.0f - ) - lightTetrahedron.forEach { sciview.addNode(it) } - - val volumeNodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } - - val v = (volumeNodes.firstOrNull() as? Volume) - if(v == null) { - logger.warn("No volume found, bailing") - return - } else { - logger.info("found ${volumeNodes.size} volume nodes. Using the first one: ${volumeNodes.first()}") - volume = v - } - - logger.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - logger.info("onDeviceConnect called, cam=${sciview.camera}") - if (device.type == TrackedDeviceType.Controller) { - logger.info("Got device ${device.name} at $timestamp") - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - when (device.role) { - TrackerRole.Invalid -> {} - TrackerRole.LeftHand -> leftVRController = device - TrackerRole.RightHand -> rightVRController = device - } - if (device.role == TrackerRole.RightHand) { - attachCursorAndTimepointWidget() - device.model?.name = "rightHand" - } else if (device.role == TrackerRole.LeftHand) { - device.model?.name = "leftHand" - setupElephantMenu() - setupGeneralMenu() - - logger.info("Set up navigation and editing controls.") - } - } - } - inputSetup() - - cellTrackingActive = true - rebuildGeometryCallback?.invoke() - launchUpdaterThread() - } - - /** Registers a new observer that will get updated whenever the VR user triggers a timepoint update. */ - fun registerObserver(observer: TimepointObserver) { - observers.add(observer) - } - - /** Unregisters the timepoint observer. */ - fun unregisterObserver(observer: TimepointObserver) { - observers.remove(observer) - } - - /** Notifies all active observers of a change of timepoint. */ - private fun notifyObservers(timepoint: Int) { - observers.forEach { it.onTimePointChanged(timepoint) } - } - - /** Attaches a column of [Gui3DElement]s to the left VR controller and adds the column to [leftMenuList]. */ - protected fun createWristMenuColumn( - vararg elements: Gui3DElement, - debug: Boolean = false, - name: String = "Menu" - ): Column { - val column = Column(*elements, centerVertically = true, centerHorizontally = true) - column.ifSpatial { - scale = Vector3f(0.05f) - position = Vector3f(0.05f, 0.05f, column.height / 20f + 0.1f) - rotation = Quaternionf().rotationXYZ(-1.57f, 1.57f, 0f) - } - leftVRController?.model?.let { - sciview.addNode(column, parent = it, activePublish = false) - if (debug) { - column.children.forEach { child -> - val bb = BoundingGrid() - bb.node = child - bb.gridColor = Vector3f(0.5f, 1f, 0.4f) - sciview.addNode(bb, parent = it) - } - } - } - column.name = name - column.pack() - leftMenuList.add(column) - return column - } - - var controllerTrackingActive = false - - /** Intermediate storage for a single track created with the controllers. - * Once tracking is finished, this track is sent to Mastodon. */ - var controllerTrackList = mutableListOf() - var startWithExistingSpot: Spot? = null - - /** This lambda is called every time the user performs a click with controller-based tracking. */ - val trackCellsWithController = ClickBehaviour { _, _ -> - if (!controllerTrackingActive) { - controllerTrackingActive = true - cursor.activateTrackingColor() - // we dont want animation, because we track step by step - playing = false - // Assume the user didn't click on an existing spot to start the track. - startWithExistingSpot = null - } - // play the volume backwards, step by step, so cell split events can simply be turned into a merge event - if (volume.currentTimepoint > 0) { - val p = cursor.getPosition() - // did the user click on an existing cell and wants to merge the track into it? - val (selected, isValidSelection) = - spotSelectCallback?.invoke(p, volume.currentTimepoint, cursor.radius, false) ?: (null to false) - // If this is the first spot we track, and its a valid existing spot, mark it as such - if (isValidSelection && controllerTrackList.size == 0) { - startWithExistingSpot = selected - logger.debug("Set startWithExistingPost to $startWithExistingSpot") - } else { - controllerTrackList.add(p) - } - logger.debug("Tracked a new spot at position $p") - logger.debug("Do we want to merge? $isValidSelection. Selected spot is $selected") - // Create a placeholder link during tracking for immediate feedback - singleLinkTrackedCallback?.invoke(p, volume.currentTimepoint, cursor.radius, enableTrackingPreview) - - volume.goToTimepoint(volume.currentTimepoint - 1) - // If the user clicked a cell and its *not* the first in the track, we assume it is a merge event and end the tracking - if (isValidSelection && controllerTrackList.size > 1) { - endControllerTracking(selected) - } - // This will also redraw all geometry using Mastodon as source - notifyObservers(volume.currentTimepoint) - } else { - sciview.camera?.showMessage("Reached the first time point!", centered = true, distance = 2f, size = 0.2f) - // Let's head back to the last timepoint for starting a new track fast-like - volume.goToLastTimepoint() - endControllerTracking() - } - } - - /** Stops the current controller tracking process and sends the created track to Mastodon. */ - private fun endControllerTracking(mergeSpot: Spot? = null) { - if (controllerTrackingActive) { - logger.info("Ending controller tracking now and sending ${controllerTrackList.size} spots to Mastodon to chew on.") - controllerTrackingActive = false - // Radius can be 0 because the actual radii were already captured during tracking - trackCreationCallback?.invoke(null, 0f, true, startWithExistingSpot, mergeSpot) - controllerTrackList.clear() - cursor.resetColor() - } - } - - fun setupElephantMenu() { - val unpressedColor = Vector3f(0.81f, 0.81f, 1f) - val touchingColor = Vector3f(0.7f, 0.65f, 1f) - val pressedColor = Vector3f(0.54f, 0.44f, 0.96f) - val stageSpotsButton = Button( - "Stage all", - command = { updateElephantActions(ElephantMode.StageSpots) }, byTouch = true, depressDelay = 500, - color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) - val trainAllButton = Button( - "Train All TPs", - command = { updateElephantActions(ElephantMode.TrainAll) }, byTouch = true, depressDelay = 500, - color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) - val predictAllButton = Button( - "Predict All", - command = { updateElephantActions(ElephantMode.PredictAll) }, byTouch = true, depressDelay = 500, - color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) - val predictTPButton = Button( - "Predict TP", - command = { updateElephantActions(ElephantMode.PredictTP) }, byTouch = true, depressDelay = 500, - color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) - val linkingButton = Button( - "NN linking", - command = { updateElephantActions(ElephantMode.NNLinking) }, byTouch = true, depressDelay = 500, - color = unpressedColor, touchingColor = touchingColor, pressedColor = pressedColor) - - leftElephantColumn = - createWristMenuColumn(stageSpotsButton, trainAllButton, predictTPButton, predictAllButton, linkingButton, name = "Stage Menu") - leftElephantColumn?.visible = false - } - - var lastButtonTime = System.currentTimeMillis() - - /** Ensure that only a single Elephant action is triggered at a time */ - private fun updateElephantActions(mode: ElephantMode) { - val buttonTime = System.currentTimeMillis() - - if ((buttonTime - lastButtonTime) > 1000) { - - thread { - when (mode) { - ElephantMode.StageSpots -> stageSpotsCallback?.invoke() - ElephantMode.TrainAll -> trainSpotsCallback?.invoke() - ElephantMode.PredictTP -> predictSpotsCallback?.invoke(false) - ElephantMode.PredictAll -> predictSpotsCallback?.invoke(true) - ElephantMode.NNLinking -> neighborLinkingCallback?.invoke() - } - - logger.info("We locked the buttons for ${(buttonTime-lastButtonTime)} ms ") - lastButtonTime = buttonTime - } - - } else { - sciview.camera?.showMessage("Have some patience!", duration = 1500, distance = 2f, size = 0.2f, centered = true) - } - - } - - fun setupGeneralMenu() { - - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - val color = Vector3f(0.8f) - val pressedColor = Vector3f(0.95f, 0.35f, 0.25f) - val touchingColor = Vector3f(0.7f, 0.55f, 0.55f) - - val undoButton = Button( - "Undo", - command = { mastodonUndoRedoCallback?.invoke(true) }, byTouch = true, depressDelay = 250, - color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val redoButton = Button( - "Redo", - command = {mastodonUndoRedoCallback?.invoke(false)}, byTouch = true, depressDelay = 250, - color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val toggleTrackingPreviewBtn = ToggleButton( - "Preview Off", "Preview On", command = { - enableTrackingPreview = !enableTrackingPreview - toggleTrackingPreviewCallback?.invoke(enableTrackingPreview) - }, byTouch = true, - color = color, - touchingColor = Vector3f(0.67f, 0.9f, 0.63f), - pressedColor = Vector3f(0.35f, 0.95f, 0.25f), - default = true - ) - val togglePlaybackDirBtn = ToggleButton( - textFalse = "BW", textTrue = "FW", command = { - direction = if (direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - }, byTouch = true, - color = Vector3f(0.52f, 0.87f, 0.86f), - touchingColor = color, - pressedColor = Vector3f(0.84f, 0.87f, 0.52f) - ) - val playSlowerBtn = Button( - "<", command = { - volumesPerSecond = maxOf(volumesPerSecond - 1f, 1f) - cam.showMessage( - "Speed: ${"%.0f".format(volumesPerSecond)} vol/s", - distance = 1.2f, size = 0.2f, centered = true - ) - }, byTouch = true, depressDelay = 250, - color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val playFasterBtn = Button( - ">", command = { - volumesPerSecond = minOf(volumesPerSecond + 1f, 20f) - cam.showMessage( - "Speed: ${"%.0f".format(volumesPerSecond)} vol/s", - distance = 1.2f, size = 0.2f, centered = true - ) - }, byTouch = true, depressDelay = 250, - color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val goToLastBtn = Button( - ">|", command = { - playing = false - volume.goToLastTimepoint() - notifyObservers(volume.currentTimepoint) - cam.showMessage("Jumped to timepoint ${volume.currentTimepoint}.", - distance = 1.2f, size = 0.2f, centered = true) - }, byTouch = true, depressDelay = 250, - color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val goToFirstBtn = Button( - "|<", command = { - playing = false - volume.goToFirstTimepoint() - notifyObservers(volume.currentTimepoint) - cam.showMessage("Jumped to timepoint ${volume.currentTimepoint}.", - distance = 1.2f, size = 0.2f, centered = true) - }, byTouch = true, depressDelay = 250, - color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - - val resetViewButton = Button( - "Recenter", command = { - resetViewCallback?.invoke() - }, byTouch = true, depressDelay = 250, - color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - - val timeControlRow = Row(goToFirstBtn, playSlowerBtn, togglePlaybackDirBtn, playFasterBtn, goToLastBtn) - val undoRedoRow = Row(undoButton, redoButton, resetViewButton) - generalMenu = createWristMenuColumn(timeControlRow, undoRedoRow, name = "Left Undo Menu") - generalMenu?.visible = false - - val toggleVolume = ToggleButton( - "Volume off", "Volume on", command = { - val state = volume.visible - setVolumeVisCallback?.invoke(!state) - }, byTouch = true, - color = color, pressedColor = pressedColor, touchingColor = touchingColor, default = true - ) - val toggleTracks = ToggleButton( - "Track off", "Track on", - command = { - trackVisibility = !trackVisibility - setTrackVisCallback?.invoke(trackVisibility) - }, - byTouch = true, color = color, pressedColor = pressedColor, touchingColor = touchingColor, default = true - ) - val toggleSpots = ToggleButton( - "Spots off", "Spots on", - command = { - spotVisibility = !spotVisibility - setSpotVisCallback?.invoke(spotVisibility) - }, - byTouch = true, color = color, pressedColor = pressedColor, touchingColor = touchingColor, default = true - ) - val toggleVisMenu = createWristMenuColumn(toggleVolume, toggleTracks, toggleSpots, toggleTrackingPreviewBtn) - toggleVisMenu.visible = false - - val mergeOverlapsButton = Button( - "Merge overlaps", command = { - mergeOverlapsCallback?.invoke(volume.currentTimepoint) - }, byTouch = true, depressDelay = 250, color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val mergeSelectedButton = Button( - "Merge selected", command = { - mergeSelectedCallback?.invoke() - }, byTouch = true, depressDelay = 250, color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - - val deleteGraphButton = Button( - "Delete Graph", command = { - deleteGraphCallback?.invoke() - }, byTouch = true, depressDelay = 250, color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val deleteTimepointButton = Button( - "Delete TP", command = { - deleteTimepointCallback?.invoke() - }, byTouch = true, depressDelay = 250, color = color, pressedColor = pressedColor, touchingColor = touchingColor - ) - val mergeRow = Row(mergeOverlapsButton, mergeSelectedButton) - val deleteRow = Row(deleteGraphButton, deleteTimepointButton) - val cleanupMenu = createWristMenuColumn(mergeRow, deleteRow) - cleanupMenu.visible = false - } - - - private fun cycleLeftMenus() { - leftMenuList.forEach { it.visible = false } - leftMenuIndex = (leftMenuIndex + 1) % leftMenuList.size - logger.debug("Cycling to ${leftMenuList[leftMenuIndex].name}") - leftMenuList[leftMenuIndex].visible = true - } - - - fun addHedgehog() { - logger.info("added hedgehog") - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false - hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) - val hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.visible = false - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) - } - - /** Attach a spherical cursor to the right controller. */ - private fun attachCursorAndTimepointWidget(debug: Boolean = false) { - // Only attach if not already attached - if (sciview.findNodes { it.name == "VR Cursor" }.isNotEmpty()) { - return - } - - volumeTimepointWidget.text = volume.currentTimepoint.toString() - volumeTimepointWidget.name = "Volume Timepoint Widget" - volumeTimepointWidget.fontColor = Vector4f(0.4f, 0.45f, 1f, 1f) - volumeTimepointWidget.spatial { - scale = Vector3f(0.07f) - position = Vector3f(-0.05f, -0.05f, 0.12f) - rotation = Quaternionf().rotationXYZ(-1.57f, -1.57f, 0f) - } - - rightVRController?.model?.let { - cursor.attachCursor(sciview, it) - sciview.addNode(volumeTimepointWidget, activePublish = false, parent = it) - } - } - - /** Object that represents the 3D cursor in form of a sphere. It needs to be attached to a VR controller via [attachCursor]. - * The current cursor position can be obtained with [getPosition]. The current radius is stored in [radius]. - * The tool can be scaled up and down with [scaleByFactor]. - * [resetColor], [activateSelectColor] and [activateTrackingColor] allow changing the cursor's color to reflect the currently active operation. */ - object CursorTool { - private val logger by lazyLogger() - var radius: Float = 0.007f - private set - val cursor = Sphere(radius) - private val initPos = Vector3f(-0.01f, -0.05f, -0.03f) - - fun getPosition() = cursor.spatial().worldPosition() - - fun attachCursor(sciview: SciView, parent: Node, debug: Boolean = false) { - cursor.name = "VR Cursor" - cursor.material { - diffuse = Vector3f(0.15f, 0.2f, 1f) - } - cursor.spatial().position = initPos - sciview.addNode(cursor, parent = parent) - - if (debug) { - val bb = BoundingGrid() - bb.node = cursor - bb.name = "Cursor BB" - bb.lineWidth = 2f - bb.gridColor = Vector3f(1f, 0.3f, 0.25f) - sciview.addNode(bb, parent = parent) - } - logger.info("Attached cursor to controller.") - } - - fun scaleByFactor(factor: Float) { - var clampedFac = 1f - // Only apply the factor if we are in the radius range 0.001f - 0.1f - if ((factor < 1f && radius > 0.001f) || (factor > 1f && radius < 0.15f)) { - clampedFac = factor - } - radius *= clampedFac - cursor.spatial().scale = Vector3f(radius/0.007f) - cursor.spatial().position = Vector3f(initPos) + Vector3f(initPos).normalize().times(radius - 0.007f) - } - - fun resetColor() { - cursor.material().diffuse = Vector3f(0.15f, 0.2f, 1f) - } - - // TODO rename to activate - fun activateSelectColor() { - cursor.material().diffuse = Vector3f(1f, 0.25f, 0.25f) - } - - fun activateTrackingColor() { - cursor.material().diffuse = Vector3f(0.65f, 1f, 0.22f) - } - - } - - open fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - listOf( - "move_forward_fast", - "move_back_fast", - "move_left_fast", - "move_right_fast").forEach { name -> - handler.getBehaviour(name)?.let { behaviour -> - mapper.setKeyBindAndBehavior(hmd, name, behaviour) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.entries.indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.entries.get((current + 1) % 3) - - when (hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogs.visible = false - hedgehogs.runRecursive { it.visible = false } - cam.showMessage("Hedgehogs hidden", distance = 2f, size = 0.2f, centered = true) - } - - HedgehogVisibility.PerTimePoint -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs shown per timepoint", distance = 2f, size = 0.2f, centered = true) - } - - HedgehogVisibility.Visible -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs visible", distance = 2f, size = 0.2f, centered = true) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - class ScaleCursorOrSpotsBehavior(val factor: Float): DragBehaviour { - var isSelected = false - override fun init(p0: Int, p1: Int) { - // determine whether we selected spots or not - isSelected = getSelectionCallback?.invoke()?.isNotEmpty() ?: false - } - - override fun drag(p0: Int, p1: Int) { - if (isSelected) { - scaleSpotsCallback?.invoke(factor, false) - } else { - // Make cursor movement a little stronger than - cursor.scaleByFactor(factor * factor) - } - } - - override fun end(p0: Int, p1: Int) { - scaleSpotsCallback?.invoke(factor, true) - } - } - - val scaleCursorOrSpotsUp = AnalogInputWrapper(ScaleCursorOrSpotsBehavior(1.02f), sciview.currentScene) - - val scaleCursorOrSpotsDown = AnalogInputWrapper(ScaleCursorOrSpotsBehavior(0.98f), sciview.currentScene) - - val faster = ClickBehaviour { _, _ -> - volumesPerSecond = maxOf(minOf(volumesPerSecond+0.2f, 20f), 1f) - cam.showMessage("Speed: ${"%.1f".format(volumesPerSecond)} vol/s",distance = 1.2f, size = 0.2f, centered = true) - } - - val slower = ClickBehaviour { _, _ -> - volumesPerSecond = maxOf(minOf(volumesPerSecond-0.2f, 20f), 1f) - cam.showMessage("Speed: ${"%.1f".format(volumesPerSecond)} vol/s",distance = 2f, size = 0.2f, centered = true) - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if (playing) { - cam.showMessage("Playing", distance = 2f, size = 0.2f, centered = true) - } else { - cam.showMessage("Paused", distance = 2f, size = 0.2f, centered = true) - } - } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt(), - centered = true) - - }, - confirmAction = { - hedgehogs.children.removeLast() - volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> - volume.removeChild(lastTrack) - } - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000, - centered = true - ) - }) - - mapper.setKeyBindAndBehavior(hmd, "stepFwd", nextTimepoint) - mapper.setKeyBindAndBehavior(hmd, "stepBwd", prevTimepoint) - - mapper.setKeyBindAndBehavior(hmd, "playback", playPause) - mapper.setKeyBindAndBehavior(hmd, "radiusIncrease", scaleCursorOrSpotsUp) - mapper.setKeyBindAndBehavior(hmd, "radiusDecrease", scaleCursorOrSpotsDown) - - /** Local class that handles double assignment of the left A key which is used to cycle menus as well as - * reset the rotation when pressed while the [VR2HandNodeTransform] is active. */ - class CycleMenuAndLockAxisBehavior(val button: OpenVRHMD.OpenVRButton, val role: TrackerRole) - : DragBehaviour { - fun registerConfig() { - logger.debug("Setting up keybinds for CycleMenuAndLockAxisBehavior") - resetRotationBtnManager.registerButtonConfig(button, role) - } - override fun init(x: Int, y: Int) { - resetRotationBtnManager.pressButton(button, role) - if (!resetRotationBtnManager.isTwoHandedActive()) { - cycleLeftMenus() - } - } - override fun drag(x: Int, y: Int) {} - override fun end(x: Int, y: Int) { - resetRotationBtnManager.releaseButton(button, role) - } - } - - val leftAButtonBehavior = CycleMenuAndLockAxisBehavior(OpenVRHMD.OpenVRButton.A, TrackerRole.LeftHand) - leftAButtonBehavior.let { - it.registerConfig() - mapper.setKeyBindAndBehavior(hmd, "cycleMenu", it) - } - - mapper.setKeyBindAndBehavior(hmd, "controllerTracking", trackCellsWithController) - - /** Several behaviors mapped per default to the right menu button. If controller tracking is active, - * end the tracking. If not, clicking will either create or delete a spot, depending on whether the user - * previously selected a spot. Holding the button for more than 0.5s deletes the whole connected branch. */ - class AddDeleteResetBehavior : DragBehaviour { - var start = System.currentTimeMillis() - var wasExecuted = false - override fun init(x: Int, y: Int) { - start = System.currentTimeMillis() - wasExecuted = false - } - override fun drag(x: Int, y: Int) { - if (System.currentTimeMillis() - start > 500 && !wasExecuted) { - val p = cursor.getPosition() - spotCreateDeleteCallback?.invoke(volume.currentTimepoint, p, cursor.radius, true, true) - wasExecuted = true - } - } - override fun end(x: Int, y: Int) { - if (controllerTrackingActive) { - endControllerTracking() - } else { - val p = cursor.getPosition() - logger.debug("Got cursor position: $p") - if (!wasExecuted) { - spotCreateDeleteCallback?.invoke(volume.currentTimepoint, p, cursor.radius, false, true) - } - } - } - } - - mapper.setKeyBindAndBehavior(hmd, "addDeleteReset", AddDeleteResetBehavior()) - - class DragSelectBehavior: DragBehaviour { - var time = System.currentTimeMillis() - override fun init(x: Int, y: Int) { - time = System.currentTimeMillis() - val p = cursor.getPosition() - cursor.activateSelectColor() - spotSelectCallback?.invoke(p, volume.currentTimepoint, cursor.radius, false) - } - override fun drag(x: Int, y: Int) { - // Only perform the selection method ten times a second - if (System.currentTimeMillis() - time > 100) { - val p = cursor.getPosition() - spotSelectCallback?.invoke(p, volume.currentTimepoint, cursor.radius, true) - time = System.currentTimeMillis() - } - } - override fun end(x: Int, y: Int) { - cursor.resetColor() - } - } - - mapper.setKeyBindAndBehavior(hmd, "select", DragSelectBehavior()) - - // this behavior is needed for touching the menu buttons - VRTouch.createAndSet(sciview.currentScene, hmd, listOf(TrackerRole.RightHand), false, customTip = cursor.cursor) - - VRGrabTheWorld.createAndSet( - sciview.currentScene, - hmd, - listOf(OpenVRHMD.OpenVRButton.Side), - listOf(TrackerRole.LeftHand), - grabButtonManager, - 1.5f - ) - - VR2HandNodeTransform.createAndSet( - hmd, - OpenVRHMD.OpenVRButton.Side, - sciview.currentScene, - lockYaxis = false, - target = volume, - onStartCallback = { - setSpotVisCallback?.invoke(false) - setTrackVisCallback?.invoke(false) - }, - onEndCallback = { - rebuildGeometryCallback?.invoke() - // Only re-enable the spots or tracks if they were enabled in the first place - setSpotVisCallback?.invoke(spotVisibility) - setTrackVisCallback?.invoke(trackVisibility) - }, - resetRotationBtnManager = resetRotationBtnManager, - resetRotationButton = MultiButtonManager.ButtonConfig(leftAButtonBehavior.button, leftAButtonBehavior.role) - ) - - // drag behavior can stay enabled regardless of current tool mode - MoveInstanceVR.createAndSet( - sciview.currentScene, hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand), - grabButtonManager, - { cursor.getPosition() }, - spotMoveInitCallback, - spotMoveDragCallback, - spotMoveEndCallback, - ) - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - logger.info("Registered VR controller bindings.") - - } - - /** - * Launches a thread that updates the volume time points, the hedgehog visibility and reference target color. - */ - fun launchUpdaterThread() { - thread { - while (!sciview.isInitialized) { - Thread.sleep(200) - } - - while (sciview.running && cellTrackingActive) { - if (playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - if (skipToNext || playing) { - skipToNext = false - if (direction == PlaybackDirection.Forward) { - notifyObservers(oldTimepoint + 1) - } else { - notifyObservers(oldTimepoint - 1) - } - } else { - skipToPrevious = false - if (direction == PlaybackDirection.Forward) { - notifyObservers(oldTimepoint - 1) - } else { - notifyObservers(oldTimepoint + 1) - } - } - - if (hedgehogs.visible) { - if (hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hh -> - val hedgehog = hh as InstancedNode - hedgehog.instances.forEach { - if (it.metadata.isNotEmpty()) { - it.visible = - (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } else { - hedgehogs.children.forEach { hh -> - val hedgehog = hh as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } - } - } - - updateLoopActions.forEach { it.invoke() } - } - - Thread.sleep((1000.0f / volumesPerSecond).toLong()) - } - logger.info("CellTracking updater thread has stopped.") - } - } - - private val updateLoopActions: ArrayList<() -> Unit> = ArrayList() - - /** Allows hooking lambdas into the main update loop. This is needed for eye tracking related actions. */ - protected fun attachToLoop(action: () -> Unit) { - updateLoopActions.add(action) - } - - /** Samples a given [volume] from an [origin] point along a [direction]. - * @return a pair of lists, containing the samples and sample positions, respectively. */ - protected fun sampleRayThroughVolume(origin: Vector3f, direction: Vector3f, volume: Volume): Pair?, List?> { - val intersection = volume.spatial().intersectAABB(origin, direction.normalize(), ignoreChildren = true) - - if (intersection is MaybeIntersects.Intersection) { - val localEntry = (intersection.relativeEntry) - val localExit = (intersection.relativeExit) - val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) - val volumeScale = (volume as RAIVolume).getVoxelScale() - return (samples?.map { it ?: 0.0f } to samplePos?.map { it?.mul(volumeScale) ?: Vector3f(0f) }) - } else { - logger.warn("Ray didn't intersect volume! Origin was $origin, direction was $direction.") - } - return (null to null) - } - - open fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = sphere.origin.minus(center) - val sphereDist = - Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = center - val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - val spine = (hedgehogs.children.last() as InstancedNode).addInstance() - spine.spatial().orientBetweenPoints(p1, p2, true, true) - spine.visible = false - - val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize(), true) - - if (volume.boundingBox?.isInside(cam.spatial().position)!!) { - logger.info("Can't track inside the volume! Please move out of the volume and try again") - return - } - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) - val localExit = (intersection.relativeExit) - // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now - val localDirection = Vector3f(0f) - val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) - val volumeScale = (volume as RAIVolume).getVoxelScale() - - if (samples != null && samplePos != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.spatial().position, - confidence, - samples.map { it ?: 0.0f }, - samplePos.map { it?.mul(volumeScale) ?: Vector3f(0f) } - ) - val count = samples.filterNotNull().count { it > 0.2f } - - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } - // TODO: Show confidence as color for the spine - spine.instancedProperties["Metadata"] = - { Vector4f(confidence, timepoint.toFloat() / volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - - protected fun writeHedgehogToFile(hedgehog: InstancedNode, hedgehogId: Int) { - val hedgehogFile = - sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint;Origin;Direction;LocalEntry;LocalExit;LocalDirection;HeadPosition;HeadOrientation;Position;Confidence;Samples\n") - - val spines = hedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write( - "${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};" + - "${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};" + - "${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";") - }\n" - ) - } - hedgehogFileWriter.close() - logger.info("Wrote hedgehog to file ${hedgehogFile.name}") - } - - protected fun writeTrackToFile( - points: List>, - hedgehogId: Int - ) { - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - trackFileWriter.newLine() - trackFileWriter.newLine() - val parentId = 0 - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - val volumeDimensions = volume.getDimensions() - points.windowed(2, 1).forEach { pair -> - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - - trackFileWriter.close() - } - - /** - * Stops the current tracking environment and restore the original state. - * This method should be overridden if functionality is extended, to make sure any extra objects are also deleted. - */ - open fun stop() { - logger.info("Objects in the scene: ${sciview.allSceneNodes.map { it.name }}") - cellTrackingActive = false - if (::lightTetrahedron.isInitialized) { - lightTetrahedron.forEach { sciview.deleteNode(it) } - } - // Try to find and delete possibly existing VR objects - listOf("Shell", "leftHand", "rightHand").forEach { - val n = sciview.find(it) - n?.let { sciview.deleteNode(n) } - } - rightVRController?.model?.let { - sciview.deleteNode(it) - } - leftVRController?.model?.let { - sciview.deleteNode(it) - } - - logger.info("Cleaned up basic VR objects. Objects left: ${sciview.allSceneNodes.map { it.name }}") - - sciview.toggleVRRendering() - logger.info("Shut down and disabled VR environment.") - rebuildGeometryCallback?.invoke() - } - -} diff --git a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingButtonMapper.kt b/src/main/kotlin/sc/iview/commands/analysis/CellTrackingButtonMapper.kt deleted file mode 100644 index 1ade6cfb..00000000 --- a/src/main/kotlin/sc/iview/commands/analysis/CellTrackingButtonMapper.kt +++ /dev/null @@ -1,164 +0,0 @@ -package sc.iview.commands.analysis - -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.OpenVRHMD.Manufacturer -import graphics.scenery.controls.OpenVRHMD.OpenVRButton -import graphics.scenery.utils.lazyLogger -import org.scijava.ui.behaviour.Behaviour -import kotlin.to - - -/** This input mapping manager provides several preconfigured profiles for different VR controller layouts. - * The active profile is stored in [currentProfile]. - * To change profile, call [loadProfile] with the new [Manufacturer] type. - * Note that for Quest-like layouts, the lower button always equals [OpenVRButton.A] - * and the upper button is always [OpenVRButton.Menu]. */ -object CellTrackingButtonMapper { - - var eyeTracking: ButtonConfig? = null - var controllerTracking: ButtonConfig? = null - var grabObserver: ButtonConfig? = null - var grabSpot: ButtonConfig? = null - var playback: ButtonConfig? = null - var cycleMenu: ButtonConfig? = null - var faster: ButtonConfig? = null - var slower: ButtonConfig? = null - var stepFwd: ButtonConfig? = null - var stepBwd: ButtonConfig? = null - var addDeleteReset: ButtonConfig? = null - var select: ButtonConfig? = null - var move_forward_fast: ButtonConfig? = null - var move_back_fast: ButtonConfig? = null - var move_left_fast: ButtonConfig? = null - var move_right_fast: ButtonConfig? = null - var radiusIncrease: ButtonConfig? = null - var radiusDecrease: ButtonConfig? = null - - private var currentProfile: Manufacturer = Manufacturer.Oculus - - val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) - - private val profiles = mapOf( - Manufacturer.HTC to mapOf( - "eyeTracking" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Trigger), - "controllerTracking" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Trigger), - "grabObserver" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Side), - "grabSpot" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Side), - "playback" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Menu), - "cycleMenu" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Menu), - "faster" to null, - "slower" to null, - "radiusIncrease" to null, - "radiusDecrease" to null, - "stepFwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Left), - "stepBwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Right), - "addDeleteReset" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), - "select" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), - "move_forward_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Up), - "move_back_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), - "move_left_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Left), - "move_right_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Right), - ), - - Manufacturer.Oculus to mapOf( - "eyeTracking" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Trigger), - "controllerTracking" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Trigger), - "grabObserver" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Side), - "grabSpot" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Side), - "playback" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.A), - "cycleMenu" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Menu), -// "faster" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), -// "slower" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Down), - "stepFwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Left), - "stepBwd" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Right), - "addDeleteReset" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Menu), - "select" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.A), - "move_forward_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Up), - "move_back_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Down), - "move_left_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Left), - "move_right_fast" to ButtonConfig(TrackerRole.LeftHand, OpenVRButton.Right), - "radiusIncrease" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Up), - "radiusDecrease" to ButtonConfig(TrackerRole.RightHand, OpenVRButton.Down), - ) - ) - - init { - loadProfile(Manufacturer.Oculus) - } - - /** Load the current profile's button mapping */ - fun loadProfile(p: Manufacturer): Boolean { - currentProfile = p - val profile = profiles[currentProfile] ?: return false - eyeTracking = profile["eyeTracking"] - controllerTracking = profile["controllerTracking"] - grabObserver = profile["grabObserver"] - grabSpot = profile["grabSpot"] - playback = profile["playback"] - cycleMenu = profile["cycleMenu"] - faster = profile["faster"] - slower = profile["slower"] - stepFwd = profile["stepFwd"] - stepBwd = profile["stepBwd"] - addDeleteReset = profile["addDeleteReset"] - select = profile["select"] - move_forward_fast = profile["move_forward_fast"] - move_back_fast = profile["move_back_fast"] - move_left_fast = profile["move_left_fast"] - move_right_fast = profile["move_right_fast"] - radiusIncrease = profile["radiusIncrease"] - radiusDecrease = profile["radiusDecrease"] - return true - } - - fun getCurrentMapping(): Map?{ - return profiles[currentProfile] - } - - fun getMapFromName(name: String): ButtonConfig? { - return when (name) { - "eyeTracking" -> eyeTracking - "controllerTracking" -> controllerTracking - "grabObserver" -> grabObserver - "grabSpot" -> grabSpot - "playback" -> playback - "cycleMenu" -> cycleMenu - "faster" -> faster - "slower" -> slower - "stepFwd" -> stepFwd - "stepBwd" -> stepBwd - "addDeleteReset" -> addDeleteReset - "select" -> select - "move_forward_fast" -> move_forward_fast - "move_back_fast" -> move_back_fast - "move_left_fast" -> move_left_fast - "move_right_fast" -> move_right_fast - "radiusIncrease" -> radiusIncrease - "radiusDecrease" -> radiusDecrease - else -> null - } - } - - /** Sets a keybinding and behavior for an [hmd], using the [name] string, a [behavior] - * and the keybinding if found in the current profile. */ - fun setKeyBindAndBehavior(hmd: OpenVRHMD, name: String, behavior: Behaviour) { - val config = getMapFromName(name) - if (config != null) { - hmd.addKeyBinding(name, config.r, config.b) - hmd.addBehaviour(name, behavior) - logger.debug("Added behavior $behavior to ${config.r}, ${config.b}.") - } else { - logger.warn("No valid button mapping found for key '$name' in current profile!") - } - } -} - - -/** Combines the [TrackerRole] ([r]) and the [OpenVRHMD.OpenVRButton] ([b]) into a single configuration. */ -data class ButtonConfig ( - /** The [TrackerRole] of this button configuration. */ - var r: TrackerRole, - /** The [OpenVRButton] of this button configuration. */ - var b: OpenVRButton -) \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt deleted file mode 100644 index d39edbaa..00000000 --- a/src/main/kotlin/sc/iview/commands/analysis/EyeTracking.kt +++ /dev/null @@ -1,509 +0,0 @@ -package sc.iview.commands.analysis - -import graphics.scenery.* -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.eyetracking.PupilEyeTracker -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import graphics.scenery.textures.Texture -import graphics.scenery.ui.Button -import graphics.scenery.ui.Column -import graphics.scenery.ui.ToggleButton -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.* -import graphics.scenery.utils.gaussSmoothing -import graphics.scenery.utils.localMaxima -import graphics.scenery.utils.toVector3f -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeoutOrNull -import net.imglib2.type.numeric.integer.UnsignedByteType -import org.apache.commons.math3.ml.clustering.Clusterable -import org.apache.commons.math3.ml.clustering.DBSCANClusterer -import org.joml.* -import org.scijava.ui.behaviour.ClickBehaviour -import sc.iview.SciView -import java.awt.image.DataBufferByte -import java.io.ByteArrayInputStream -import java.nio.file.Files -import java.nio.file.Paths -import java.util.concurrent.CompletableFuture -import java.util.concurrent.TimeUnit -import java.util.concurrent.TimeoutException -import javax.imageio.ImageIO -import kotlin.concurrent.thread -import kotlin.math.PI -import kotlin.time.TimeSource - -/** - * Tracking class used for communicating with eye trackers, tracking cells with them in a sciview VR environment. - * It calls the Hedgehog analysis on the eye tracking results and communicates the results to Mastodon via - * [trackCreationCallback], which is called on every spine graph vertex that is extracted - */ -class EyeTracking( - sciview: SciView, - resolutionScale: Float = 1f, -): CellTrackingBase(sciview, resolutionScale) { - - lateinit var pupilTracker: PupilEyeTracker - val calibrationTarget = Icosphere(0.02f, 2) - val laser = Cylinder(0.005f, 0.2f, 10) - - val confidenceThreshold = 0.60f - - private lateinit var debugBoard: TextBoard - - var leftEyeTrackColumn: Column? = null - - enum class TrackingType { Follow, Pick } - - private var currentTrackingType = TrackingType.Follow - - fun establishEyeTrackerConnection(): Boolean { - return try { - val future = CompletableFuture.supplyAsync { - PupilEyeTracker( - calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, - port = System.getProperty("PupilPort", "50020").toInt() - ) - } - pupilTracker = future.get(4, TimeUnit.SECONDS) - true - } catch (e: TimeoutException) { - logger.warn("Eye tracker initialization timed out after 2 seconds. Resuming with default VR.") - false - } catch (e: Exception) { - logger.error("Eye tracker initialization failed with ${e.message}") - false - } - } - - override fun run() { - // Check whether we already initialized the pupilTracker or not - if (!::pupilTracker.isInitialized) { - if (!establishEyeTrackerConnection()) { - logger.error("Failed to initialize eye tracker") - // Handle the failure - maybe throw an exception or set a flag - return - } - } - - // Do all the things for general VR startup before setting up the eye tracking environment - super.run() - - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - referenceTarget.name = "Reference Target" - sciview.camera?.addChild(referenceTarget) - - calibrationTarget.visible = false - calibrationTarget.material { - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(1.0f, 1.0f, 1.0f) - } - calibrationTarget.name = "Calibration Target" - sciview.camera?.addChild(calibrationTarget) - - laser.visible = false - laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f) } - laser.name = "Laser" - sciview.addNode(laser) - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - sciview.addNode(hedgehogs) - - val eyeFrames = Mesh("eyeFrames") - val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) - val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) - left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) - left.spatial().rotation = left.spatial().rotation.rotationZ(PI.toFloat()) - right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) - eyeFrames.addChild(left) - eyeFrames.addChild(right) - - sciview.addNode(eyeFrames) - - val pupilFrameLimit = 20 - var lastFrame = System.nanoTime() - - pupilTracker.subscribeFrames { eye, texture -> - if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { - return@subscribeFrames - } - - val node = if(eye == 1) { - left - } else { - right - } - - val stream = ByteArrayInputStream(texture) - val image = ImageIO.read(stream) - val data = (image.raster.dataBuffer as DataBufferByte).data - - node.ifMaterial { - textures["diffuse"] = Texture( - Vector3i(image.width, image.height, 1), - 3, - UnsignedByteType(), - BufferUtils.allocateByteAndPut(data) - ) } - - lastFrame = System.nanoTime() - } - - // TODO: Replace with cam.showMessage() - debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - if (device.type == TrackedDeviceType.Controller) { - setupEyeTracking() - setupEyeTrackingMenu() - } - } - - // Attach a behavior to the main loop that stops the eye tracking once we reached the first time point - // and analyzes the created track. - attachToLoop { - val newTimepoint = volume.viewerState.currentTimepoint - if (eyeTrackingActive && newTimepoint == 0) { - eyeTrackingActive = false - playing = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - logger.info("Deactivated eye tracking by reaching timepoint 0.") - sciview.camera!!.showMessage("Tracking deactivated.", distance = 2f, size = 0.2f, centered = true) - analyzeEyeTrack() - } - } - - } - - - private fun setupEyeTracking() { - val cam = sciview.camera as? DetachedHeadCamera ?: return - - val toggleTracking = ClickBehaviour { _, _ -> - if (!pupilTracker.isCalibrated) { - logger.warn("Can't do eye tracking because eye trackers are not calibrated yet.") - return@ClickBehaviour - } - if (eyeTrackingActive) { - logger.info("deactivated tracking through user input.") - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 2f, size = 0.2f, centered = true) - if (currentTrackingType == TrackingType.Follow) { - analyzeEyeTrack() - } else { - analyzeGazeClusters() - } - playing = false - } else { - logger.info("activating tracking...") - playing = if (currentTrackingType == TrackingType.Follow) { - true - } else { - false - } - addHedgehog() - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 2f, size = 0.2f, centered = true) - } - eyeTrackingActive = !eyeTrackingActive - } - - mapper.setKeyBindAndBehavior(hmd, "eyeTracking", toggleTracking) - } - - private fun calibrateEyeTrackers(force: Boolean = false) { - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - pupilTracker.gazeConfidenceThreshold = confidenceThreshold - if (!pupilTracker.isCalibrated || force) { - logger.info("Calibrating pupil trackers...") - - volume.visible = false - - pupilTracker.onCalibrationInProgress = { - cam.showMessage( - "Crunching equations ...", - distance = 2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), - duration = 15000, centered = true - ) - } - - pupilTracker.onCalibrationFailed = { - cam.showMessage( - "Calibration failed.", - distance = 2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), - centered = true - ) - } - - pupilTracker.onCalibrationSuccess = { - cam.showMessage( - "Calibration succeeded!", - distance = 2f, size = 0.2f, - messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), - centered = true - ) - - for (i in 0 until 20) { - referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f) } - Thread.sleep(100) - referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f) } - Thread.sleep(30) - } - - if (!pupilTracker.isCalibrated) { - hmd.removeBehaviour("start_calibration") - hmd.removeKeyBinding("start_calibration") - } - - volume.visible = true - playing = false - } - - pupilTracker.unsubscribeFrames() - sciview.deleteNode(sciview.find("eyeFrames")) - - logger.info("Starting eye tracker calibration") - cam.showMessage("Follow the white rabbit.", distance = 2f, size = 0.2f,duration = 1500, centered = true) - - pupilTracker.calibrate(cam, hmd, - generateReferenceData = true, - calibrationTarget = calibrationTarget) - - pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { - - PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> - if (gaze.confidence > confidenceThreshold) { - val p = gaze.gazePoint() - referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units - referenceTarget.spatial().position = p - (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" - - val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() - val direction = (pointWorld - headCenter).normalize() - - if (eyeTrackingActive) { - addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) - } - } - } - } - logger.info("Calibration routine done.") - } - } - } - - private fun setupEyeTrackingMenu() { - - val calibrateButton = Button("Calibrate", - command = { calibrateEyeTrackers() }, - byTouch = true, depressDelay = 500) - - val toggleHedgehogsBtn = ToggleButton( - "Hedgehogs Off", - "Hedgehogs On", - command = { - hedgehogVisibility = if (hedgehogVisibility == HedgehogVisibility.Hidden) { - HedgehogVisibility.PerTimePoint - } else { - HedgehogVisibility.Hidden - } - }, - byTouch = true - ) - - val toggleTrackTypeBtn = ToggleButton( - "Follow Cell", - "Count Cells", - command = { - currentTrackingType = if (currentTrackingType == TrackingType.Follow) { - TrackingType.Pick - } else { - TrackingType.Follow - } - }, - byTouch = true, - color = Vector3f(0.65f, 1f, 0.22f), - pressedColor = Vector3f(0.15f, 0.2f, 1f) - ) - - leftEyeTrackColumn = - createWristMenuColumn(toggleTrackTypeBtn, toggleHedgehogsBtn, calibrateButton, name = "Eye Tracking Menu") - leftEyeTrackColumn?.visible = false - } - - /** Writes the accumulated gazes (hedgehog) to a file, analyzes it, - * sends the track to Mastodon and writes the track to a file. */ - private fun analyzeEyeTrack() { - val lastHedgehog = hedgehogs.children.last() as InstancedNode - val hedgehogId = hedgehogIds.incrementAndGet() - - writeHedgehogToFile(lastHedgehog, hedgehogId) - - val spines = getSpinesFromHedgehog(lastHedgehog) - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) - h.run() - } - - if(track == null) { - logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - if (trackCreationCallback != null && rebuildGeometryCallback != null) { - trackCreationCallback?.invoke(track.points, cursor.radius,false, null, null) - rebuildGeometryCallback?.invoke() - } else { - logger.warn("Tried to send track data to Mastodon but couldn't find the callbacks!") - } - - writeTrackToFile(track.points, hedgehogId) - - } - - private fun getSpinesFromHedgehog(hedgehog: InstancedNode): List { - return hedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - } - - /** Performs an analysis of collected gazes (hedgehogs) by first calculating the rotational distance between - * subsequent gazes, then discards all gazes larger than 0.3x median distance, clusters the remaining directions - * and samples the volume using the cluster centers as directions. It then extracts the first local minima and sends - * them as spots to Mastodon. */ - private fun analyzeGazeClusters() { - logger.info("Starting analysis of gaze clusters...") - val lastHedgehog = hedgehogs.children.last() as InstancedNode - val hedgehogId = hedgehogIds.incrementAndGet() - - writeHedgehogToFile(lastHedgehog, hedgehogId) - // Get spines from the most recent hedgehog - val spines = getSpinesFromHedgehog(lastHedgehog) - logger.info("Starting with ${spines.size} spines") - - // Calculate the distance from each direction to its neighbor - val speeds = spines.zipWithNext { a, b -> a.direction.distance(b.direction) } - logger.info("Min speed: ${speeds.min()}, max speed: ${speeds.max()}") - val medianSpeed = speeds.sorted()[speeds.size/2] - logger.info("Median speed: $medianSpeed") - - // Clean the list of spines by removing the ones that are too far from their neighbors - val cleanedSpines = spines.filterIndexed { index, _ -> speeds[index] < 0.3 * medianSpeed } - logger.info("After cleaning: ${cleanedSpines.size} spines remain") - - var start = TimeSource.Monotonic.markNow() - // Assuming ten times the median distance is a good clustering value... - val clustering = DBSCANClusterer((10 * medianSpeed).toDouble(), 3) - - // Create a map to efficiently find spine metadata by direction - val spineByDirection = cleanedSpines.associateBy { it.direction.toDoubleArray().contentHashCode() } - - val clusters = clustering.cluster(cleanedSpines.map { - Clusterable { - // On the fly conversion of a Vector3f to a double array - it.direction.toDoubleArray() - } - }) - logger.info("Clustering took ${TimeSource.Monotonic.markNow() - start}") - logger.info("We got ${clusters.size} clusters") - - // Extract the mean direction for each cluster, - // and find the corresponding start positions and average them too - val clusterCenters = clusters.map { cluster -> - var meanDir = Vector3f() - var meanPos = Vector3f() - - // Each "point" in the cluster is actually the ray direction - cluster.points.forEach { point -> - // Accumulate the directions - meanDir += point.point.toVector3f() - // Now grab the spine itself so we can also access its origin - val spine = spineByDirection[point.point.contentHashCode()] - if (spine != null) { - meanPos += spine.origin - } else { - logger.warn("Could not find spine for direction: ${point.point.contentToString()}") - } - } - // Calculate means by dividing by cluster size - meanDir /= cluster.points.size.toFloat() - meanPos /= cluster.points.size.toFloat() - - logger.debug("MeanDir for cluster is $meanDir") - logger.debug("MeanPos for cluster is $meanPos") - - (meanPos to meanDir) - } - - // We only need the analyzer to access the smoothing and maxima search functions - val analyzer = HedgehogAnalysis(cleanedSpines, Matrix4f(volume.spatial().world)) - - start = TimeSource.Monotonic.markNow() - val spots = clusterCenters.map { (origin, direction) -> - val (samples, samplePos) = sampleRayThroughVolume(origin, direction, volume) - var spotPos: Vector3f? = null - if (samples != null && samplePos != null) { - val smoothed = gaussSmoothing(samples, 4) - val rayMax = smoothed.max() - // take the first local maximum that is at least 20% of the global maximum to prevent spot creation in noisy areas - localMaxima(smoothed).firstOrNull {it.second > 0.2 * rayMax}?.let { (index, sample) -> - spotPos = samplePos[index] - } - } - spotPos - } - logger.info("Sampling volume and spot extraction took ${TimeSource.Monotonic.markNow() - start}") - spots.filterNotNull().forEach { spot -> - spotCreateDeleteCallback?.invoke(volume.currentTimepoint, spot, cursor.radius, false, false) - } - } - - /** Toggles the VR rendering off, cleans up eyetracking-related scene objects and removes the light tetrahedron - * that was created for the calibration routine. */ - override fun stop() { - pupilTracker.unsubscribeFrames() - logger.info("Stopped volume and hedgehog updater thread.") - val n = sciview.find("eyeFrames") - n?.let { sciview.deleteNode(it) } - // Delete definitely existing objects - listOf(referenceTarget, calibrationTarget, laser, debugBoard, hedgehogs).forEach { - try { - sciview.deleteNode(it) - } catch (e: Exception) { - logger.warn("Failed to delete $it") - } - } - logger.info("Successfully cleaned up eye tracking environment.") - super.stop() - } - -} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt deleted file mode 100644 index ee55ce38..00000000 --- a/src/main/kotlin/sc/iview/commands/analysis/HedgehogAnalysis.kt +++ /dev/null @@ -1,410 +0,0 @@ -package sc.iview.commands.analysis - -import org.joml.Vector3f -import org.joml.Matrix4f -import org.joml.Quaternionf -import graphics.scenery.utils.extensions.* -import graphics.scenery.utils.gaussSmoothing -import graphics.scenery.utils.lazyLogger -import graphics.scenery.utils.localMaxima -import graphics.scenery.utils.stdDev -import org.slf4j.LoggerFactory -import java.io.File -import kotlin.collections.iterator -import kotlin.math.sqrt - -/** - * Performs analysis over a collection of eye-tracking spines (aka hedgehog). Extracts a list of local maxima from - * the sampled volume, removes statistical outliers and performs a graph optimization over the remaining maxima to - * extract the likeliest path of the cell. The companion object contains methods to load CSV files. - * @author Ulrik Günther - */ -class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { - - private val logger by lazyLogger() - - val timepoints = LinkedHashMap>() - - var avgConfidence = 0.0f - private set - var totalSampleCount = 0 - private set - - /** Data class for collecting track points, consisting of positions and a [SpineGraphVertex], and the averaged - * confidences of all spines. Returned by [kotlin.run]. */ - data class Track( - val points: List>, - val confidence: Float - ) - - init { - logger.info("Starting analysis with ${spines.size} spines") - - spines.forEach { spine -> - val timepoint = spine.timepoint - val current = timepoints[timepoint] - - if(current == null) { - timepoints[timepoint] = arrayListOf(spine) - } else { - current.add(spine) - } - - avgConfidence += spine.confidence - totalSampleCount++ - } - - avgConfidence /= totalSampleCount - } - - - - /** Cell positions extracted from gaze analysis are collected in this data class together with other information - * such as the volume [value] at this point, and the [previous] and [next] vertices. */ - data class SpineGraphVertex(val timepoint: Int, - val position: Vector3f, - val worldPosition: Vector3f, - val index: Int, - val value: Float, - val metadata : SpineMetadata? = null, - var previous: SpineGraphVertex? = null, - var next: SpineGraphVertex? = null) { - - fun distance(): Float { - val n = next - return if(n != null) { - val t = (n.worldPosition - this.worldPosition) - sqrt(t.x*t.x + t.y*t.y + t.z*t.z) - } else { - 0.0f - } - } - - fun drop() { - previous?.next = next - next?.previous = previous - } - - override fun toString() : String { - return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" - } - } - - data class VertexWithDistance(val vertex: SpineGraphVertex, val distance: Float) - - fun run(): Track? { - - // Adapt thresholds based on data from the first spine - val startingThreshold = timepoints.entries.first().value.first.samples.min() * 2f + 0.002f - val localMaxThreshold = timepoints.entries.first().value.first.samples.max() * 0.2f - val zscoreThreshold = 2.0f - val removeTooFarThreshold = 5.0f - - if(timepoints.isEmpty()) { - return null - } - - - //step1: find the startingPoint by using startingThreshold - val startingPoint = timepoints.entries.firstOrNull { entry -> - entry.value.any { metadata -> metadata.samples.any { it > startingThreshold } } - } ?: return null - - logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold), localMayThreshold=$localMaxThreshold") - - // filter timepoints, remove all before the starting point - timepoints.filter { it.key > startingPoint.key } - .forEach { timepoints.remove(it.key) } - - // Stop timepoints after reaching 0 - val result = mutableMapOf>() - var foundZero = false - - for ((time, value) in timepoints) { - if (foundZero) { - break - } - result[time] = value - if (time == 0) { - foundZero = true - } - } - timepoints.clear() - timepoints.putAll(result) - - logger.info("${timepoints.size} timepoints left") - - // step2: find the maxIndices along the spine - // this will be a list of lists, where each entry in the first-level list - // corresponds to a time point, which then contains a list of vertices within that timepoint. - val candidates: List> = timepoints.map { tp -> - val vs = tp.value.mapIndexedNotNull { i, spine -> - // First apply a subtle smoothing kernel to prevent many close/similar local maxima - val smoothedSamples = gaussSmoothing(spine.samples, 4) - // determine local maxima (and their indices) along the spine, aka, actual things the user might have - // seen when looking into the direction of the spine - val maxIndices = localMaxima(smoothedSamples) - logger.debug("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") - - // if there actually are local maxima, generate a graph vertex for them with all the necessary metadata - if(maxIndices.isNotEmpty()) { - //maxIndices. -// filter the maxIndices which are too far away, which can be removed - //filter { it.first <1200}. - maxIndices.map { index -> - logger.debug("Generating vertex at index $index") - // get the position of the current index along the spine - val position = spine.samplePosList[index.first] - val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() - SpineGraphVertex(tp.key, - position, - worldPosition, - index.first, - index.second, - spine) - } - } else { - null - } - } - vs - }.flatten() - - logger.info("SpineGraphVertices extracted") - - // step3: connect localMaximal points between 2 candidate spines according to the shortest path principle - // get the initial vertex, this one is assumed to always be in front, and have a local maximum - aka, what - // the user looks at first is assumed to be the actual cell they want to track - val initial = candidates.first().first { it.value > startingThreshold } - var current = initial - var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> - // calculate world-space distances between current point, and all candidate - // vertices, sorting them by distance - val vertices = vs - .filter { it.value > localMaxThreshold } - .map { vertex -> - val t = current.worldPosition - vertex.worldPosition - val distance = t.length() - VertexWithDistance(vertex, distance) - } - .sortedBy { it.distance } - - val closest = vertices.firstOrNull() - if(closest != null && closest.distance > 0) { - // create a linked list between current and closest vertices - current.next = closest.vertex - closest.vertex.previous = current - current = closest.vertex - current - } else { - null - } - }.toMutableList() - - // calculate average path lengths over all - val beforeCount = shortestPath.size - var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - var stdDevPathLength = shortestPath.map { it.distance() }.stdDev() - logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - - fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) - - //step4: if some path is longer than multiple average length, it should be removed - // TODO Don't remove vertices along the path, as that doesn't translate well to Mastodon tracks. Find a different way? -// while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { -// shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() -// shortestPath.windowed(3, 1, partialWindows = true).forEach { -// // this reconnects the neighbors after the offending vertex has been removed -// it.getOrNull(0)?.next = it.getOrNull(1) -// it.getOrNull(1)?.previous = it.getOrNull(0) -// it.getOrNull(1)?.next = it.getOrNull(2) -// it.getOrNull(2)?.previous = it.getOrNull(1) -// } -// } - - // recalculate statistics after offending vertex removal - avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - stdDevPathLength = shortestPath.map { it.distance() }.stdDev().toFloat() - - //step5: remove some vertices according to zscoreThreshold -// var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } -// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") -// while(remaining > 0) { -// val outliers = shortestPath -// .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } -// .map { -// val idx = shortestPath.indexOf(it) -// listOf(idx-1,idx,idx+1) -// }.flatten() -// -// shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() -// remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } -// -// shortestPath.windowed(3, 1, partialWindows = true).forEach { -// it.getOrNull(0)?.next = it.getOrNull(1) -// it.getOrNull(1)?.previous = it.getOrNull(0) -// it.getOrNull(1)?.next = it.getOrNull(2) -// it.getOrNull(2)?.previous = it.getOrNull(1) -// } -// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") -// } - -// val afterCount = shortestPath.size -// logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") - val singlePoints = shortestPath - .groupBy { it.timepoint } - .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata?.confidence ?: 0f } } - .filter { - (it.metadata?.direction?.dot(it.previous!!.metadata?.direction) ?: 0f) > 0.5f - } - - - logger.info("Returning ${singlePoints.size} points") - - - return Track(singlePoints.map { it.position to it}, avgConfidence) - } - - companion object { - private val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) - - fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { - logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val confidence = tokens[1].toFloat() - val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - Vector3f(0.0f), - Vector3f(0.0f), - 0.0f, - Vector3f(0.0f), - Vector3f(0.0f), - Vector3f(0.0f), - Vector3f(0.0f), - Quaternionf(), - Vector3f(0.0f), - confidence, - samples - ) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, Matrix4f()) - } - - private fun String.toVector3f(): Vector3f { - val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} - - if (array[0] == "+Inf" || array[0] == "-Inf") - return Vector3f(0.0f,0.0f,0.0f) - - return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) - } - - private fun String.toQuaternionf(): Quaternionf { - val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} - return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) - } - fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f,separator: String = ";"): HedgehogAnalysis { - logger.info("Loading spines from complete CSV with Matrix at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - logger.info("lines number: " + lines.size) - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val origin = tokens[1].toVector3f() - val direction = tokens[2].toVector3f() - val localEntry = tokens[3].toVector3f() - val localExit = tokens[4].toVector3f() - val localDirection = tokens[5].toVector3f() - val headPosition = tokens[6].toVector3f() - val headOrientation = tokens[7].toQuaternionf() - val position = tokens[8].toVector3f() - val confidence = tokens[9].toFloat() - val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - origin, - direction, - 0.0f, - localEntry, - localExit, - localDirection, - headPosition, - headOrientation, - position, - confidence, - samples - ) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, matrix4f) - } - - fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { - logger.info("Loading spines from complete CSV at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val origin = tokens[1].toVector3f() - val direction = tokens[2].toVector3f() - val localEntry = tokens[3].toVector3f() - val localExit = tokens[4].toVector3f() - val localDirection = tokens[5].toVector3f() - val headPosition = tokens[6].toVector3f() - val headOrientation = tokens[7].toQuaternionf() - val position = tokens[8].toVector3f() - val confidence = tokens[9].toFloat() - val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - origin, - direction, - 0.0f, - localEntry, - localExit, - localDirection, - headPosition, - headOrientation, - position, - confidence, - samples - ) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, Matrix4f()) - } - } -} - -fun main(args: Array) { - val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") - // main should only be called for testing purposes - if (args.isNotEmpty()) { - val file = File(args[0]) - val analysis = HedgehogAnalysis.fromCSV(file) - val results = analysis.run() - logger.info("Results: \n$results") - } -} diff --git a/src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt b/src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt deleted file mode 100644 index dc7f1ebf..00000000 --- a/src/main/kotlin/sc/iview/commands/analysis/SpineMetadata.kt +++ /dev/null @@ -1,23 +0,0 @@ -package sc.iview.commands.analysis - -import org.joml.Quaternionf -import org.joml.Vector3f - -/** - * Data class to store metadata for spines of the hedgehog. - */ -data class SpineMetadata( - val timepoint: Int, - val origin: Vector3f, - val direction: Vector3f, - val distance: Float, - val localEntry: Vector3f, - val localExit: Vector3f, - val localDirection: Vector3f, - val headPosition: Vector3f, - val headOrientation: Quaternionf, - val position: Vector3f, - val confidence: Float, - val samples: List, - val samplePosList: List = ArrayList() -) diff --git a/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt b/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt deleted file mode 100644 index 87f68aae..00000000 --- a/src/main/kotlin/sc/iview/controls/behaviours/MoveInstanceVR.kt +++ /dev/null @@ -1,86 +0,0 @@ -package sc.iview.controls.behaviours - -import graphics.scenery.Scene -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.MultiButtonManager -import org.joml.Vector3f -import org.scijava.ui.behaviour.DragBehaviour -import kotlin.collections.forEach -import kotlin.let - -class MoveInstanceVR( - val buttonmanager: MultiButtonManager, - val button: OpenVRHMD.OpenVRButton, - val trackerRole: TrackerRole, - val getTipPosition: () -> Vector3f, - val spotMoveInitCallback: ((Vector3f) -> Unit)? = null, - val spotMoveDragCallback: ((Vector3f) -> Unit)? = null, - val spotMoveEndCallback: ((Vector3f) -> Unit)? = null, -): DragBehaviour { - - override fun init(x: Int, y: Int) { - buttonmanager.pressButton(button, trackerRole) - if (!buttonmanager.isTwoHandedActive()) { - spotMoveInitCallback?.invoke(getTipPosition()) - } - } - - override fun drag(x: Int, y: Int) { - // Only perform the single hand behavior when no other grab button is currently active - // to prevent simultaneous execution of behaviors - if (!buttonmanager.isTwoHandedActive()) { - spotMoveDragCallback?.invoke(getTipPosition()) - } - } - - override fun end(x: Int, y: Int) { - if (!buttonmanager.isTwoHandedActive()) { - spotMoveEndCallback?.invoke(getTipPosition()) - } - buttonmanager.releaseButton(button, trackerRole) - } - - companion object { - - /** - * Convenience method for adding grab behaviour - */ - fun createAndSet( - scene: Scene, - hmd: OpenVRHMD, - buttons: List, - controllerSide: List, - buttonmanager: MultiButtonManager, - getTipPosition: () -> Vector3f, - spotMoveInitCallback: ((Vector3f) -> Unit)? = null, - spotMoveDragCallback: ((Vector3f) -> Unit)? = null, - spotMoveEndCallback: ((Vector3f) -> Unit)? = null, - ) { - hmd.events.onDeviceConnect.add { _, device, _ -> - if (device.type == TrackedDeviceType.Controller) { - device.model?.let { controller -> - if (controllerSide.contains(device.role)) { - buttons.forEach { button -> - val name = "VRDrag:${hmd.trackingSystemName}:${device.role}:$button" - val grabBehaviour = MoveInstanceVR( - buttonmanager, - button, - device.role, - getTipPosition, - spotMoveInitCallback, - spotMoveDragCallback, - spotMoveEndCallback - ) - buttonmanager.registerButtonConfig(button, device.role) - hmd.addBehaviour(name, grabBehaviour) - hmd.addKeyBinding(name, device.role, button) - } - } - } - } - } - } - } -} \ No newline at end of file From b0a28dec86f795216a01326ca405cea171d0817f Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 2 Feb 2026 14:13:15 +0100 Subject: [PATCH 11/11] Move TimepointObserver from sciview to scenery, refactor it to make it generic --- .../sc/iview/commands/analysis/TimepointObserver.kt | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 src/main/kotlin/sc/iview/commands/analysis/TimepointObserver.kt diff --git a/src/main/kotlin/sc/iview/commands/analysis/TimepointObserver.kt b/src/main/kotlin/sc/iview/commands/analysis/TimepointObserver.kt deleted file mode 100644 index 584cb5d8..00000000 --- a/src/main/kotlin/sc/iview/commands/analysis/TimepointObserver.kt +++ /dev/null @@ -1,13 +0,0 @@ -package sc.iview.commands.analysis - -/** - * Interface to allow subscription to timepoint updates, especially for updating sciview contents - * after a user triggered a timepoint change via controller input. - */ -interface TimepointObserver { - - /** - * Called when the timepoint was updated. - */ - fun onTimePointChanged(timepoint: Int) -}