Skip to content

Commit 0cd17da

Browse files
fhilgersfhilgers
authored
Merge pull request #137 from amosproj/feat/96-React_Metadata_Widget_New_Frontend
Feat/96 react metadata widget new frontend Signed-off-by: Felix Hilgers <felix.hilgers@fau.de>
2 parents 3d56f17 + 7577380 commit 0cd17da

8 files changed

Lines changed: 671 additions & 38 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,9 @@ install-frontend:
8686
cd src/frontend && npm install
8787

8888
install-backend:
89-
# Auto-uses .python-version (3.11)
89+
# Auto-uses .python-version (3.11)
9090
cd src/backend && uv python install
91-
# core + dev + inference + onnx-tools + onnx-cpu
91+
# core + dev + inference + onnx-tools + onnx-cpu
9292
cd src/backend && uv sync --extra dev --extra inference --extra onnx-tools --extra onnx-cpu
9393

9494
lint: lint-frontend lint-backend lint-licensing

‎src/frontend/src/App.tsx‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,27 +10,31 @@ import { useAnalyzerWebSocket } from './hooks/useAnalyzerWebSocket';
1010

1111
import Header from './components/Header';
1212
import ConnectionControls from './components/ConnectionControls';
13-
import DetectionInfo from './components/DetectionInfo';
1413
import VideoPlayer, { VideoPlayerHandle } from './components/VideoPlayer';
14+
import MetadataWidget from './components/MetadataWidget';
1515

1616
function App() {
1717
const videoPlayerRef = useRef<VideoPlayerHandle>(null);
18+
const videoContainerRef = useRef<HTMLDivElement>(null);
1819

1920
const [overlayFps, setOverlayFps] = useState<number>(0);
21+
const [isMetadataWidgetOpen, setIsMetadataWidgetOpen] = useState(true);
2022

2123
// WebRTC connection to webcam service (for raw video)
2224
const {
2325
videoRef,
2426
connectionState: videoState,
2527
latencyMs,
2628
isPaused,
29+
stats,
2730
connect: connectVideo,
2831
disconnect: disconnectVideo,
2932
togglePlayPause,
3033
enterFullscreen,
3134
} = useWebRTCPlayer({
3235
signalingEndpoint: 'http://localhost:8000', // Webcam service
3336
autoPlay: true,
37+
containerRef: videoContainerRef,
3438
});
3539

3640
// WebSocket connection to analyzer service (for metadata)
@@ -91,16 +95,22 @@ function App() {
9195
<VideoPlayer
9296
ref={videoPlayerRef}
9397
videoRef={videoRef}
98+
containerRef={videoContainerRef}
9499
videoState={videoState}
95100
isPaused={isPaused}
96101
onTogglePlay={togglePlayPause}
97102
onFullscreen={enterFullscreen}
98103
onOverlayFpsUpdate={setOverlayFps}
104+
metadataWidget={
105+
<MetadataWidget
106+
streamMetadata={stats}
107+
detectionMetadata={latestMetadata}
108+
defaultGrouped={false}
109+
isOpen={isMetadataWidgetOpen}
110+
onToggle={() => setIsMetadataWidgetOpen(!isMetadataWidgetOpen)}
111+
/>
112+
}
99113
/>
100-
101-
{latestMetadata && latestMetadata.detections.length > 0 && (
102-
<DetectionInfo detections={latestMetadata.detections} />
103-
)}
104114
</div>
105115
);
106116
}

‎src/frontend/src/components/DetectionInfo.tsx‎

Lines changed: 152 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
* SPDX-License-Identifier: MIT
55
*/
66

7+
import { useMemo, memo } from 'react';
8+
import { getCocoLabel } from '../constants/cocoLabels';
9+
710
export interface Detection {
811
id: string;
912
label: string;
@@ -20,43 +23,167 @@ export interface Position {
2023

2124
export interface DetectionInfoProps {
2225
detections: Detection[];
26+
showGrouped?: boolean;
27+
}
28+
29+
interface GroupedDetection {
30+
label: string;
31+
count: number;
32+
minConfidence: number;
33+
maxConfidence: number;
34+
minDistance?: number;
35+
maxDistance?: number;
2336
}
2437

25-
export default function DetectionInfo({ detections }: DetectionInfoProps) {
38+
function DetectionInfo({
39+
detections,
40+
showGrouped = false,
41+
}: DetectionInfoProps) {
2642
if (!detections || detections.length === 0) {
2743
return null;
2844
}
2945

46+
// Sort detections by distance (closest first)
47+
const sortedDetections = useMemo(() => {
48+
return [...detections].sort((a, b) => {
49+
// If both have distance, sort by distance
50+
if (a.distance !== undefined && b.distance !== undefined) {
51+
return a.distance - b.distance;
52+
}
53+
// If only one has distance, put it first
54+
if (a.distance !== undefined) return -1;
55+
if (b.distance !== undefined) return 1;
56+
// Otherwise maintain order
57+
return 0;
58+
});
59+
}, [detections]);
60+
61+
// Group detections by label
62+
const groupedDetections = useMemo(() => {
63+
const groups = new Map<string, GroupedDetection>();
64+
65+
sortedDetections.forEach((det) => {
66+
const labelName = getCocoLabel(det.label);
67+
const existing = groups.get(labelName);
68+
69+
if (existing) {
70+
existing.count++;
71+
existing.minConfidence = Math.min(
72+
existing.minConfidence,
73+
det.confidence
74+
);
75+
existing.maxConfidence = Math.max(
76+
existing.maxConfidence,
77+
det.confidence
78+
);
79+
80+
if (det.distance !== undefined) {
81+
existing.minDistance =
82+
existing.minDistance !== undefined
83+
? Math.min(existing.minDistance, det.distance)
84+
: det.distance;
85+
existing.maxDistance =
86+
existing.maxDistance !== undefined
87+
? Math.max(existing.maxDistance, det.distance)
88+
: det.distance;
89+
}
90+
} else {
91+
groups.set(labelName, {
92+
label: labelName,
93+
count: 1,
94+
minConfidence: det.confidence,
95+
maxConfidence: det.confidence,
96+
minDistance: det.distance,
97+
maxDistance: det.distance,
98+
});
99+
}
100+
});
101+
102+
return Array.from(groups.values());
103+
}, [sortedDetections]);
104+
105+
if (showGrouped) {
106+
return (
107+
<div className="bg-[#2a2a2a] border border-[#404040] p-5 rounded-lg shadow-[0_4px_20px_rgba(0,0,0,0.4)]">
108+
<h3 className="my-0 mb-4 text-[#00d4ff] text-xl">
109+
Detections ({detections.length} objects)
110+
</h3>
111+
<div className="max-h-96 overflow-y-auto space-y-2">
112+
{groupedDetections.map((group) => (
113+
<GroupedDetectionCard key={group.label} group={group} />
114+
))}
115+
</div>
116+
</div>
117+
);
118+
}
119+
30120
return (
31121
<div className="bg-[#2a2a2a] border border-[#404040] p-5 rounded-lg shadow-[0_4px_20px_rgba(0,0,0,0.4)]">
32122
<h3 className="my-0 mb-4 text-[#00d4ff] text-xl">
33123
Latest Detections ({detections.length})
34124
</h3>
35-
<div className="flex flex-wrap gap-2.5">
36-
{detections.map((detection) => (
37-
<div
38-
key={detection.id}
39-
className="flex items-center gap-2 px-3 py-2 bg-[#404040] rounded-md border-l-[3px] border-l-[#00d4ff] border border-[#555]"
40-
>
41-
<span className="font-semibold text-[#e0e0e0]">
42-
{detection.label}
43-
</span>
44-
<span className="bg-gradient-to-br from-[#74b9ff] to-[#0984e3] text-white px-2 py-0.5 rounded text-xs font-semibold font-mono shadow-[0_2px_4px_rgba(116,185,255,0.3)]">
45-
{(detection.confidence * 100).toFixed(1)}%
46-
</span>
47-
{detection.distance && (
48-
<span className="bg-gradient-to-br from-[#00d4aa] to-[#00b894] text-white px-2 py-0.5 rounded text-xs font-semibold font-mono shadow-[0_2px_4px_rgba(0,212,170,0.3)]">
49-
{detection.distance.toFixed(2)}m
50-
</span>
51-
)}
52-
<span className="bg-gradient-to-br from-orange-700 to-orange-800 text-white px-2 py-0.5 rounded text-xs font-semibold font-mono shadow-[0_2px_4px_rgba(116,185,255,0.3)]">
53-
x={detection.position.x.toFixed(1)}m,y=
54-
{detection.position.y.toFixed(1)}m,z=
55-
{detection.position.z.toFixed(1)}m
56-
</span>
57-
</div>
58-
))}
125+
<div className="max-h-96 overflow-y-auto">
126+
<div className="flex flex-wrap gap-2.5">
127+
{sortedDetections.map((detection) => (
128+
<DetectionCard key={detection.id} detection={detection} />
129+
))}
130+
</div>
59131
</div>
60132
</div>
61133
);
62134
}
135+
136+
// Memoized detection card component
137+
const DetectionCard = memo(({ detection }: { detection: Detection }) => {
138+
const labelName = getCocoLabel(detection.label);
139+
140+
return (
141+
<div className="flex flex-col items-center gap-2 px-3 py-2 bg-[#404040] w-full rounded-md border-l-[3px] border-l-[#00d4ff] border border-[#555]">
142+
<span className="font-semibold text-[#e0e0e0]">{labelName}</span>
143+
<span className="bg-gradient-to-br from-[#74b9ff] to-[#0984e3] text-white px-2 py-0.5 rounded text-xs font-semibold font-mono shadow-[0_2px_4px_rgba(116,185,255,0.3)]">
144+
{(detection.confidence * 100).toFixed(1)}%
145+
</span>
146+
{detection.distance !== undefined && (
147+
<span className="bg-gradient-to-br from-[#00d4aa] to-[#00b894] text-white px-2 py-0.5 rounded text-xs font-semibold font-mono shadow-[0_2px_4px_rgba(0,212,170,0.3)]">
148+
{detection.distance.toFixed(2)}m
149+
</span>
150+
)}
151+
<span className="bg-gradient-to-br from-orange-700 to-orange-800 text-white px-2 py-0.5 rounded text-xs font-semibold font-mono shadow-[0_2px_4px_rgba(116,185,255,0.3)]">
152+
x={detection.position.x.toFixed(1)}m,y=
153+
{detection.position.y.toFixed(1)}m,z=
154+
{detection.position.z.toFixed(1)}m
155+
</span>
156+
</div>
157+
);
158+
});
159+
160+
DetectionCard.displayName = 'DetectionCard';
161+
162+
// Memoized grouped detection card component
163+
const GroupedDetectionCard = memo(({ group }: { group: GroupedDetection }) => {
164+
return (
165+
<div className="flex items-center justify-between px-4 py-3 bg-[#404040] rounded-md border-l-[3px] border-l-[#00d4ff] border border-[#555]">
166+
<div className="flex items-center gap-3">
167+
<span className="font-semibold text-[#e0e0e0] text-lg">
168+
{group.count}× {group.label}
169+
</span>
170+
<span className="text-[#888] text-xs">
171+
{group.minConfidence === group.maxConfidence
172+
? `${(group.minConfidence * 100).toFixed(1)}%`
173+
: `${(group.minConfidence * 100).toFixed(1)}%-${(group.maxConfidence * 100).toFixed(1)}%`}
174+
</span>
175+
</div>
176+
{group.minDistance !== undefined && (
177+
<span className="bg-gradient-to-br from-[#00d4aa] to-[#00b894] text-white px-3 py-1 rounded text-sm font-semibold shadow-[0_2px_4px_rgba(0,212,170,0.3)]">
178+
{group.minDistance === group.maxDistance
179+
? `${group.minDistance.toFixed(2)}m`
180+
: `${group.minDistance.toFixed(2)}-${group.maxDistance!.toFixed(2)}m`}
181+
</span>
182+
)}
183+
</div>
184+
);
185+
});
186+
187+
GroupedDetectionCard.displayName = 'GroupedDetectionCard';
188+
189+
export default memo(DetectionInfo);

‎src/frontend/src/components/MetadataWidget.tsx‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* SPDX-FileCopyrightText: 2025 robot-visual-perception
3+
*
4+
* SPDX-License-Identifier: MIT
5+
*/
6+
7+
import { useState } from 'react';
8+
import { MetadataFrame } from './video/VideoOverlay';
9+
import DetectionInfo from './DetectionInfo';
10+
import StreamInfo, { type StreamInfoProps } from './StreamInfo';
11+
12+
export interface MetadataWidgetProps {
13+
/** Detection metadata from AI backend */
14+
detectionMetadata?: MetadataFrame | null;
15+
/** Stream-related metadata (resolution, network quality, etc.) */
16+
streamMetadata?: StreamInfoProps;
17+
/** Optional: Start with grouped view */
18+
defaultGrouped?: boolean;
19+
/** Whether the widget is currently open */
20+
isOpen: boolean;
21+
/** Callback to toggle widget visibility */
22+
onToggle: () => void;
23+
}
24+
25+
/**
26+
* MetadataWidget orchestrates DetectionInfo and StreamInfo components
27+
*
28+
* Displays:
29+
* - Stream Info: Network quality, video quality metrics
30+
* - Detection Info: Object detections with grouping option
31+
* - Toggle button for showing/hiding the widget
32+
*/
33+
function MetadataWidget({
34+
detectionMetadata,
35+
streamMetadata,
36+
defaultGrouped = false,
37+
isOpen,
38+
onToggle,
39+
}: MetadataWidgetProps) {
40+
const [showGrouped, setShowGrouped] = useState(defaultGrouped);
41+
42+
const hasDetections =
43+
detectionMetadata && detectionMetadata.detections.length > 0;
44+
45+
return (
46+
<>
47+
{/* Toggle button */}
48+
<button
49+
onClick={onToggle}
50+
className="fixed right-5 top-[80px] z-50 bg-[#404040] hover:bg-[#505050] text-[#00d4ff] rounded-lg p-2 transition-colors border border-[#555] shadow-lg"
51+
aria-label="Toggle Metadata Widget"
52+
>
53+
<svg
54+
width="24"
55+
height="24"
56+
viewBox="0 0 24 24"
57+
fill="none"
58+
stroke="currentColor"
59+
strokeWidth="2"
60+
strokeLinecap="round"
61+
strokeLinejoin="round"
62+
>
63+
{isOpen ? <path d="M9 18l6-6-6-6" /> : <path d="M15 18l-6-6 6-6" />}
64+
</svg>
65+
</button>
66+
67+
{/* Widget content */}
68+
{isOpen && (
69+
<div className="fixed right-5 top-[120px] w-[320px] max-h-[calc(100vh-140px)] overflow-y-auto z-50 transition-all duration-300">
70+
<div className="space-y-4">
71+
{/* Stream Info Section */}
72+
{streamMetadata && <StreamInfo {...streamMetadata} />}
73+
74+
{/* Detection Info Section */}
75+
{hasDetections && (
76+
<div>
77+
{/* Toggle button for grouped/detail view */}
78+
<div className="mb-3 flex justify-end">
79+
<button
80+
onClick={() => setShowGrouped(!showGrouped)}
81+
className="text-xs px-3 py-1.5 bg-[#404040] hover:bg-[#505050] text-[#00d4ff] rounded border border-[#555] transition-colors"
82+
>
83+
{showGrouped ? 'Show Details' : 'Group by Type'}
84+
</button>
85+
</div>
86+
87+
<DetectionInfo
88+
detections={detectionMetadata.detections}
89+
showGrouped={showGrouped}
90+
/>
91+
</div>
92+
)}
93+
94+
{/* No detections message */}
95+
{!hasDetections && detectionMetadata && (
96+
<div className="bg-[#2a2a2a] border border-[#404040] p-5 rounded-lg shadow-[0_4px_20px_rgba(0,0,0,0.4)]">
97+
<p className="text-[#888] text-sm italic text-center">
98+
No objects detected
99+
</p>
100+
</div>
101+
)}
102+
</div>
103+
</div>
104+
)}
105+
</>
106+
);
107+
}
108+
109+
export default MetadataWidget;

0 commit comments

Comments
 (0)