[frontend] screenshot through element capture api when available

Author: velzie

frontend/src/components/Shell.tsx

@@ -1,6 +1,7 @@
 import { css, type Component } from "dreamland/core";
 import { browser } from "../Browser";
 import { forceScreenshot, popTab, pushTab } from "../Browser";
+import { takeScreenshotGDM } from "../screenshot";
 
 export const Shell: Component = function (cx) {
 	pushTab.listen((tab) => {
@@ -53,41 +54,51 @@ export const Shell: Component = function (cx) {
 				</div>
 			</div>
 		);
-
-		setInterval(() => forceScreenshot(tab), 1000);
 	});
 	popTab.listen((tab) => {
 		const container = cx.root.querySelector(`[data-tab="${tab.id}"]`);
 		if (!container) throw new Error(`No container found for tab ${tab.id}`);
 		container.remove();
 	});
-	// forceScreenshot.listen(async (tab) => {
-	// 	const container = cx.root.querySelector(
-	// 		`[data-tab="${tab.id}"]`
-	// 	) as HTMLElement;
-	// 	if (!container) throw new Error(`No container found for tab ${tab.id}`);
+	forceScreenshot.listen(async (tab) => {
+		const container = cx.root.querySelector(
+			`[data-tab="${tab.id}"]`
+		) as HTMLElement;
+		if (!container) throw new Error(`No container found for tab ${tab.id}`);
 
-	// 	// tab.screenshot = URL.createObjectURL(await toBlob(container));
-	// });
+		let blob = await takeScreenshotGDM(container);
+		if (blob) tab.screenshot = URL.createObjectURL(blob);
+	});
 
 	return <div></div>;
 };
+
 Shell.style = css`
 	:scope {
 		flex: 1;
 		overflow: hidden;
 		width: 100%;
+		position: relative;
 	}
 	.unfocus {
 		pointer-events: none;
 	}
 	.container {
+		position: absolute;
 		width: 100%;
 		height: 100%;
-		display: none;
+		display: flex;
+		top: 0;
+		left: 0;
+		z-index: -1;
+		/*display: none;*/
+
+		/*https://screen-share.github.io/element-capture/#elements-eligible-for-restriction*/
+		isolation: isolate;
+		transform-style: flat;
 	}
 	.container.active {
-		display: flex;
+		z-index: 1;
 	}
 	.container .devtools {
 		position: relative;

frontend/src/components/TabStrip.tsx

@@ -7,7 +7,7 @@ import { IconButton } from "./IconButton";
 import type { Tab } from "../Tab";
 // import html2canvas from "html2canvas";
 import { setContextMenu } from "./Menu";
-import { browser } from "../Browser";
+import { browser, forceScreenshot } from "../Browser";
 
 export const DragTab: Component<
 	{
@@ -74,6 +74,7 @@ export const DragTab: Component<
 				this.transitionend();
 			}}
 			on:mouseenter={() => {
+				forceScreenshot(this.tab);
 				if (hoverTimeout) clearTimeout(hoverTimeout);
 				hoverTimeout = window.setTimeout(() => {
 					this.tooltipActive = true;
@@ -87,11 +88,14 @@ export const DragTab: Component<
 			<div class="tooltip" class:active={use(this.tooltipActive)}>
 				<span class="title">{use(this.tab.title)}</span>
 				<span class="hostname">{use(this.tab.url.hostname)}</span>
-				{/*<img src={use(this.tab.screenshot)} class="img" />*/}
-				<div
-					style={use`background-image: -moz-element(#tab${this.tab.id})`}
-					class="img"
-				></div>
+				{window.chrome ? (
+					<img src={use(this.tab.screenshot)} class="img" />
+				) : (
+					<div
+						style={use`background-image: -moz-element(#tab${this.tab.id})`}
+						class="img"
+					></div>
+				)}
 			</div>
 			<div
 				class="dragroot"
@@ -430,7 +434,9 @@ export const Tabs: Component<
 
 		calcDragPos(e, tab);
 
-		if (this.activetab != tab) this.activetab = tab;
+		if (this.activetab != tab) {
+			this.activetab = tab;
+		}
 	};
 
 	const transitionend = () => {

frontend/src/screenshot.ts

@@ -0,0 +1,50 @@
+let stream: MediaStream | null;
+let track: MediaStreamTrack | null;
+let video: HTMLVideoElement | null;
+
+export async function startCapture() {
+	if (stream) return;
+
+	stream = await navigator.mediaDevices.getDisplayMedia({
+		video: {
+			displaySurface: "window",
+		},
+		//@ts-expect-error untyped
+		preferCurrentTab: true,
+		audio: false,
+	});
+	track = stream.getVideoTracks()[0];
+
+	video = document.createElement("video");
+	video.srcObject = stream;
+	video.play();
+}
+
+const canvas = document.createElement("canvas");
+const ctx = canvas.getContext("2d")!;
+export async function takeScreenshotGDM(
+	container: HTMLElement
+): Promise<Blob | null> {
+	if (!stream || !track || !video) return null;
+	try {
+		//@ts-expect-error untyped
+		const restrictionTarget = await RestrictionTarget.fromElement(container);
+		//@ts-expect-error untyped
+		await track.restrictTo(restrictionTarget);
+
+		await new Promise((r) => setTimeout(r, 100));
+		const settings = track.getSettings();
+		canvas.width = settings.width!;
+		canvas.height = settings.height!;
+		ctx.drawImage(video, 0, 0);
+		const blob: Blob | null = await new Promise((resolve) =>
+			canvas.toBlob(resolve, "image/png")
+		);
+
+		return blob;
+	} catch (e) {
+		console.error(e);
+
+		return null;
+	}
+}