Skip to content

Commit c4b72ed

Browse files
authored
Merge pull request #11 from JingxuanKang/cursor/tab-seat-image-paste-e6e1
Tab-seat image paste via focused ClipboardEvent
2 parents 365131d + 77ffdd1 commit c4b72ed

8 files changed

Lines changed: 289 additions & 25 deletions

File tree

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,9 @@ Passwords are stored as per-user salted scrypt hashes. Sign-in is rate limited p
104104

105105
## Clipboard
106106

107-
The clipboard is two-way between your machine and the desk on the VNC (first-login) path. Text and screenshots both work there.
107+
The clipboard is two-way between your machine and the desk on the exclusive VNC path. Text and screenshots both work there (gpc-clipd / xclip). That path does not need the debug port.
108108

109-
Tab seats cannot use that X11 clip relay — it is one clipboard for the whole desktop, not one per tab. v1 pastes text into the focused tab via CDP and copies the tab's selection (or the page clipboard when the page allows it). Images are not pasted on tab seats.
109+
Tab seats exist only when multi-user / CDP is on for that account. They cannot use the X11 clip relay — it is one clipboard for the whole desktop, not one per tab. They paste text via CDP `Input.insertText` and images (png/jpeg/webp) via a synthetic `ClipboardEvent` on `document.activeElement`. Click the composer first; if nothing is focused the UI says to click the input. They never write the shared X11 clipboard.
110110

111111
## Sharing and memory isolation
112112

README.zh-CN.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,9 @@ Cloud / CI 虚拟机往往不跑桌面镜像,无法证明 Chromium 是活的
104104

105105
## 剪贴板
106106

107-
VNC(首次登录)路径上,剪贴板在本机和桌面之间是双向的,文字和截图都可以。
107+
独占 VNC 路径上,剪贴板在本机和桌面之间是双向的,文字和截图都可以(gpc-clipd / xclip),不依赖调试口
108108

109-
分屏席位不能走那条 X11 剪贴板中继——整台桌面只有一块剪贴板,无法按标签页隔离。v1 用 CDP 把文字贴进当前标签、读取该页选区(页面允许时也读页内剪贴板)。分屏里暂不支持粘贴图片
109+
分屏席位只在该账号开启「多人分屏 / CDP」后才有。它们不能走 X11 剪贴板中继——整台桌面只有一块剪贴板,无法按标签页隔离。文字走 CDP `Input.insertText`,图片(png/jpeg/webp)在当前焦点元素上派发合成的 `ClipboardEvent`。先点一下输入框;没有焦点时界面会提示「点一下输入框再粘贴」。不会写入整桌 X11 剪贴板
110110

111111
## 分享与记忆隔离
112112

gateway/server.mjs

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import { applyDeskProxyLive, applyDeskProxiesLive } from "../lib/proxy.mjs";
1515
import { createSeatRegistry, parseTabSeatCap, publicSeat } from "../lib/seats.mjs";
1616
import { applyDeskCdpLive, attachSeatTarget, closeTarget, createParkedChatGPTTab, evaluateOnTarget, forgetDeskBrowser, targetExists } from "../lib/cdp.mjs";
1717
import { startSeatScreencast } from "../lib/screencast.mjs";
18+
import { applyTabPastePlan, tabPastePlan } from "../lib/tab-paste.mjs";
1819
import { WebSocketServer } from "ws";
1920

2021
const PORT = Number(process.env.PORT || 8080);
@@ -468,16 +469,20 @@ async function handleApi(req, res, url, sess) {
468469
const ct = String(req.headers["content-type"] || "text/plain; charset=utf-8");
469470
const tab = seats.ofUser(id, sess.user.id);
470471
if (tab?.mode === "tab") {
471-
if (!ct.startsWith("text/")) return json(res, 400, { error: "分屏席位暂不支持粘贴图片" });
472+
const plan = tabPastePlan(ct, body);
473+
if (plan.error) return json(res, plan.status || 400, { error: plan.error });
472474
try {
473-
const text = body.toString("utf8");
474475
const attached = await attachSeatTarget(id, tab.targetId);
475476
try {
476-
await attached.cdp.send("Input.insertText", { text: text.slice(0, 64 * 1024) }, attached.sessionId);
477+
const out = await applyTabPastePlan(
478+
(method, params) => attached.cdp.send(method, params, attached.sessionId),
479+
plan,
480+
);
481+
if (!out.ok) return json(res, out.status || 400, { error: out.error || "无法粘贴" });
482+
return json(res, 200, { ok: true, scoped: "tab", kind: out.kind });
477483
} finally {
478484
await attached.release();
479485
}
480-
return json(res, 200, { ok: true, scoped: "tab" });
481486
} catch {
482487
return json(res, 502, { error: "无法粘贴" });
483488
}

gateway/web/app.js

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -453,7 +453,7 @@ function renderSettings() {
453453
<section class="panel">
454454
<div class="panel-head">
455455
<b>复制粘贴</b>
456-
<em>在桌面画面里直接 ⌘C / ⌘V,双向生效。多人分屏时只支持文字,图片粘贴暂不可用。</em>
456+
<em>在桌面画面里直接 ⌘C / ⌘V,双向生效。独占 VNC 走整桌剪贴板(文字和截图)。开启多人分屏后,分屏席位把文字和图片贴进当前输入框(先点一下输入框)。</em>
457457
</div>
458458
</section>
459459
<section class="panel">
@@ -813,16 +813,35 @@ async function onTabPaste(e) {
813813
if (state.view !== "desk" || state.deskMode !== "tab" || !e.clipboardData) return;
814814
e.preventDefault();
815815
e.stopPropagation();
816+
for (const it of Array.from(e.clipboardData.items || [])) {
817+
if (it.kind === "file" && /^image\/(png|jpeg|jpg|webp)$/i.test(it.type)) {
818+
const f = it.getAsFile();
819+
if (!f) continue;
820+
const mime = it.type === "image/jpg" ? "image/jpeg" : it.type;
821+
const buf = await f.arrayBuffer();
822+
setChip("pasting");
823+
toast("pasting", "image");
824+
const r = await sendDeskPaste(buf, mime);
825+
if (r.ok) {
826+
setChip("image");
827+
toast("图片已粘贴到输入框", "image");
828+
} else {
829+
setChip("waiting");
830+
toast(r.error || "图片没贴进去");
831+
}
832+
return;
833+
}
834+
}
816835
const text = e.clipboardData.getData("text/plain");
817836
if (!text) {
818-
toast("分屏席位暂不支持粘贴图片");
837+
toast("点一下输入框再粘贴");
819838
return;
820839
}
821840
setChip("pasting");
822-
const ok = await sendDeskPaste(text, "text/plain; charset=utf-8");
823-
if (ok) setChip("text", text);
841+
const r = await sendDeskPaste(text, "text/plain; charset=utf-8");
842+
if (r.ok) setChip("text", text);
824843
else setChip("waiting");
825-
toast(ok ? "已粘贴" : "文字没贴进去");
844+
toast(r.ok ? "已粘贴" : r.error || "文字没贴进去");
826845
}
827846

828847
function bindDesk() {
@@ -909,14 +928,16 @@ function bindDesk() {
909928

910929
async function sendDeskPaste(body, mime) {
911930
const id = state.deskId;
912-
if (!id) return false;
931+
if (!id) return { ok: false, error: "无法粘贴" };
913932
const r = await fetch(`/api/desks/${id}/paste`, {
914933
method: "POST",
915934
credentials: "same-origin",
916935
headers: { "content-type": mime },
917936
body,
918937
});
919-
return r.ok;
938+
const data = await r.json().catch(() => ({}));
939+
if (!r.ok) return { ok: false, error: data.error || "无法粘贴" };
940+
return { ok: true };
920941
}
921942

922943
async function copyFromDesk() {
@@ -992,23 +1013,23 @@ function bindClipboard(iframe) {
9921013
remember("image", buf, mime, "", f);
9931014
setChip("pasting");
9941015
toast("pasting", "image");
995-
const ok = await sendDeskPaste(buf, mime);
996-
if (ok) setChip("image");
1016+
const pasted = await sendDeskPaste(buf, mime);
1017+
if (pasted.ok) setChip("image");
9971018
else setChip("waiting");
998-
toast(ok ? "图片已粘贴到输入框" : "图片没贴进去,点顶栏再试", "image");
999-
return ok;
1019+
toast(pasted.ok ? "图片已粘贴到输入框" : pasted.error || "图片没贴进去,点顶栏再试", "image");
1020+
return pasted.ok;
10001021
}
10011022
}
10021023
const text = cd.getData("text/plain");
10031024
if (text) {
10041025
remember("text", text, "text/plain; charset=utf-8", text, null);
10051026
setChip("pasting");
1006-
const ok = await sendDeskPaste(text, "text/plain; charset=utf-8");
1007-
if (ok) setChip("text", text);
1027+
const pasted = await sendDeskPaste(text, "text/plain; charset=utf-8");
1028+
if (pasted.ok) setChip("text", text);
10081029
else setChip("waiting");
10091030
const preview = text.replace(/\s+/g, " ").trim().slice(0, 18);
1010-
toast(ok ? `已粘贴「${preview}${text.trim().length > 18 ? "…" : ""}」` : "文字没贴进去");
1011-
return ok;
1031+
toast(pasted.ok ? `已粘贴「${preview}${text.trim().length > 18 ? "…" : ""}」` : pasted.error || "文字没贴进去");
1032+
return pasted.ok;
10121033
}
10131034
return false;
10141035
};

lib/screencast.mjs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* (page viewport only — no tab strip). Pointer / keyboard go back as Input.*.
44
*/
55
import { attachSeatTarget } from "./cdp.mjs";
6+
import { applyTabPastePlan, tabPasteFromMessage } from "./tab-paste.mjs";
67

78
export const KEY_CODES = {
89
Enter: 13,
@@ -239,8 +240,10 @@ export async function startSeatScreencast({
239240
}
240241
return;
241242
}
242-
if (msg.type === "paste" && typeof msg.text === "string") {
243-
await sendOnSession("Input.insertText", { text: msg.text.slice(0, 64 * 1024) });
243+
if (msg.type === "paste") {
244+
const plan = tabPasteFromMessage(msg);
245+
const out = await applyTabPastePlan(sendOnSession, plan);
246+
if (!out.ok) sendJson(ws, { type: "error", error: out.error || "无法粘贴" });
244247
}
245248
} catch {
246249
/* input is best-effort */

lib/tab-paste.mjs

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
/**
2+
* Tab-seat paste. Text uses CDP Input.insertText. Images dispatch a synthetic
3+
* ClipboardEvent on document.activeElement (contenteditable / textarea / input).
4+
* Never writes the shared desk clipboard — that would leak to the other seat.
5+
*/
6+
7+
export const TAB_PASTE_TEXT_MAX = 64 * 1024;
8+
export const TAB_PASTE_IMAGE_MAX = 4 * 1024 * 1024;
9+
export const TAB_PASTE_IMAGE_TYPES = ["image/png", "image/jpeg", "image/webp"];
10+
export const TAB_PASTE_NEED_FOCUS = "点一下输入框再粘贴";
11+
12+
function mimeOf(contentType) {
13+
const ct = String(contentType || "")
14+
.split(";")[0]
15+
.trim()
16+
.toLowerCase();
17+
if (ct === "image/jpg") return "image/jpeg";
18+
return ct;
19+
}
20+
21+
export function filenameForMime(mime) {
22+
if (mime === "image/jpeg") return "image.jpg";
23+
if (mime === "image/webp") return "image.webp";
24+
return "image.png";
25+
}
26+
27+
export function classifyTabPaste(contentType, byteLength) {
28+
const mime = mimeOf(contentType);
29+
const n = Number(byteLength) || 0;
30+
if (!n) return { error: "空内容", status: 400 };
31+
if (mime.startsWith("text/") || mime === "" || mime === "application/json") {
32+
if (n > TAB_PASTE_TEXT_MAX) return { error: "太大了", status: 413 };
33+
return { kind: "text", mime: mime || "text/plain" };
34+
}
35+
if (TAB_PASTE_IMAGE_TYPES.includes(mime)) {
36+
if (n > TAB_PASTE_IMAGE_MAX) return { error: "太大了", status: 413 };
37+
return { kind: "image", mime };
38+
}
39+
return { error: "分屏席位只支持文字和图片(png/jpeg/webp)", status: 400 };
40+
}
41+
42+
/**
43+
* Runtime.evaluate body. Uses document.activeElement only — no chatgpt.com
44+
* class names or composer selectors. If nothing pasteable is focused, returns
45+
* { error: "need-focus" } so the UI can say TAB_PASTE_NEED_FOCUS.
46+
*/
47+
export function imagePasteExpression({ mime, base64, filename } = {}) {
48+
const m = JSON.stringify(String(mime || "image/png"));
49+
const b = JSON.stringify(String(base64 || ""));
50+
const n = JSON.stringify(String(filename || filenameForMime(mime)));
51+
return `(() => {
52+
const mime = ${m};
53+
const b64 = ${b};
54+
const name = ${n};
55+
const el = document.activeElement;
56+
const tag = el && el.tagName ? String(el.tagName).toUpperCase() : "";
57+
const type = el && el.type != null ? String(el.type) : "text";
58+
const focused = !!(
59+
el &&
60+
el !== document.body &&
61+
el !== document.documentElement &&
62+
(
63+
el.isContentEditable === true ||
64+
tag === "TEXTAREA" ||
65+
(tag === "INPUT" && /^(text|search|url|email|password|tel|number)?$/i.test(type))
66+
)
67+
);
68+
if (!focused) return { ok: false, error: "need-focus" };
69+
const bin = atob(b64);
70+
const bytes = new Uint8Array(bin.length);
71+
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
72+
const file = new File([bytes], name, { type: mime });
73+
const dt = new DataTransfer();
74+
dt.items.add(file);
75+
let ev;
76+
try {
77+
ev = new ClipboardEvent("paste", { bubbles: true, cancelable: true, clipboardData: dt });
78+
} catch (e) {
79+
ev = new Event("paste", { bubbles: true, cancelable: true });
80+
}
81+
if (!ev.clipboardData) {
82+
Object.defineProperty(ev, "clipboardData", { value: dt, configurable: true });
83+
}
84+
el.dispatchEvent(ev);
85+
return { ok: true, kind: "image" };
86+
})()`;
87+
}
88+
89+
export function tabPastePlan(contentType, bytes) {
90+
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || []);
91+
const classified = classifyTabPaste(contentType, buf.length);
92+
if (classified.error) return classified;
93+
if (classified.kind === "text") {
94+
return {
95+
kind: "text",
96+
method: "Input.insertText",
97+
params: { text: buf.toString("utf8").slice(0, TAB_PASTE_TEXT_MAX) },
98+
};
99+
}
100+
return {
101+
kind: "image",
102+
method: "Runtime.evaluate",
103+
params: {
104+
expression: imagePasteExpression({
105+
mime: classified.mime,
106+
base64: buf.toString("base64"),
107+
filename: filenameForMime(classified.mime),
108+
}),
109+
awaitPromise: true,
110+
returnByValue: true,
111+
},
112+
};
113+
}
114+
115+
export function interpretTabPasteEvaluate(value) {
116+
if (value?.ok) return { ok: true, kind: value.kind || "image" };
117+
if (value?.error === "need-focus") {
118+
return { ok: false, error: TAB_PASTE_NEED_FOCUS, status: 400, code: "TAB_PASTE_NEED_FOCUS" };
119+
}
120+
return { ok: false, error: value?.error || "无法粘贴", status: 502 };
121+
}
122+
123+
export async function applyTabPastePlan(send, plan) {
124+
if (!plan || plan.error) {
125+
return { ok: false, error: plan?.error || "无法粘贴", status: plan?.status || 400 };
126+
}
127+
if (plan.kind === "text") {
128+
await send("Input.insertText", plan.params);
129+
return { ok: true, kind: "text" };
130+
}
131+
const raw = await send("Runtime.evaluate", plan.params);
132+
if (raw?.exceptionDetails) return { ok: false, error: "无法粘贴", status: 502 };
133+
return interpretTabPasteEvaluate(raw?.result?.value ?? raw);
134+
}
135+
136+
export function tabPasteFromMessage(msg) {
137+
if (!msg || msg.type !== "paste") return null;
138+
if (typeof msg.text === "string") {
139+
return tabPastePlan("text/plain; charset=utf-8", Buffer.from(msg.text, "utf8"));
140+
}
141+
if (typeof msg.image === "string") {
142+
return tabPastePlan(msg.mime || "image/png", Buffer.from(msg.image, "base64"));
143+
}
144+
return { error: "空内容", status: 400 };
145+
}

0 commit comments

Comments
 (0)