From 5af7660c10f7cff9bdc3c7f3b17f49658e3b4ef4 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:51:30 +0200 Subject: [PATCH 1/3] feat(terminal): kitty graphics state, replay, and child input encoding The session's terminal becomes the owner of two things a consumer cannot correctly own itself. Kitty graphics. libghostty holds the image storage; `pty-terminal::graphics` turns its borrowed handles into owned values (image bytes, placement identity, resolved source crop, rendered pixel and cell size, position in the window that was read) and puts the storage back on the wire for the ATTACH/PEEK replay. That last part is the point: a client that attaches after the child drew an image never saw the DATA that carried it, and libghostty's VT serialization keeps the placeholder cells but neither the images nor the placements. One bound covers state and wire, so nothing the terminal accepts is unreachable by a late client. A virtual placement is located by decoding its placeholder cells, which is what survives scrolling and a windowed read. Child input. Keys (kitty keyboard included), mouse, focus, and paste are encoded inside the terminal, from the terminal's own state, because what the child expects depends on modes the child itself set: DECCKM, the keypad mode, modifyOtherKeys, the kitty flags, the tracking mode and report format, bracketed paste. A second encoder outside would be a second implementation of the kitty keyboard protocol. Cell pixel metrics travel from the client on ATTACH and RESIZE as an optional suffix older readers ignore: they come from a font on the client's host, and a placement that named neither `c=` nor `r=` derives its cell extent from them. docs/decisions/0012-kitty-graphics-replay.md records the deviations, the bounds, and the test index. Refs #3 agent-identity: dev3.direct.omp.2gz9tcpa agent-persona: generalist agent-supervisor: unavailable agent-tool: OMP agent-tool-version: 18.1.2 agent-runtime: OMP 18.1.2 tooling-profile: dotfiles@7534055 --- Cargo.lock | 81 ++ Cargo.toml | 1 + crates/pty-core/src/protocol.rs | 47 + crates/pty-core/tests/protocol.rs | 51 +- crates/pty-terminal/Cargo.toml | 1 + crates/pty-terminal/src/actor.rs | 239 +++- crates/pty-terminal/src/graphics.rs | 1099 ++++++++++++++++++ crates/pty-terminal/src/handle.rs | 296 ++++- crates/pty-terminal/src/input.rs | 320 +++++ crates/pty-terminal/src/lib.rs | 14 + crates/pty-terminal/src/screenshot.rs | 4 +- crates/pty-terminal/src/serialize.rs | 52 +- crates/pty-terminal/tests/graphics.rs | 931 +++++++++++++++ crates/pty-terminal/tests/handle.rs | 228 ++++ crates/pty-terminal/tests/input.rs | 242 ++++ crates/pty-terminal/tests/replay.rs | 2 +- crates/pty/src/daemon/clients.rs | 27 +- crates/pty/src/daemon/lifecycle.rs | 22 +- docs/decisions/0012-kitty-graphics-replay.md | 210 ++++ 19 files changed, 3840 insertions(+), 27 deletions(-) create mode 100644 crates/pty-terminal/src/graphics.rs create mode 100644 crates/pty-terminal/src/input.rs create mode 100644 crates/pty-terminal/tests/graphics.rs create mode 100644 crates/pty-terminal/tests/input.rs create mode 100644 docs/decisions/0012-kitty-graphics-replay.md diff --git a/Cargo.lock b/Cargo.lock index c79e032..839e1d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -172,6 +178,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -360,6 +375,15 @@ dependencies = [ "regex", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filedescriptor" version = "0.8.3" @@ -383,6 +407,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -747,6 +782,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -1027,6 +1082,19 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -1130,6 +1198,7 @@ version = "0.13.0-rust" dependencies = [ "libc", "libghostty-vt", + "png", "portable-pty", "pty-core", "serde_json", @@ -1498,6 +1567,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.3" @@ -2151,6 +2226,12 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 0f1b9e7..bf40fcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1.0.229", features = ["derive"] } serde_json = { version = "1.0.151", features = ["preserve_order"] } indexmap = { version = "2.14.0", features = ["serde"] } notify = "8.2.0" +png = "0.18.1" toml = { version = "1.1.3", features = ["preserve_order"] } # Workspace crates. diff --git a/crates/pty-core/src/protocol.rs b/crates/pty-core/src/protocol.rs index 748378c..2164843 100644 --- a/crates/pty-core/src/protocol.rs +++ b/crates/pty-core/src/protocol.rs @@ -114,11 +114,35 @@ fn size_payload(rows: u16, cols: u16) -> [u8; 4] { [r[0], r[1], c[0], c[1]] } +/// `rows u16BE, cols u16BE, cell_width u16BE, cell_height u16BE`: the size +/// payload with the client's cell pixel metrics appended. +/// +/// The four extra bytes are an *optional suffix*, which is what makes this +/// safe to send to any peer: every reader of a size payload takes rows and +/// cols from the first four bytes and the frame carries its own length, so a +/// daemon that predates this (the Node one included) reads the size it +/// always read and ignores the rest. +fn size_cell_payload(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> [u8; 8] { + let s = size_payload(rows, cols); + let w = cell_width.to_be_bytes(); + let h = cell_height.to_be_bytes(); + [s[0], s[1], s[2], s[3], w[0], w[1], h[0], h[1]] +} + /// Encode an ATTACH with a terminal size (4-byte payload). pub fn encode_attach(rows: u16, cols: u16) -> Vec { encode_packet(MessageType::Attach, &size_payload(rows, cols)) } +/// Encode an ATTACH that also declares the client's cell pixel size +/// (8-byte payload; see [`decode_cell`]). +pub fn encode_attach_with_cell(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> Vec { + encode_packet( + MessageType::Attach, + &size_cell_payload(rows, cols, cell_width, cell_height), + ) +} + /// Encode a DETACH. pub fn encode_detach() -> Vec { encode_packet(MessageType::Detach, &[]) @@ -129,6 +153,14 @@ pub fn encode_resize(rows: u16, cols: u16) -> Vec { encode_packet(MessageType::Resize, &size_payload(rows, cols)) } +/// Encode a RESIZE that also declares the client's cell pixel size. +pub fn encode_resize_with_cell(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> Vec { + encode_packet( + MessageType::Resize, + &size_cell_payload(rows, cols, cell_width, cell_height), + ) +} + /// Encode a GEOMETRY (effective shared rows/cols, server → client). pub fn encode_geometry(rows: u16, cols: u16) -> Vec { encode_packet(MessageType::Geometry, &size_payload(rows, cols)) @@ -177,6 +209,21 @@ pub fn decode_size(payload: &[u8]) -> (u16, u16) { (rows, cols) } +/// The cell pixel size a client appended to an ATTACH or RESIZE payload, or +/// `None` when it sent the plain 4-byte size or a degenerate zero. +/// +/// Cell metrics are the client's to know — they come from its font, on its +/// host — so a session daemon can only be told. `None` means nobody has, and +/// the reader keeps its own deterministic fallback. +pub fn decode_cell(payload: &[u8]) -> Option<(u16, u16)> { + if payload.len() < 8 { + return None; + } + let width = u16::from_be_bytes([payload[4], payload[5]]); + let height = u16::from_be_bytes([payload[6], payload[7]]); + (width > 0 && height > 0).then_some((width, height)) +} + /// Decode a GEOMETRY payload (rows, cols); same layout and fallback as /// [`decode_size`]. pub fn decode_geometry(payload: &[u8]) -> (u16, u16) { diff --git a/crates/pty-core/tests/protocol.rs b/crates/pty-core/tests/protocol.rs index 3fada09..2754343 100644 --- a/crates/pty-core/tests/protocol.rs +++ b/crates/pty-core/tests/protocol.rs @@ -1,9 +1,10 @@ //! Port of the pty project's `tests/protocol.test.ts`. use pty_core::protocol::{ - MAX_PACKET_LENGTH, MessageType, PacketReader, decode_exit, decode_geometry, decode_size, - encode_attach, encode_data, encode_detach, encode_exit, encode_geometry, encode_packet, - encode_resize, encode_screen, encode_status, encode_status_response, + MAX_PACKET_LENGTH, MessageType, PacketReader, decode_cell, decode_exit, decode_geometry, + decode_size, encode_attach, encode_attach_with_cell, encode_data, encode_detach, encode_exit, + encode_geometry, encode_packet, encode_resize, encode_resize_with_cell, encode_screen, + encode_status, encode_status_response, }; use pty_core::stats::{ClientStats, ConnectionStats, Constrains, StatsResult}; @@ -36,6 +37,50 @@ fn attach_byte_identical_to_hand_built_packet() { ); } +/// The cell pixel size is an optional suffix on a size payload: a reader that +/// does not know about it takes the same rows and cols it always did, which +/// is what makes this safe to send to any daemon. +#[test] +fn attach_can_declare_a_cell_size_without_changing_the_size_it_carries() { + let with_cell = encode_attach_with_cell(24, 80, 9, 18); + assert_eq!( + with_cell, + encode_packet(MessageType::Attach, &[0, 24, 0, 80, 0, 9, 0, 18]) + ); + + let mut reader = PacketReader::new(); + let packets = reader.feed(&with_cell).unwrap(); + assert_eq!(packets[0].type_, MessageType::Attach); + assert_eq!( + decode_size(&packets[0].payload), + (24, 80), + "the size is where it always was" + ); + assert_eq!(decode_cell(&packets[0].payload), Some((9, 18))); +} + +#[test] +fn resize_can_declare_a_cell_size() { + let mut reader = PacketReader::new(); + let packets = reader.feed(&encode_resize_with_cell(30, 100, 7, 15)).unwrap(); + assert_eq!(packets[0].type_, MessageType::Resize); + assert_eq!(decode_size(&packets[0].payload), (30, 100)); + assert_eq!(decode_cell(&packets[0].payload), Some((7, 15))); +} + +/// No declaration is the normal case, and it must not be mistaken for one. +#[test] +fn a_plain_size_payload_declares_no_cell() { + assert_eq!(decode_cell(&encode_attach(24, 80)[5..]), None); + assert_eq!(decode_cell(&encode_resize(24, 80)[5..]), None); + assert_eq!(decode_cell(&[]), None); + assert_eq!( + decode_cell(&[0, 24, 0, 80, 0, 0, 0, 0]), + None, + "a zero cell is no cell" + ); +} + /// node: tests/protocol.test.ts:288-297 #[test] fn round_trips_geometry() { diff --git a/crates/pty-terminal/Cargo.toml b/crates/pty-terminal/Cargo.toml index 92bf1b2..8850cf4 100644 --- a/crates/pty-terminal/Cargo.toml +++ b/crates/pty-terminal/Cargo.toml @@ -12,6 +12,7 @@ libghostty-vt.workspace = true portable-pty.workspace = true pty-core.workspace = true libc.workspace = true +png.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/crates/pty-terminal/src/actor.rs b/crates/pty-terminal/src/actor.rs index 4cd1ca2..ee948f0 100644 --- a/crates/pty-terminal/src/actor.rs +++ b/crates/pty-terminal/src/actor.rs @@ -16,6 +16,8 @@ use std::rc::Rc; use libghostty_vt::style::RgbColor; use libghostty_vt::terminal::{Options, Terminal}; +use crate::graphics::{self, CellSize, GraphicsOptions, GraphicsState, ImageBytes}; +use crate::input::{self, KeyEvent, MouseEvent}; use crate::queries; use crate::screenshot::{self, Screenshot}; use crate::serialize::{self, SerializeOpts}; @@ -66,6 +68,15 @@ pub struct Modes { pub mouse_1002: bool, /// `?1003`. pub mouse_1003: bool, + /// `?9` — X10 mouse reporting: button presses only, no releases, no + /// motion, and no wheel. Deliberately *not* part of + /// [`Modes::mouse_tracking`]: an input encoder has to treat it + /// differently from `?1000`/`?1002`/`?1003`, which do report the wheel. + pub mouse_9: bool, + /// `?1` (DECCKM) — application cursor keys: the child wants `SS3 A` for + /// Up, not `CSI A`. Node does not track it; an input encoder cannot + /// encode an arrow key correctly without it. + pub app_cursor: bool, /// `?1049` / `?1047` / `?47`. pub alt_screen: bool, /// `?25 l` seen and not yet undone by `?25 h`. @@ -84,12 +95,20 @@ impl Modes { self.mouse_1000 || self.mouse_1002 || self.mouse_1003 } + /// Whether any mouse reporting at all is on, X10 included. Use + /// [`Modes::mouse_tracking`] for "reports the wheel". + pub fn mouse_reporting(&self) -> bool { + self.mouse_9 || self.mouse_tracking() + } + fn apply_dec(&mut self, mode: u16, set: bool, events: &mut Vec) { match mode { 1006 => self.sgr_mouse = set, 1000 => self.mouse_1000 = set, 1002 => self.mouse_1002 = set, 1003 => self.mouse_1003 = set, + 9 => self.mouse_9 = set, + 1 => self.app_cursor = set, 1049 | 1047 | 47 => self.alt_screen = set, 25 => { if set { @@ -138,6 +157,14 @@ pub struct TerminalActor { events: Vec, last_title: Option, scrollback: usize, + /// How big a cell is on the surface that draws this terminal, zero when + /// undeclared. It comes from the client (a font on its host), travels on + /// ATTACH and RESIZE, and decides the cell extent of any placement that + /// did not name `c=`/`r=` itself. + cell: CellSize, + /// Whether kitty graphics are on, and with what bounds. Off means the + /// storage limit is zero, which is libghostty's own "protocol disabled". + graphics: Option, /// The normal screen, serialized at the moment the child switched to the /// alternate one. libghostty gives no reader for the screen that is not /// active, and a replay has to carry both: a client that reconnects while @@ -184,6 +211,8 @@ impl TerminalActor { events: Vec::new(), last_title: None, scrollback, + cell: CellSize::default(), + graphics: None, normal_replay: None, } } @@ -193,6 +222,198 @@ impl TerminalActor { TerminalActor::new(24, 80, DEFAULT_SCROLLBACK) } + /// Turn kitty graphics on: a bounded image storage, the cell metrics + /// every placement question is answered in, and a PNG decoder for + /// `f=100` transmissions (installed on this thread, where the terminal + /// lives). Returns false when libghostty refused the options. + /// + /// `storage_bytes` is clamped to [`graphics::MAX_STORAGE_BYTES`] so + /// everything the terminal accepts can also be replayed; read back what + /// took effect with [`TerminalActor::graphics_options`]. A zero + /// `opts.cell` leaves the metrics undeclared — see + /// [`TerminalActor::set_cell_size`]. + /// + /// Off by default: an owner that never reads images should not let a + /// child make the terminal hold any. + pub fn enable_graphics(&mut self, opts: GraphicsOptions) -> bool { + let opts = GraphicsOptions { + storage_bytes: opts.storage_bytes.min(graphics::MAX_STORAGE_BYTES), + ..opts + }; + let previous = self.graphics; + if opts.cell.is_declared() { + self.cell = opts.cell; + } + if self.apply_cell().is_err() { + self.rollback_graphics(previous); + return false; + } + if self.term.set_apc_max_bytes_kitty(opts.apc_max_bytes).is_err() + || self + .term + .set_kitty_image_storage_limit(opts.storage_bytes) + .is_err() + { + self.rollback_graphics(previous); + return false; + } + // The child must not be able to name a path or a shared-memory + // segment the owner never authorized: inline transmission only. + let _ = self.term.set_kitty_image_from_file_allowed(false); + let _ = self.term.set_kitty_image_from_temp_file_allowed(false); + let _ = self.term.set_kitty_image_from_shared_mem_allowed(false); + if !graphics::install_png_decoder() { + self.rollback_graphics(previous); + return false; + } + self.graphics = Some(GraphicsOptions { + cell: self.cell, + ..opts + }); + true + } + + /// Put the storage limit back where a failed [`TerminalActor::enable_graphics`] + /// found it. + /// + /// Leaving a non-zero limit behind after a failure is the worst of both + /// worlds: `self.graphics` is `None`, so a child can fill the storage + /// with images that no read can reach and `clear_graphics` will not + /// free. + fn rollback_graphics(&mut self, previous: Option) { + let limit = previous.map(|o| o.storage_bytes).unwrap_or(0); + let _ = self.term.set_kitty_image_storage_limit(limit); + self.graphics = previous; + } + + /// Push the current grid and cell metrics into libghostty. + fn apply_cell(&mut self) -> Result<(), libghostty_vt::error::Error> { + let cell = self.cell.or_fallback(); + self.term + .resize(self.cols().max(1), self.rows().max(1), cell.width, cell.height) + .map(|_| ()) + } + + /// Declare how big a cell is on the surface that draws this terminal. + /// + /// Nothing else can know: the metrics come from a font on the client's + /// host, which a session daemon may never see. They change no bytes — + /// only derived geometry, which is why a placement that named `c=`/`r=` + /// itself is unaffected and one that did not now gets a cell extent that + /// matches what the client will actually draw. + /// + /// A zero width or height clears the declaration and returns the + /// terminal to [`CellSize::FALLBACK`]. + pub fn set_cell_size(&mut self, cell: CellSize) { + if self.cell == cell { + return; + } + self.cell = cell; + let _ = self.apply_cell(); + if let Some(opts) = &mut self.graphics { + opts.cell = cell; + } + } + + /// The graphics options in effect, or `None` when graphics are off. + pub fn graphics_options(&self) -> Option { + self.graphics + } + + /// The image-storage generation: 0 when graphics are off or nothing has + /// ever been stored. Unchanged means the images and the set of placements + /// are identical; geometry can still have moved. + pub fn graphics_generation(&self) -> u64 { + if self.graphics.is_none() { + return 0; + } + graphics::generation(&self.term) + } + + /// The images and placements for the window `scroll_offset` rows above + /// the live viewport — the same window [`TerminalActor::snapshot`] reads, + /// so cell positions in both line up. + pub fn graphics_state(&self, scroll_offset: usize) -> GraphicsState { + if self.graphics.is_none() { + return GraphicsState::default(); + } + graphics::read(&self.term, self.cell, scroll_offset) + } + + /// The pixels of one image, copied out of the storage. `None` when it is + /// not there (a delete won the race). + pub fn image_bytes(&self, id: u32) -> Option { + self.graphics.and_then(|_| graphics::image_bytes(&self.term, id)) + } + + /// Drop every image and placement, keeping the protocol on: what a pane + /// does when it closes or is reused. Zeroing the limit is libghostty's + /// own delete-everything path, so this needs no synthesized escape. + pub fn clear_graphics(&mut self) { + let Some(opts) = self.graphics else { return }; + let _ = self.term.set_kitty_image_storage_limit(0); + let _ = self.term.set_kitty_image_storage_limit(opts.storage_bytes); + } + + /// Raise or lower the storage limit, clamped to + /// [`graphics::MAX_STORAGE_BYTES`] so everything stored can still be + /// replayed. Zero turns the protocol off and deletes everything stored. + /// + /// A non-zero limit goes through [`TerminalActor::enable_graphics`], so + /// there is one path that turns graphics on: raising the limit on an + /// actor that never enabled them installs the PNG decoder and the cell + /// metrics too, instead of leaving a terminal that stores images it + /// cannot decode or measure. + pub fn set_graphics_storage_limit(&mut self, bytes: u64) { + if bytes == 0 { + let _ = self.term.set_kitty_image_storage_limit(0); + self.graphics = None; + return; + } + let opts = self.graphics.unwrap_or(GraphicsOptions { + cell: self.cell, + ..GraphicsOptions::DEFAULT + }); + self.enable_graphics(GraphicsOptions { + storage_bytes: bytes, + ..opts + }); + } + + /// The declared cell metrics, or a zero size when nobody has declared + /// them. [`CellSize::or_fallback`] gives what geometry actually used. + pub fn cell_size(&self) -> CellSize { + self.cell + } + + /// The bytes the child expects for one key event, given the keyboard + /// state it asked for (DECCKM, keypad, `modifyOtherKeys`, kitty + /// keyboard flags). Empty for an event the child should not see. + /// + /// This is the only child key encoder: a consumer that owns a surface + /// sends [`KeyEvent`]s, not bytes (see [`crate::input`]). + pub fn encode_key(&self, ev: &KeyEvent) -> Vec { + input::key(&self.term, ev) + } + + /// The bytes for one mouse event, or `None` when the mode the child + /// chose does not report it (no tracking, or a wheel notch under `?9`) + /// and the surface keeps the event. + pub fn encode_mouse(&self, ev: &MouseEvent) -> Option> { + input::mouse(&self.term, &self.modes, ev, self.cell_size()) + } + + /// The bytes for a focus change, or `None` when the child did not ask + /// for focus events. + pub fn encode_focus(&self, gained: bool) -> Option> { + input::focus(&self.modes, gained) + } + + /// Pasted text, bracketed when the child asked for it. + pub fn encode_paste(&self, text: &str) -> Vec { + input::paste(&self.modes, text) + } + /// The normal screen as it was when the child entered the alternate /// one, or `None` when the normal screen is the active one. pub fn normal_replay(&self) -> Option<&str> { @@ -236,7 +457,7 @@ impl TerminalActor { (false, true) => { self.flush_feed(&mut feed); self.normal_replay = - Some(crate::serialize::vt(&self.term, true)); + Some(crate::serialize::vt(&self.term, true, self.cell)); } // Back on the normal screen: it serializes itself. (true, false) => self.normal_replay = None, @@ -335,9 +556,14 @@ impl TerminalActor { self.events.push(TerminalEvent::Notification(n)); } - /// Resize the terminal (the primary screen reflows). + /// Resize the terminal (the primary screen reflows). The cell metrics + /// stay as they are: a resize is a change of grid, not of font, and + /// dropping them would make every placement's geometry unanswerable. pub fn resize(&mut self, cols: u16, rows: u16) { - let _ = self.term.resize(cols.max(1), rows.max(1), 0, 0); + let cell = self.cell.or_fallback(); + let _ = self + .term + .resize(cols.max(1), rows.max(1), cell.width, cell.height); } /// Full reset (RIS): screen, scrollback, modes, title. The tracked mode @@ -349,6 +575,13 @@ impl TerminalActor { self.modes = Modes::default(); self.shared.borrow_mut().titles.clear(); self.shared.borrow_mut().bells = 0; + // RIS restores libghostty's defaults, which include no image storage + // and no cell metrics. An owner that asked for graphics keeps them + // across the reset that precedes a SCREEN replay — otherwise the + // replay's own images would be rejected. + if let Some(opts) = self.graphics { + self.enable_graphics(opts); + } } /// The plain-text screen: rows right-trimmed of never-written cells diff --git a/crates/pty-terminal/src/graphics.rs b/crates/pty-terminal/src/graphics.rs new file mode 100644 index 0000000..be42ba2 --- /dev/null +++ b/crates/pty-terminal/src/graphics.rs @@ -0,0 +1,1099 @@ +//! Kitty graphics state: the durable, typed image state behind +//! [`crate::actor::TerminalActor`] and [`crate::handle::TerminalHandle`]. +//! +//! libghostty owns the image storage; this module is the boundary that turns +//! its borrowed handles into owned, `Send` values a compositor can hold across +//! frames, and that puts the storage back on the wire for the ATTACH/PEEK +//! replay (see [`replay`]). +//! +//! What a consumer gets: +//! +//! - [`GraphicsState`]: the storage generation, every [`ImageDesc`] that has a +//! placement, and every [`Placement`] with its resolved source crop, cell +//! extent, and position in the window it was read for. +//! - [`ImageBytes`]: the pixels themselves, copied once, on request. Metadata +//! is cheap enough to read per frame; bytes are not, so they are keyed by +//! [`ImageDesc::generation`] and fetched only when that changes. +//! +//! Positions are typed rather than optional numbers ([`PlacementPosition`]): +//! libghostty resolves a cursor-positioned placement to a viewport cell, but a +//! virtual (Unicode placeholder) placement has no position of its own — it is +//! wherever its placeholder cells are. Those cells are ordinary text, each +//! naming its own image row and column, so this module decodes them from the +//! grid; that is what survives scrolling, reflow, and a windowed read. +//! +//! Kitty only: SIXEL and the iTerm2 protocol have no equivalent +//! arbitrary-pane contract and are not read here. + +use libghostty_vt::alloc::{Allocator, Bytes}; +use libghostty_vt::kitty::graphics as gfx; +use libghostty_vt::style::StyleColor; +use libghostty_vt::terminal::{Point, PointCoordinate, PointSpace, Terminal}; + +/// The Unicode placeholder character (U+10EEEE) a virtual placement is drawn +/// with. +pub const PLACEHOLDER: char = '\u{10eeee}'; + +/// Cell pixel metrics. Kitty graphics geometry is defined in pixels, so a +/// terminal that stores images has to know how big a cell is: a placement +/// that did not say `c=`/`r=` gets its cell extent from the image's pixel +/// size divided by this. +/// +/// The metrics belong to whoever draws the cells — a font, on a host the +/// session daemon may never see — so they travel from the client +/// ([`crate::handle::AttachOptions::graphics`], carried on ATTACH and +/// RESIZE) rather than being assumed. Zero means undeclared, and derived +/// geometry uses [`CellSize::FALLBACK`] until someone says otherwise. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CellSize { + /// Cell width in pixels; zero when undeclared. + pub width: u32, + /// Cell height in pixels; zero when undeclared. + pub height: u32, +} + +impl CellSize { + /// The conventional 8x16 monospace cell, used only when nobody has + /// declared the real one. Deterministic on purpose: two clients reading + /// an undeclared terminal must agree, even though both are guessing. + pub const FALLBACK: CellSize = CellSize { + width: 8, + height: 16, + }; + + /// Whether a real cell size was declared. + pub fn is_declared(&self) -> bool { + self.width > 0 && self.height > 0 + } + + /// This size, or [`CellSize::FALLBACK`] when undeclared. + pub fn or_fallback(self) -> CellSize { + if self.is_declared() { + self + } else { + CellSize::FALLBACK + } + } +} + +/// The largest image storage this module supports, and therefore the largest +/// a replay can have to carry: 32 MiB, four 2048x2048 RGBA images. +/// +/// One number bounds both on purpose. A storage limit above what a replay +/// carries would mean a terminal that accepts an image a late client can +/// never receive — a supported state that silently loses data, which is the +/// failure this module exists to prevent. +/// [`crate::actor::TerminalActor::enable_graphics`] clamps to it, so a caller +/// asking for more gets this and can see that it did +/// ([`crate::actor::TerminalActor::graphics_options`]). +pub const MAX_STORAGE_BYTES: u64 = 32 * 1024 * 1024; + +/// How much graphics state a terminal may hold, and how it measures cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GraphicsOptions { + /// Image storage limit in bytes, clamped to [`MAX_STORAGE_BYTES`]. Zero + /// disables the protocol; this is the only bound on how much a child can + /// make the terminal hold, and it is also the bound on a replay. + pub storage_bytes: u64, + /// Cell metrics used for every pixel/cell conversion. Zero means + /// undeclared: nobody has told the terminal how big a cell is, and + /// derived geometry falls back to [`CellSize::FALLBACK`]. + pub cell: CellSize, + /// Cap on the bytes one APC command may buffer (`None` keeps + /// libghostty's default). + pub apc_max_bytes: Option, +} + +impl GraphicsOptions { + /// The full supported storage, a conventional 8x16 cell, libghostty's + /// APC cap. + pub const DEFAULT: GraphicsOptions = GraphicsOptions { + storage_bytes: MAX_STORAGE_BYTES, + cell: CellSize { + width: 8, + height: 16, + }, + apc_max_bytes: None, + }; +} + +impl Default for GraphicsOptions { + fn default() -> Self { + GraphicsOptions::DEFAULT + } +} + +/// Pixel format of stored image data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PixelFormat { + /// 24-bit RGB (kitty `f=24`). + Rgb, + /// 32-bit RGBA (kitty `f=32`). + Rgba, + /// PNG bytes (kitty `f=100`), still encoded. + Png, + /// 8-bit grayscale. + Gray, + /// 8-bit grayscale + alpha. + GrayAlpha, +} + +impl PixelFormat { + /// The kitty `f=` value, or `None` for a format the protocol cannot + /// express (grayscale). + pub fn kitty_format(self) -> Option { + match self { + PixelFormat::Rgb => Some(24), + PixelFormat::Rgba => Some(32), + PixelFormat::Png => Some(100), + PixelFormat::Gray | PixelFormat::GrayAlpha => None, + } + } + + /// Whether the data is raw pixels (and therefore needs `s=`/`v=` when + /// transmitted). + pub fn is_raw(self) -> bool { + !matches!(self, PixelFormat::Png) + } +} + +/// Compression applied to stored image data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Compression { + /// Stored uncompressed. + None, + /// zlib deflate (kitty `o=z`). + ZlibDeflate, +} + +/// One stored image, without its pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ImageDesc { + /// Kitty image id (`i=`). + pub id: u32, + /// Kitty image number (`I=`), zero when the child used none. + pub number: u32, + /// The stamp libghostty assigned when these pixels entered the storage. + /// A cache keyed on `(id, generation)` is never stale: the same id with + /// new pixels gets a new stamp. + pub generation: u64, + /// Width in pixels. + pub width: u32, + /// Height in pixels. + pub height: u32, + /// Pixel format. + pub format: PixelFormat, + /// Compression. + pub compression: Compression, + /// Length of the stored data in bytes. + pub len: usize, +} + +/// A stored image with its pixels copied out. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageBytes { + /// What the pixels describe. + pub desc: ImageDesc, + /// The bytes, exactly as libghostty stores them (`desc.format`). + pub data: Vec, +} + +/// The part of the image a placement shows, in image pixels, already resolved +/// (kitty's "0 means the whole dimension") and clamped to the image. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SourceRect { + /// Left edge in image pixels. + pub x: u32, + /// Top edge in image pixels. + pub y: u32, + /// Width in image pixels. + pub width: u32, + /// Height in image pixels. + pub height: u32, +} + +/// Where a virtual placement's placeholder cells are in the window that was +/// read. +/// +/// A placeholder cell names its own image row and column, so a window that +/// shows only part of an image still says exactly which part: `cell_row` / +/// `cell_col` are the image cell indices of the top-left visible cell, and +/// `origin_row` / `origin_col` are where the image's own cell (0, 0) would be +/// — negative when it has scrolled above the window. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PlaceholderRect { + /// Window row of the first placeholder row. + pub row: u16, + /// Window column of the first placeholder column. + pub col: u16, + /// Placeholder rows present in the window. + pub rows: u16, + /// Placeholder columns present in the window. + pub cols: u16, + /// Image cell row of the cell at (`row`, `col`). + pub cell_row: u16, + /// Image cell column of the cell at (`row`, `col`). + pub cell_col: u16, + /// Window row of the placement's own row 0; negative when scrolled above. + pub origin_row: i32, + /// Window column of the placement's own column 0. + pub origin_col: i32, +} + +/// Where a placement is, in the window it was read for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PlacementPosition { + /// Stored, but nothing of it is in this window: scrolled out, or a + /// virtual placement whose placeholder cells are elsewhere (or gone). + Offscreen, + /// A cursor-positioned placement (`a=p` without `U=1`). `row` is negative + /// when the top of the image has scrolled above the window. + Direct { + /// Window column of the top-left corner. + col: i32, + /// Window row of the top-left corner. + row: i32, + /// Cells wide. + cols: u32, + /// Cells tall. + rows: u32, + }, + /// A virtual placement (`U=1`), located by its placeholder cells. + Placeholder(PlaceholderRect), +} + +/// One placement: an image, where it is, and which part of it shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Placement { + /// The image it draws. + pub image_id: u32, + /// Kitty placement id (`p=`), zero when the child used none. With the + /// image id this is the placement's identity. + pub placement_id: u32, + /// Whether it is a virtual (Unicode placeholder) placement. + pub is_virtual: bool, + /// Kitty `z=`. + pub z: i32, + /// The image generation this placement was resolved against. + pub image_generation: u64, + /// The source crop, resolved and clamped. + pub source: SourceRect, + /// Pixel offset inside the first cell (`X=`, `Y=`). + pub cell_offset: (u32, u32), + /// Rendered size in pixels, after crop and aspect ratio. + pub pixel_size: (u32, u32), + /// Rendered size in cells. + pub cell_size: (u32, u32), + /// The raw `c=` and `r=` the child asked for; zero means "natural size". + pub requested_cells: (u32, u32), + /// Where it is in this window. + pub position: PlacementPosition, +} + +/// Everything a compositor needs for one frame. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GraphicsState { + /// Whether the protocol is on (a non-zero storage limit). + pub enabled: bool, + /// The storage generation. Unchanged means the images and the set of + /// placements are byte-for-byte what they were; geometry may still have + /// moved (scroll, resize), so a dirty frame still re-reads positions. + pub generation: u64, + /// The storage limit in bytes. + pub storage_bytes: u64, + /// The cell metrics the geometry was computed with, always a usable + /// size. + pub cell: CellSize, + /// Whether `cell` is what a client declared, or + /// [`CellSize::FALLBACK`] because nobody has. A consumer that draws + /// pixels should declare its own + /// ([`crate::handle::TerminalHandle::set_cell_size`]) rather than + /// trust a fallback: `c=`/`r=` placements are exact either way, but a + /// placement that left its size implicit is only as right as this. + pub cell_declared: bool, + /// Every image that has at least one placement, by id. + pub images: Vec, + /// Every placement, in libghostty's iteration order. + pub placements: Vec, +} + +impl Default for GraphicsState { + fn default() -> Self { + GraphicsState { + enabled: false, + generation: 0, + storage_bytes: 0, + cell: CellSize::FALLBACK, + cell_declared: false, + images: Vec::new(), + placements: Vec::new(), + } + } +} + +impl GraphicsState { + /// The description of `id`, if it is in this state. + pub fn image(&self, id: u32) -> Option<&ImageDesc> { + self.images.iter().find(|i| i.id == id) + } + + /// The placements of `id`. + pub fn placements_of(&self, id: u32) -> impl Iterator { + self.placements.iter().filter(move |p| p.image_id == id) + } + + /// Placements with anything in the window, in draw order (lowest `z` + /// first, ties in storage order). + pub fn visible(&self) -> Vec<&Placement> { + let mut v: Vec<&Placement> = self + .placements + .iter() + .filter(|p| p.position != PlacementPosition::Offscreen) + .collect(); + v.sort_by_key(|p| p.z); + v + } +} + +// --------------------------------------------------------------------------- +// Reading the storage +// --------------------------------------------------------------------------- + +/// Whether the protocol is enabled on `term`. +pub fn enabled(term: &Terminal) -> bool { + term.kitty_image_storage_limit().unwrap_or(0) > 0 +} + +/// The storage generation, or 0 when the storage is empty or disabled. +/// +/// Cheap: one read, no iteration. A caller that sees an unchanged generation +/// can skip [`read`]'s image work, but not its geometry work. +pub fn generation(term: &Terminal) -> u64 { + term.kitty_graphics() + .and_then(|g| g.generation()) + .unwrap_or(0) +} + +/// Read the whole state for the window starting `scroll_offset` rows above +/// the live viewport (0 = the live viewport), the same window +/// [`crate::snapshot::snapshot`] reads. +/// +/// Positions are relative to that window, so a grid and a state read with the +/// same offset line up cell for cell. +pub fn read(term: &Terminal, cell: CellSize, scroll_offset: usize) -> GraphicsState { + let storage_bytes = term.kitty_image_storage_limit().unwrap_or(0); + let mut state = GraphicsState { + enabled: storage_bytes > 0, + generation: 0, + storage_bytes, + cell: cell.or_fallback(), + cell_declared: cell.is_declared(), + images: Vec::new(), + placements: Vec::new(), + }; + if !state.enabled { + return state; + } + let Ok(graphics) = term.kitty_graphics() else { + return state; + }; + state.generation = graphics.generation().unwrap_or(0); + let Ok(mut iter) = gfx::PlacementIterator::new() else { + return state; + }; + let Ok(mut placements) = iter.update(&graphics) else { + return state; + }; + + // The buffer row this window starts at, the same one + // `crate::snapshot::snapshot` uses. Direct placements resolve in screen + // space, so this is what turns them into window rows. + let window_start = term + .scrollback_rows() + .unwrap_or(0) + .saturating_sub(scroll_offset) as i64; + + // The placeholder scan is one pass over the window and only happens once + // a virtual placement is actually there. + let mut placeholders: Option> = None; + + while let Some(p) = placements.next() { + let Ok(image_id) = p.image_id() else { continue }; + let Some(image) = graphics.image(image_id) else { + continue; + }; + let Some(desc) = describe(image_id, &image) else { + continue; + }; + let is_virtual = p.is_virtual().unwrap_or(false); + let placement_id = p.placement_id().unwrap_or(0); + let source = p + .source_rect(&image) + .map(|r| SourceRect { + x: r.x, + y: r.y, + width: r.width, + height: r.height, + }) + .unwrap_or(SourceRect { + x: 0, + y: 0, + width: desc.width, + height: desc.height, + }); + let pixel_size = p + .pixel_size(&image, term) + .map(|s| (s.width, s.height)) + .unwrap_or((0, 0)); + let cell_size = p + .grid_size(&image, term) + .map(|s| (s.cols, s.rows)) + .unwrap_or((0, 0)); + let position = if is_virtual { + let found = placeholders.get_or_insert_with(|| scan_placeholders(term, scroll_offset)); + // A placeholder cell names a placement by its underline colour, + // and a cell that carries none names the image's default + // placement. So an exact match wins, and a placement falls back + // to the cells that named no placement at all rather than + // reporting itself absent. + found + .iter() + .find(|(i, pl, _)| *i == image_id && *pl == placement_id) + .or_else(|| found.iter().find(|(i, pl, _)| *i == image_id && *pl == 0)) + .map(|(_, _, r)| PlacementPosition::Placeholder(*r)) + .unwrap_or(PlacementPosition::Offscreen) + } else { + // Not `viewport_pos`: that answers only for the live viewport and + // reports nothing for a placement above it, which would make a + // scrolled-back window lose exactly the images it should show. + // The placement's own rectangle resolves in screen space, so it + // answers for any window — including a negative row for a + // placement whose top has scrolled above this one. + match screen_origin(&p, &image, term) { + Some((col, screen_row)) => PlacementPosition::Direct { + col, + row: (screen_row - window_start) as i32, + cols: cell_size.0, + rows: cell_size.1, + }, + None => PlacementPosition::Offscreen, + } + }; + if !state.images.iter().any(|i| i.id == desc.id) { + state.images.push(desc); + } + state.placements.push(Placement { + image_id, + placement_id, + is_virtual, + z: p.z().unwrap_or(0), + image_generation: desc.generation, + source, + cell_offset: (p.x_offset().unwrap_or(0), p.y_offset().unwrap_or(0)), + pixel_size, + cell_size, + requested_cells: (p.columns().unwrap_or(0), p.rows().unwrap_or(0)), + position, + }); + } + state +} + +/// The top-left corner of a cursor-positioned placement, as a column and an +/// absolute buffer (screen-space) row. +/// +/// The placement's own rectangle is the only answer that works for a window +/// other than the live viewport: `PlacementIteration::viewport_pos` is +/// defined against the viewport and reports nothing for a placement above +/// it, which is precisely the placement a scrolled-back reader wants. +fn screen_origin( + p: &gfx::PlacementIteration<'_, '_>, + image: &gfx::Image<'_>, + term: &Terminal, +) -> Option<(i32, i64)> { + let rect = p.rect(image, term).ok()?; + let point = term + .point_from_grid_ref(&rect.start(), PointSpace::Screen) + .ok()??; + Some((point.x as i32, point.y as i64)) +} + +/// The pixels of `id`, copied out of the storage. `None` when the image is +/// gone, so a caller that raced a delete learns it here. +pub fn image_bytes(term: &Terminal, id: u32) -> Option { + let graphics = term.kitty_graphics().ok()?; + let image = graphics.image(id)?; + let desc = describe(id, &image)?; + let data = image.data().ok()?; + Some(ImageBytes { + desc, + data: data.to_vec(), + }) +} + +fn describe(id: u32, image: &gfx::Image<'_>) -> Option { + let width = image.width().ok()?; + let height = image.height().ok()?; + if width == 0 || height == 0 { + return None; + } + let format = match image.format().ok()? { + gfx::ImageFormat::Rgb => PixelFormat::Rgb, + gfx::ImageFormat::Rgba => PixelFormat::Rgba, + gfx::ImageFormat::Png => PixelFormat::Png, + gfx::ImageFormat::Gray => PixelFormat::Gray, + gfx::ImageFormat::GrayAlpha => PixelFormat::GrayAlpha, + _ => return None, + }; + let compression = match image.compression().ok()? { + gfx::Compression::ZlibDeflate => Compression::ZlibDeflate, + _ => Compression::None, + }; + Some(ImageDesc { + id, + number: image.number().unwrap_or(0), + generation: image.generation().unwrap_or(0), + width, + height, + format, + compression, + len: image.data().map(|d| d.len()).unwrap_or(0), + }) +} + +// --------------------------------------------------------------------------- +// Placeholder cells +// --------------------------------------------------------------------------- + +/// The row/column diacritics, in index order (kitty's +/// `gen/rowcolumn-diacritics.txt`). Sorted, so a codepoint's index is a +/// binary search. +const ROWCOLUMN_DIACRITICS: [u32; 297] = [ + 0x0305, 0x030d, 0x030e, 0x0310, 0x0312, 0x033d, 0x033e, 0x033f, 0x0346, 0x034a, 0x034b, + 0x034c, 0x0350, 0x0351, 0x0352, 0x0357, 0x035b, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, 0x0483, 0x0484, 0x0485, + 0x0486, 0x0487, 0x0592, 0x0593, 0x0594, 0x0595, 0x0597, 0x0598, 0x0599, 0x059c, 0x059d, + 0x059e, 0x059f, 0x05a0, 0x05a1, 0x05a8, 0x05a9, 0x05ab, 0x05ac, 0x05af, 0x05c4, 0x0610, + 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, 0x0657, 0x0658, 0x0659, 0x065a, + 0x065b, 0x065d, 0x065e, 0x06d6, 0x06d7, 0x06d8, 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06df, + 0x06e0, 0x06e1, 0x06e2, 0x06e4, 0x06e7, 0x06e8, 0x06eb, 0x06ec, 0x0730, 0x0732, 0x0733, + 0x0735, 0x0736, 0x073a, 0x073d, 0x073f, 0x0740, 0x0741, 0x0743, 0x0745, 0x0747, 0x0749, + 0x074a, 0x07eb, 0x07ec, 0x07ed, 0x07ee, 0x07ef, 0x07f0, 0x07f1, 0x07f3, 0x0816, 0x0817, + 0x0818, 0x0819, 0x081b, 0x081c, 0x081d, 0x081e, 0x081f, 0x0820, 0x0821, 0x0822, 0x0823, + 0x0825, 0x0826, 0x0827, 0x0829, 0x082a, 0x082b, 0x082c, 0x082d, 0x0951, 0x0953, 0x0954, + 0x0f82, 0x0f83, 0x0f86, 0x0f87, 0x135d, 0x135e, 0x135f, 0x17dd, 0x193a, 0x1a17, 0x1a75, + 0x1a76, 0x1a77, 0x1a78, 0x1a79, 0x1a7a, 0x1a7b, 0x1a7c, 0x1b6b, 0x1b6d, 0x1b6e, 0x1b6f, + 0x1b70, 0x1b71, 0x1b72, 0x1b73, 0x1cd0, 0x1cd1, 0x1cd2, 0x1cda, 0x1cdb, 0x1ce0, 0x1dc0, + 0x1dc1, 0x1dc3, 0x1dc4, 0x1dc5, 0x1dc6, 0x1dc7, 0x1dc8, 0x1dc9, 0x1dcb, 0x1dcc, 0x1dd1, + 0x1dd2, 0x1dd3, 0x1dd4, 0x1dd5, 0x1dd6, 0x1dd7, 0x1dd8, 0x1dd9, 0x1dda, 0x1ddb, 0x1ddc, + 0x1ddd, 0x1dde, 0x1ddf, 0x1de0, 0x1de1, 0x1de2, 0x1de3, 0x1de4, 0x1de5, 0x1de6, 0x1dfe, + 0x20d0, 0x20d1, 0x20d4, 0x20d5, 0x20d6, 0x20d7, 0x20db, 0x20dc, 0x20e1, 0x20e7, 0x20e9, + 0x20f0, 0x2cef, 0x2cf0, 0x2cf1, 0x2de0, 0x2de1, 0x2de2, 0x2de3, 0x2de4, 0x2de5, 0x2de6, + 0x2de7, 0x2de8, 0x2de9, 0x2dea, 0x2deb, 0x2dec, 0x2ded, 0x2dee, 0x2def, 0x2df0, 0x2df1, + 0x2df2, 0x2df3, 0x2df4, 0x2df5, 0x2df6, 0x2df7, 0x2df8, 0x2df9, 0x2dfa, 0x2dfb, 0x2dfc, + 0x2dfd, 0x2dfe, 0x2dff, 0xa66f, 0xa67c, 0xa67d, 0xa6f0, 0xa6f1, 0xa8e0, 0xa8e1, 0xa8e2, + 0xa8e3, 0xa8e4, 0xa8e5, 0xa8e6, 0xa8e7, 0xa8e8, 0xa8e9, 0xa8ea, 0xa8eb, 0xa8ec, 0xa8ed, + 0xa8ee, 0xa8ef, 0xa8f0, 0xa8f1, 0xaab0, 0xaab2, 0xaab3, 0xaab7, 0xaab8, 0xaabe, 0xaabf, + 0xaac1, 0xfe20, 0xfe21, 0xfe22, 0xfe23, 0xfe24, 0xfe25, 0xfe26, 0x10a0f, 0x10a38, 0x1d185, + 0x1d186, 0x1d187, 0x1d188, 0x1d189, 0x1d1aa, 0x1d1ab, 0x1d1ac, 0x1d1ad, 0x1d242, 0x1d243, + 0x1d244, +]; + +/// The index this diacritic encodes, if it is one. +fn diacritic_index(c: char) -> Option { + ROWCOLUMN_DIACRITICS + .binary_search(&(c as u32)) + .ok() + .map(|i| i as u16) +} + +/// One accumulating placeholder region. +struct Accum { + image_id: u32, + placement_id: u32, + row: u16, + col: u16, + last_row: u16, + last_col: u16, + cell_row: u16, + cell_col: u16, +} + +/// Walk the window once and collect, per (image id, placement id), the +/// placeholder cells that are in it. +/// +/// The scan is bounded by the window: rows x cols cell reads, never the whole +/// scrollback. Kitty's inheritance rules live in [`placeholder_cell`]. +fn scan_placeholders(term: &Terminal, scroll_offset: usize) -> Vec<(u32, u32, PlaceholderRect)> { + let rows_n = term.rows().unwrap_or(0); + let cols = term.cols().unwrap_or(0); + let base_y = term.scrollback_rows().unwrap_or(0); + let len = term.total_rows().unwrap_or(rows_n as usize); + let start = base_y.saturating_sub(scroll_offset); + let live = start == base_y; + + let mut acc: Vec = Vec::new(); + for r in 0..rows_n { + if start + r as usize >= len { + break; + } + // The cell to the left, when it was a placeholder: what an omitted + // diacritic inherits from. + let mut prev: Option = None; + for x in 0..cols { + let point = if live { + Point::Active(PointCoordinate { x, y: r as u32 }) + } else { + Point::Screen(PointCoordinate { + x, + y: (start + r as usize) as u32, + }) + }; + let Some(cell) = placeholder_cell(term, point, prev) else { + prev = None; + continue; + }; + prev = Some(cell); + match acc + .iter_mut() + .find(|a| a.image_id == cell.image_id && a.placement_id == cell.placement_id) + { + Some(a) => { + a.last_row = a.last_row.max(r); + a.last_col = a.last_col.max(x); + a.row = a.row.min(r); + if x < a.col { + a.col = x; + a.cell_col = cell.cell_col; + } + if r < a.row || (r == a.row && x == a.col) { + a.cell_row = cell.cell_row; + } + } + None => acc.push(Accum { + image_id: cell.image_id, + placement_id: cell.placement_id, + row: r, + col: x, + last_row: r, + last_col: x, + cell_row: cell.cell_row, + cell_col: cell.cell_col, + }), + } + } + } + acc.into_iter() + .map(|a| { + ( + a.image_id, + a.placement_id, + PlaceholderRect { + row: a.row, + col: a.col, + rows: a.last_row - a.row + 1, + cols: a.last_col - a.col + 1, + cell_row: a.cell_row, + cell_col: a.cell_col, + origin_row: a.row as i32 - a.cell_row as i32, + origin_col: a.col as i32 - a.cell_col as i32, + }, + ) + }) + .collect() +} + +/// What one placeholder cell says. Also what the cell to its right inherits +/// when it leaves a diacritic out. +#[derive(Clone, Copy)] +struct PlaceholderCell { + image_id: u32, + placement_id: u32, + cell_row: u16, + cell_col: u16, + /// The high byte of the image id, carried separately because a + /// continuation cell inherits it rather than restating it. + id_high: u16, +} + +/// Decode the cell at `point` as a placeholder, or `None` when it is not one. +/// +/// The image id is the cell's foreground colour — 24 bits in truecolor, or a +/// palette index in 256-colour mode, both of which kitty allows — plus an +/// optional high byte from a third diacritic. The placement id is the +/// underline colour, read the same way; no underline colour means the +/// image's default placement. +/// +/// Kitty's inheritance rules apply to a cell that omits diacritics: it takes +/// the row, the column plus one, and the image-id high byte from the cell to +/// its left. That is what makes the compact form (`U+10EEEE` repeated with no +/// diacritics after the first cell) decode to a real rectangle. +fn placeholder_cell( + term: &Terminal, + point: Point, + prev: Option, +) -> Option { + let g = term.grid_ref(point).ok()?; + let mut buf = [char::default(); 8]; + let n = g.graphemes(&mut buf).ok()?; + let chars = &buf[..n]; + if chars.first() != Some(&PLACEHOLDER) { + return None; + } + let style = g.style().ok()?; + let base = color_id(style.fg_color)?; + let placement_id = color_id(style.underline_color).unwrap_or(0); + // Only a cell that names the same image and placement can be continued. + let left = prev.filter(|p| p.image_id & 0x00ff_ffff == base && p.placement_id == placement_id); + let mut diacritics = chars[1..].iter().filter_map(|&c| diacritic_index(c)); + let cell_row = diacritics + .next() + .or_else(|| left.map(|p| p.cell_row)) + .unwrap_or(0); + let cell_col = diacritics + .next() + .or_else(|| left.map(|p| p.cell_col + 1)) + .unwrap_or(0); + let id_high = diacritics + .next() + .or_else(|| left.map(|p| p.id_high)) + .unwrap_or(0); + Some(PlaceholderCell { + image_id: ((id_high as u32) << 24) | base, + placement_id, + cell_row, + cell_col, + id_high, + }) +} + +/// The 24-bit id a placeholder colour names: truecolor components, or a +/// palette index (kitty's 256-colour form, which limits ids to 8 bits). +/// `None` for a default colour, which names nothing. +fn color_id(color: StyleColor) -> Option { + match color { + StyleColor::Rgb(c) => Some(((c.r as u32) << 16) | ((c.g as u32) << 8) | c.b as u32), + StyleColor::Palette(i) => Some(i.0 as u32), + StyleColor::None => None, + } +} + +// --------------------------------------------------------------------------- +// Replay +// --------------------------------------------------------------------------- + +/// Put the storage back on the wire: one kitty transmission per image, then +/// one placement command per placement, in the protocol the child used. +/// +/// This is what makes graphics survive the ATTACH/PEEK replay. libghostty's +/// VT serialization keeps the placeholder cells (ordinary text with a +/// foreground colour) but not the images or the placements, so a client that +/// only replayed the VT body would hold placeholders naming images it does +/// not have. +/// +/// Everything the storage holds is emitted. The bound on a replay is the +/// bound on the state — [`MAX_STORAGE_BYTES`], which +/// [`crate::actor::TerminalActor::enable_graphics`] clamps the storage limit +/// to — so an image the terminal accepted can always be replayed. A second, +/// smaller replay cap would mean a supported image that a late client can +/// never receive, which is exactly the state this whole module exists to +/// prevent. +/// +/// The block carries no cursor movement of its own beyond a save/restore +/// around a cursor-positioned placement, so it can be appended to a replay +/// payload without disturbing it. +pub fn replay(term: &Terminal, cell: CellSize) -> String { + let cell = cell.or_fallback(); + let state = read(term, cell, 0); + if state.placements.is_empty() { + return String::new(); + } + let rows = term.rows().unwrap_or(0) as i32; + let cols = term.cols().unwrap_or(0) as i32; + + let mut ids: Vec = state.images.iter().map(|i| i.id).collect(); + ids.sort_unstable(); + let mut out = String::new(); + let mut sent: Vec = Vec::with_capacity(ids.len()); + for id in ids { + let Some(bytes) = image_bytes(term, id) else { + continue; + }; + let Some(f) = bytes.desc.format.kitty_format() else { + // Grayscale has no kitty `f=`; nothing correct to emit. + continue; + }; + if bytes.data.is_empty() { + continue; + } + transmit(&mut out, &bytes, f); + sent.push(id); + } + for p in &state.placements { + if !sent.contains(&p.image_id) { + continue; + } + if p.is_virtual { + // A virtual placement's command carries no position — its cells + // do, and they are in the VT body wherever they are, including + // in the scrollback. So where it currently shows has nothing to + // do with whether it can be replayed. + place_virtual(&mut out, p); + continue; + } + // A placement the child put at the cursor is restored by putting the + // cursor back, so it needs a cell in the active area. One whose top + // has scrolled above it is still partly on screen: anchor it at row + // 0 and advance the crop by the rows that are gone, which is what + // the source terminal is showing. + let PlacementPosition::Direct { col, row, .. } = p.position else { + continue; + }; + let clipped = (-row).max(0); + if row >= rows || col < 0 || col >= cols || clipped >= p.cell_size.1 as i32 { + continue; + } + place_direct(&mut out, p, col, row.max(0), clipped as u32, cell); + } + out +} + +/// `ESC _G a=t,...; ESC \`, chunked at 4096 base64 bytes (the +/// protocol's limit, and what OMP emits). +fn transmit(out: &mut String, image: &ImageBytes, f: u32) { + let mut params = format!("a=t,q=2,i={},f={}", image.desc.id, f); + if image.desc.format.is_raw() { + params.push_str(&format!(",s={},v={}", image.desc.width, image.desc.height)); + } + if image.desc.compression == Compression::ZlibDeflate { + params.push_str(",o=z"); + } + let payload = base64(&image.data); + let mut chunks = payload.as_bytes().chunks(4096).peekable(); + let mut first = true; + while let Some(chunk) = chunks.next() { + let more = chunks.peek().is_some(); + let chunk = std::str::from_utf8(chunk).unwrap_or_default(); + if first { + out.push_str("\x1b_G"); + out.push_str(¶ms); + if more { + out.push_str(",m=1"); + } + out.push(';'); + first = false; + } else { + out.push_str("\x1b_Gq=2,m="); + out.push(if more { '1' } else { '0' }); + out.push(';'); + } + out.push_str(chunk); + out.push_str("\x1b\\"); + } +} + +/// `ESC _G a=p,U=1,... ESC \`: the virtual placement the placeholder cells in +/// the VT body refer to. +fn place_virtual(out: &mut String, p: &Placement) { + out.push_str("\x1b_Ga=p,U=1,q=2"); + out.push_str(&placement_params(p)); + out.push_str("\x1b\\"); +} + +/// A cursor-positioned placement: save the cursor, put it on the placement's +/// cell, place, restore. +/// +/// `clipped_rows` is how many of its rows have scrolled above the active +/// area; the crop starts that many cells lower so what is emitted is what is +/// still on screen. +fn place_direct(out: &mut String, p: &Placement, col: i32, row: i32, clipped_rows: u32, cell: CellSize) { + out.push_str(&format!("\x1b7\x1b[{};{}H\x1b_Ga=p,q=2", row + 1, col + 1)); + out.push_str(&placement_params(&clip_top(p, clipped_rows, cell))); + out.push_str("\x1b\\\x1b8"); +} + +/// The same placement with its first `rows` cells of image cut off: the crop +/// moves down and shrinks, and the requested row count follows. +fn clip_top(p: &Placement, rows: u32, cell: CellSize) -> Placement { + if rows == 0 { + return *p; + } + let cut = (rows * cell.height.max(1)).min(p.source.height); + let mut clipped = *p; + clipped.source.y += cut; + clipped.source.height -= cut; + clipped.requested_cells.1 = p.requested_cells.1.saturating_sub(rows); + clipped +} + +/// The parameters both placement forms share. +/// +/// The source rectangle is emitted in full, always: `w=`/`h=` default to 0, +/// which the protocol reads as "the whole image", so omitting them turns a +/// cropped placement into an uncropped one squeezed into the cropped +/// placement's cell box — the wrong pixels, at the right size. +fn placement_params(p: &Placement) -> String { + let mut s = format!(",i={}", p.image_id); + if p.placement_id != 0 { + s.push_str(&format!(",p={}", p.placement_id)); + } + if p.requested_cells.0 != 0 { + s.push_str(&format!(",c={}", p.requested_cells.0)); + } + if p.requested_cells.1 != 0 { + s.push_str(&format!(",r={}", p.requested_cells.1)); + } + s.push_str(&format!( + ",x={},y={},w={},h={}", + p.source.x, p.source.y, p.source.width, p.source.height + )); + if p.cell_offset.0 != 0 { + s.push_str(&format!(",X={}", p.cell_offset.0)); + } + if p.cell_offset.1 != 0 { + s.push_str(&format!(",Y={}", p.cell_offset.1)); + } + if p.z != 0 { + s.push_str(&format!(",z={}", p.z)); + } + s +} + +/// Standard base64, no line breaks: what the protocol's payload is. +fn base64(data: &[u8]) -> String { + const ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let b = [ + chunk[0], + chunk.get(1).copied().unwrap_or(0), + chunk.get(2).copied().unwrap_or(0), + ]; + let n = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | b[2] as u32; + out.push(ALPHABET[(n >> 18) as usize & 63] as char); + out.push(ALPHABET[(n >> 12) as usize & 63] as char); + out.push(if chunk.len() > 1 { + ALPHABET[(n >> 6) as usize & 63] as char + } else { + '=' + }); + out.push(if chunk.len() > 2 { + ALPHABET[n as usize & 63] as char + } else { + '=' + }); + } + out +} + +// --------------------------------------------------------------------------- +// PNG +// --------------------------------------------------------------------------- + +/// Largest image a `f=100` transmission may decode to. Storage is bounded +/// too, but the decode buffer exists before storage sees it. +const MAX_DECODED_PNG_BYTES: usize = 64 * 1024 * 1024; + +/// The decoder libghostty calls for `f=100`. Without one it rejects PNG +/// transmissions, which is what most senders (OMP included) use. +#[derive(Default)] +struct PngDecoder { + buf: Vec, +} + +impl gfx::DecodePng for PngDecoder { + fn decode_png<'alloc>( + &mut self, + alloc: &'alloc Allocator<'_>, + data: &[u8], + ) -> Option> { + let mut decoder = png::Decoder::new(std::io::Cursor::new(data)); + // Normalize to 8 bits per channel: expand a palette, expand a low + // bit depth, strip 16-bit. It does NOT make everything RGBA — + // `Transformations::ALPHA` only adds alpha to a paletted image, and + // there is no grayscale-to-RGB rule at all — so a grayscale PNG + // arrives as `Grayscale` or `GrayscaleAlpha` and the expansion to + // RGBA happens below. Rejecting those instead (which is what + // checking for `Rgba` did) silently drops every monochrome plot and + // optipng-converted asset a child sends. + decoder.set_transformations(png::Transformations::normalize_to_color8()); + let mut reader = decoder.read_info().ok()?; + let size = reader.output_buffer_size()?; + let (width, height) = reader.info().size(); + let rgba_len = (width as usize) + .checked_mul(height as usize)? + .checked_mul(4)?; + if size == 0 || rgba_len == 0 || rgba_len > MAX_DECODED_PNG_BYTES { + return None; + } + self.buf.clear(); + self.buf.resize(size, 0); + let info = reader.next_frame(&mut self.buf).ok()?; + if info.bit_depth != png::BitDepth::Eight { + return None; + } + let src = &self.buf[..info.buffer_size()]; + let mut bytes = Bytes::new_with_alloc(alloc, rgba_len).ok()?; + match info.color_type { + png::ColorType::Rgba => bytes.copy_from_slice(src), + png::ColorType::Rgb => expand(src, 3, &mut bytes, |px, out| { + out.copy_from_slice(&[px[0], px[1], px[2], 0xff]) + }), + png::ColorType::GrayscaleAlpha => expand(src, 2, &mut bytes, |px, out| { + out.copy_from_slice(&[px[0], px[0], px[0], px[1]]) + }), + png::ColorType::Grayscale => expand(src, 1, &mut bytes, |px, out| { + out.copy_from_slice(&[px[0], px[0], px[0], 0xff]) + }), + // `normalize_to_color8` leaves no other 8-bit output type. + _ => return None, + } + Some(gfx::DecodedImage { + width: info.width, + height: info.height, + data: bytes, + }) + } +} + +/// Widen `src`, `stride` bytes per pixel, into 8-bit RGBA. +fn expand(src: &[u8], stride: usize, out: &mut [u8], px: impl Fn(&[u8], &mut [u8])) { + for (i, chunk) in src.chunks_exact(stride).enumerate() { + let Some(dst) = out.get_mut(i * 4..i * 4 + 4) else { + return; + }; + px(chunk, dst); + } +} + +/// Install the PNG decoder for this thread's terminals. Idempotent; the +/// decoder is thread-local in libghostty, so it must run on the thread that +/// owns the terminal. +pub fn install_png_decoder() -> bool { + gfx::set_png_decoder(Some(Box::new(PngDecoder::default()))).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn base64_matches_the_reference_alphabet() { + assert_eq!(base64(b""), ""); + assert_eq!(base64(b"f"), "Zg=="); + assert_eq!(base64(b"fo"), "Zm8="); + assert_eq!(base64(b"foo"), "Zm9v"); + assert_eq!(base64(b"foobar"), "Zm9vYmFy"); + assert_eq!(base64(&[0xff, 0xff, 0xff]), "////"); + assert_eq!(base64(&[0xfb, 0xf0]), "+/A="); + } + + #[test] + fn diacritics_are_an_index_table() { + assert_eq!(diacritic_index('\u{305}'), Some(0)); + assert_eq!(diacritic_index('\u{30d}'), Some(1)); + assert_eq!(diacritic_index('\u{1d244}'), Some(296)); + assert_eq!(diacritic_index('a'), None); + } +} diff --git a/crates/pty-terminal/src/handle.rs b/crates/pty-terminal/src/handle.rs index 67ef57a..2af9cb5 100644 --- a/crates/pty-terminal/src/handle.rs +++ b/crates/pty-terminal/src/handle.rs @@ -20,11 +20,14 @@ use std::time::{Duration, Instant}; use portable_pty::{CommandBuilder, MasterPty, PtySize, native_pty_system}; use pty_core::protocol::{ - MessageType, Packet, PacketReader, decode_exit, decode_size, encode_attach, encode_data, encode_peek, - encode_detach, encode_resize, + MessageType, Packet, PacketReader, decode_exit, decode_size, encode_attach, + encode_attach_with_cell, encode_data, encode_detach, encode_peek, encode_resize, + encode_resize_with_cell, }; use crate::actor::{Modes, Notification, Range, TerminalActor, TerminalEvent}; +use crate::graphics::{CellSize, GraphicsOptions, GraphicsState, ImageBytes}; +use crate::input::{KeyEvent, MouseEvent}; use crate::serialize::SerializeOpts; use crate::snapshot::CellGrid; @@ -66,6 +69,10 @@ pub struct SpawnOptions { pub env: Vec<(String, String)>, /// Scrollback lines. pub scrollback: usize, + /// Kitty graphics: a bounded image storage and the cell metrics + /// placements are measured in. `None` leaves the protocol off, so a + /// child cannot make the terminal hold images nobody reads. + pub graphics: Option, } impl Default for SpawnOptions { @@ -76,6 +83,7 @@ impl Default for SpawnOptions { cwd: None, env: Vec::new(), scrollback: 0, + graphics: None, } } } @@ -92,6 +100,9 @@ pub struct AttachOptions { pub readonly: bool, /// Scrollback lines kept locally. pub scrollback: usize, + /// Kitty graphics, as in [`SpawnOptions::graphics`]. An attached client + /// needs it to hold the images the daemon's replay carries. + pub graphics: Option, } impl Default for AttachOptions { @@ -101,6 +112,7 @@ impl Default for AttachOptions { cols: 80, readonly: false, scrollback: 0, + graphics: None, } } } @@ -121,6 +133,19 @@ pub enum HandleEvent { Exited(i32), /// OSC 9 / 99 / 777. Notification(Notification), + /// The image storage changed (transmit, placement, or delete): the new + /// generation. + Graphics(u64), + /// The socket to the session daemon went away. The handle is still + /// alive and [`TerminalHandle::reconnect`] can be called; the session + /// itself did not exit — that is [`HandleEvent::Exited`]. A consumer + /// with a detached state hears this, rather than polling + /// [`TerminalHandle::connected`]. + Disconnected, + /// A new socket is up under this attempt id, after + /// [`TerminalHandle::reconnect`]. The screen follows as the attempt's + /// first SCREEN, so `Connected` means "reattached", not "ready". + Connected(AttemptId), } enum Msg { @@ -133,6 +158,22 @@ enum Msg { Snapshot { offset: usize, reply: Sender<(u64, CellGrid)> }, Plain { range: Range, reply: Sender }, Serialize { opts: SerializeOpts, reply: Sender }, + Graphics { offset: usize, reply: Sender }, + ImageBytes { id: u32, reply: Sender> }, + ClearGraphics, + CellSize(CellSize), + Key(KeyEvent), + Mouse(MouseEvent), + Focus(bool), + Paste(String), + EncodeKey { + ev: KeyEvent, + reply: Sender>, + }, + EncodeMouse { + ev: MouseEvent, + reply: Sender>>, + }, SetPalette(Vec<(u8, u8, u8)>), Reconnect { reply: Sender> }, Close, @@ -154,6 +195,9 @@ struct State { base_y: usize, len: usize, scrollback: usize, + /// The last published image-storage generation, so a consumer can tell + /// an image change from any other dirty frame without asking the actor. + graphics_generation: u64, snap_cache: Option<(u64, CellGrid)>, } @@ -203,7 +247,8 @@ impl Core { /// fan out events. fn publish(&mut self) { let events = self.actor.take_events(); - let rev = { + let generation = self.actor.graphics_generation(); + let (rev, graphics_changed) = { let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner()); st.rev += 1; st.snap_cache = None; @@ -214,7 +259,9 @@ impl Core { st.title = self.actor.title(); st.base_y = self.actor.base_y(); st.len = self.actor.buffer_length(); - st.rev + let changed = st.graphics_generation != generation; + st.graphics_generation = generation; + (st.rev, changed) }; self.shared.cv.notify_all(); for ev in events { @@ -226,6 +273,9 @@ impl Core { }; self.shared.emit(ev); } + if graphics_changed { + self.shared.emit(HandleEvent::Graphics(generation)); + } self.shared.emit(HandleEvent::Dirty(rev)); } @@ -293,6 +343,7 @@ impl Core { *stream = None; } self.shared.cv.notify_all(); + self.shared.emit(HandleEvent::Disconnected); } fn input(&mut self, data: &[u8]) { @@ -330,7 +381,10 @@ impl Core { return; } if let Some(s) = stream { - let _ = s.write_all(&encode_resize(rows, cols)); + let _ = s.write_all(&match cell_pixels(opts) { + Some((w, h)) => encode_resize_with_cell(rows, cols, w, h), + None => encode_resize(rows, cols), + }); let _ = s.flush(); } // Applied locally as well: a daemon that speaks GEOMETRY will @@ -372,6 +426,7 @@ impl Core { st.connected = true; } self.shared.cv.notify_all(); + self.shared.emit(HandleEvent::Connected(self.attempt)); Ok(()) } @@ -408,6 +463,68 @@ impl Core { Msg::Serialize { opts, reply } => { let _ = reply.send(self.actor.serialize(opts)); } + Msg::Graphics { offset, reply } => { + let _ = reply.send(self.actor.graphics_state(offset)); + } + Msg::ImageBytes { id, reply } => { + let _ = reply.send(self.actor.image_bytes(id)); + } + Msg::ClearGraphics => { + self.actor.clear_graphics(); + self.publish(); + } + Msg::CellSize(cell) => { + self.actor.set_cell_size(cell); + if let Backend::Attach { stream, opts, .. } = &mut self.backend + && !opts.readonly + && let Some(s) = stream + { + // The daemon holds the session's terminal, so it needs + // the metrics too: its own replay answers geometry for + // every other client. + if let Some(g) = &mut opts.graphics { + g.cell = cell; + } + if let Some((w, h)) = cell_pixels(opts) { + let _ = s.write_all(&encode_resize_with_cell( + self.actor.rows(), + self.actor.cols(), + w, + h, + )); + let _ = s.flush(); + } + } + self.publish(); + } + Msg::Key(ev) => { + let bytes = self.actor.encode_key(&ev); + if !bytes.is_empty() { + self.input(&bytes); + } + } + Msg::Mouse(ev) => { + if let Some(bytes) = self.actor.encode_mouse(&ev) { + self.input(&bytes); + } + } + Msg::Focus(gained) => { + if let Some(bytes) = self.actor.encode_focus(gained) { + self.input(&bytes); + } + } + Msg::Paste(text) => { + let bytes = self.actor.encode_paste(&text); + if !bytes.is_empty() { + self.input(&bytes); + } + } + Msg::EncodeKey { ev, reply } => { + let _ = reply.send(self.actor.encode_key(&ev)); + } + Msg::EncodeMouse { ev, reply } => { + let _ = reply.send(self.actor.encode_mouse(&ev)); + } Msg::SetPalette(colors) => { self.actor.set_palette(&colors); self.publish(); @@ -446,6 +563,15 @@ fn run(mut core: Core, rx: Receiver) { core.shutdown(); } +/// The cell pixel size this client declares to the daemon, if any. Only a +/// client that asked for graphics has a reason to: the metrics exist so the +/// session can answer geometry for a placement that left its size implicit. +fn cell_pixels(opts: &AttachOptions) -> Option<(u16, u16)> { + let cell = opts.graphics?.cell; + cell.is_declared() + .then(|| (cell.width.min(u16::MAX as u32) as u16, cell.height.min(u16::MAX as u32) as u16)) +} + /// Connect to the daemon, send ATTACH, and start a reader thread that tags /// every packet with `attempt`. fn connect_and_attach( @@ -460,7 +586,10 @@ fn connect_and_attach( let hello = if opts.readonly { encode_peek(false, false) } else { - encode_attach(opts.rows, opts.cols) + match cell_pixels(opts) { + Some((w, h)) => encode_attach_with_cell(opts.rows, opts.cols, w, h), + None => encode_attach(opts.rows, opts.cols), + } }; (&stream).write_all(&hello)?; (&stream).flush()?; @@ -499,6 +628,22 @@ fn exit_code(status: portable_pty::ExitStatus) -> i32 { } } +/// The actor both constructors build. Graphics have to be turned on here, +/// on the actor thread: libghostty's PNG decoder is thread-local and the +/// terminal is `!Send`. +fn new_actor( + rows: u16, + cols: u16, + scrollback: usize, + graphics: Option, +) -> TerminalActor { + let mut actor = TerminalActor::new(rows, cols, scrollback); + if let Some(opts) = graphics { + actor.enable_graphics(opts); + } + actor +} + /// A live terminal you can write to, resize, and read typed cells from. /// Cheap to share (`Send + Sync`); every method is non-blocking except the /// explicit waits and the reads that must ask the actor thread. @@ -587,11 +732,12 @@ impl TerminalHandle { }); } - let (rows, cols, scrollback) = (opts.rows, opts.cols, opts.scrollback); + let (rows, cols, scrollback, gfx) = + (opts.rows, opts.cols, opts.scrollback, opts.graphics); let core_shared = shared.clone(); std::thread::spawn(move || { let core = Core { - actor: TerminalActor::new(rows, cols, scrollback), + actor: new_actor(rows, cols, scrollback, gfx), attempt, shared: core_shared, backend: Backend::Spawn { master, writer }, @@ -634,7 +780,7 @@ impl TerminalHandle { let core_tx = tx.clone(); std::thread::spawn(move || { let core = Core { - actor: TerminalActor::new(opts.rows, opts.cols, opts.scrollback), + actor: new_actor(opts.rows, opts.cols, opts.scrollback, opts.graphics), attempt, shared: core_shared, backend: Backend::Attach { @@ -801,6 +947,138 @@ impl TerminalHandle { reply_rx.recv().unwrap_or_default() } + /// The kitty graphics state for the window `scroll_offset` rows above + /// the live viewport — the same window [`TerminalHandle::snapshot`] + /// reads, so a grid and a graphics state taken with the same offset line + /// up cell for cell. + /// + /// Empty (and `enabled: false`) when the handle was built without + /// [`SpawnOptions::graphics`] / [`AttachOptions::graphics`]. + pub fn graphics(&self, scroll_offset: usize) -> GraphicsState { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::Graphics { + offset: scroll_offset, + reply: reply_tx, + }) + .is_err() + { + return GraphicsState::default(); + } + reply_rx.recv().unwrap_or_default() + } + + /// The pixels of one image. `None` when it is not stored (any more). + /// Cache the result on [`crate::graphics::ImageDesc::generation`]: it + /// changes whenever the pixels behind an id do. + pub fn image_bytes(&self, id: u32) -> Option { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::ImageBytes { id, reply: reply_tx }) + .is_err() + { + return None; + } + reply_rx.recv().unwrap_or(None) + } + + /// The last published image-storage generation, without asking the + /// actor. Unchanged means the images and the set of placements are + /// identical; placement geometry can still have moved, so a dirty frame + /// still re-reads [`TerminalHandle::graphics`]. + pub fn graphics_generation(&self) -> u64 { + self.state(|st| st.graphics_generation) + } + + /// Drop every image and placement, keeping the protocol on: what a pane + /// does when it closes or is reused for something else. + pub fn clear_graphics(&self) { + let _ = self.tx.send(Msg::ClearGraphics); + } + + /// Declare how big a cell is on the surface that draws this terminal. + /// + /// Only the client knows: the metrics come from its font, on its host. An + /// attached handle also tells the daemon (a RESIZE carrying the cell + /// size), because the session's own terminal is what answers geometry for + /// every other client and for the replay. Nothing about the bytes + /// changes; a placement that named `c=`/`r=` is unaffected, and one that + /// left its size implicit now gets the cell extent this surface will + /// actually draw. + /// + /// Until someone declares it, geometry uses + /// [`crate::graphics::CellSize::FALLBACK`] and + /// [`crate::graphics::GraphicsState::cell_declared`] is false. + pub fn set_cell_size(&self, width: u32, height: u32) { + let _ = self.tx.send(Msg::CellSize(CellSize { width, height })); + } + + /// Send one key event to the child, encoded for the keyboard state the + /// child asked for. Ordered with [`TerminalHandle::write`] and the other + /// `send_*` methods; ignored by a read-only attach. + /// + /// A consumer that reserves keys for itself decides that before calling: + /// this encoder never swallows a key. + pub fn send_key(&self, ev: &KeyEvent) { + let _ = self.tx.send(Msg::Key(ev.clone())); + } + + /// Send one mouse event, if the mode the child chose reports it. Use + /// [`TerminalHandle::encode_mouse`] first when the surface wants to keep + /// the event otherwise (a wheel notch the child would not hear). + pub fn send_mouse(&self, ev: &MouseEvent) { + let _ = self.tx.send(Msg::Mouse(*ev)); + } + + /// Report a focus change, if the child asked for focus events. + pub fn send_focus(&self, gained: bool) { + let _ = self.tx.send(Msg::Focus(gained)); + } + + /// Paste text, bracketed when the child asked for it. Check + /// [`crate::input::paste_is_safe`] first if the surface wants to confirm + /// a multi-line paste. + pub fn send_paste(&self, text: &str) { + let _ = self.tx.send(Msg::Paste(text.to_string())); + } + + /// The bytes [`TerminalHandle::send_key`] would write, without writing + /// them. Empty for an event the child should not see. + pub fn encode_key(&self, ev: &KeyEvent) -> Vec { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::EncodeKey { + ev: ev.clone(), + reply: reply_tx, + }) + .is_err() + { + return Vec::new(); + } + reply_rx.recv().unwrap_or_default() + } + + /// The bytes [`TerminalHandle::send_mouse`] would write, or `None` when + /// the child's mode does not report this event — which is how a surface + /// learns it may keep the wheel for its own scrolling. + pub fn encode_mouse(&self, ev: &MouseEvent) -> Option> { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::EncodeMouse { + ev: *ev, + reply: reply_tx, + }) + .is_err() + { + return None; + } + reply_rx.recv().unwrap_or(None) + } + /// The current revision; bumps on every change. pub fn rev(&self) -> u64 { self.state(|st| st.rev) diff --git a/crates/pty-terminal/src/input.rs b/crates/pty-terminal/src/input.rs new file mode 100644 index 0000000..18a044d --- /dev/null +++ b/crates/pty-terminal/src/input.rs @@ -0,0 +1,320 @@ +//! Encoding semantic input for the child: keys, mouse, focus, paste. +//! +//! This is the other half of the terminal boundary. A consumer that owns a +//! surface (a TUI pane, a GUI widget) knows *what the user did*; only the +//! terminal knows what bytes the child expects for it, because that depends on +//! state the child itself set: DECCKM, the keypad mode, `modifyOtherKeys`, the +//! kitty keyboard flags, the mouse tracking mode and report format, focus +//! reporting, bracketed paste. Splitting that knowledge — semantic events on +//! one side, a second encoder on the other — means two implementations of the +//! kitty keyboard protocol and a class of parity bug nobody can test. +//! +//! So the events here are deliberately dumb and owned ([`KeyEvent`], +//! [`MouseEvent`]), and the encoding happens inside the terminal: +//! [`crate::actor::TerminalActor::encode_key`] and friends, or +//! [`crate::handle::TerminalHandle::send_key`] to encode and write in one +//! ordered step. +//! +//! The encoders themselves are libghostty's ([`libghostty_vt::key`], +//! [`libghostty_vt::mouse`], [`libghostty_vt::focus`], +//! [`libghostty_vt::paste`]), configured from the live terminal. The key and +//! modifier vocabulary is re-exported rather than re-declared: a second +//! `Key` enum would be a translation table that silently drifts. +//! +//! What this module does *not* do: decide anything. It never swallows a key +//! for its own use and it has no notion of a shortcut, a leader, or a detach +//! sequence. A consumer that reserves keys evaluates them before it calls +//! here. + +use libghostty_vt::terminal::Terminal; +use libghostty_vt::{focus, key, mouse, paste}; + +pub use libghostty_vt::key::{Action as KeyAction, Key, KittyKeyFlags, Mods}; +pub use libghostty_vt::mouse::{Action as MouseAction, Button as MouseButton}; + +use crate::actor::Modes; +use crate::graphics::CellSize; + +/// The cell metrics used for mouse coordinates when the terminal has none +/// (kitty graphics are what otherwise gives a terminal a cell size). Only the +/// ratio of position to cell matters for a cell-addressed report, so any +/// consistent pair works; SGR-pixels reports scale with it. +const DEFAULT_CELL: CellSize = CellSize { + width: 8, + height: 16, +}; + +/// One key event from the surface. +/// +/// The three text-ish fields are separate on purpose, because the kitty +/// keyboard protocol reports them separately: +/// +/// - `key` is the logical key (`Key::KeyA`, `Key::ArrowUp`, `Key::Enter`). +/// - `text` is what the key produced with the user's layout and shift state — +/// kitty's "associated text", reported when the child asked for +/// [`KittyKeyFlags::REPORT_ASSOCIATED`]. +/// - `unshifted` is the same key without shift, which is what the shifted-key +/// alternate is derived against for +/// [`KittyKeyFlags::REPORT_ALTERNATES`]. +/// +/// Folding shift into one character upstream loses the alternate, and with it +/// any way for the child to tell `shift+a` from `A` typed on a layout where +/// they differ. +/// +/// A character key needs at least one of `text` and `unshifted` to have an +/// identity the protocol can report: an event that carries neither encodes to +/// nothing under the kitty protocol, because there is no codepoint to name. +/// Named keys (`Key::Enter`, `Key::ArrowUp`) need neither. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyEvent { + /// The logical key. + pub key: Key, + /// Modifiers held. + pub mods: Mods, + /// Press, repeat, or release. Release only reaches the child when it + /// asked for [`KittyKeyFlags::REPORT_EVENTS`]. + pub action: KeyAction, + /// The text the key produced (shift applied), or `None` for a key that + /// produces none. Must not be a C0 control or a platform function-key + /// code: pass `None` and let the logical key speak. + pub text: Option, + /// The same key with no shift applied. + pub unshifted: Option, + /// Modifiers the surface already consumed (a compose or IME step), which + /// the child should not see again. + pub consumed_mods: Mods, + /// Whether this event is part of an in-progress composition. + pub composing: bool, +} + +impl KeyEvent { + /// A plain press of `key` with no modifiers and no text. + pub fn press(key: Key) -> KeyEvent { + KeyEvent { + key, + mods: Mods::empty(), + action: KeyAction::Press, + text: None, + unshifted: None, + consumed_mods: Mods::empty(), + composing: false, + } + } + + /// A press of a character key: `text` is what it typed, `unshifted` the + /// same key without shift. + pub fn typed(key: Key, text: &str, unshifted: Option) -> KeyEvent { + KeyEvent { + text: Some(text.to_string()), + unshifted, + ..KeyEvent::press(key) + } + } + + /// The same event with `mods` held. + pub fn with_mods(mut self, mods: Mods) -> KeyEvent { + self.mods = mods; + self + } +} + +/// One mouse event from the surface, addressed in terminal cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MouseEvent { + /// Press, release, or motion. A wheel notch is a `Press` of + /// [`MouseButton::Four`]/[`MouseButton::Five`] (vertical) or + /// [`MouseButton::Six`]/[`MouseButton::Seven`] (horizontal), as in the + /// protocol. + pub action: MouseAction, + /// The button, or `None` for motion with no button. + pub button: Option, + /// Modifiers held. + pub mods: Mods, + /// Column in the grid, 0-based. + pub col: u16, + /// Row in the grid, 0-based. + pub row: u16, + /// Whether any button is held, which is what distinguishes a drag from a + /// bare move for `?1002`. + pub any_button_pressed: bool, +} + +impl MouseEvent { + /// A press of `button` at a cell. + pub fn press(button: MouseButton, col: u16, row: u16) -> MouseEvent { + MouseEvent { + action: MouseAction::Press, + button: Some(button), + mods: Mods::empty(), + col, + row, + any_button_pressed: true, + } + } + + /// One wheel notch at a cell: `up` chooses button 4 or 5. + pub fn wheel(up: bool, col: u16, row: u16) -> MouseEvent { + MouseEvent { + any_button_pressed: false, + ..MouseEvent::press( + if up { + MouseButton::Four + } else { + MouseButton::Five + }, + col, + row, + ) + } + } + + /// Whether this is a wheel notch rather than a real button. + pub fn is_wheel(&self) -> bool { + matches!( + self.button, + Some(MouseButton::Four | MouseButton::Five | MouseButton::Six | MouseButton::Seven) + ) + } +} + +/// Encode a key event for the child, using `term`'s own keyboard state +/// (DECCKM, keypad, `modifyOtherKeys`, and the kitty keyboard flags). +/// +/// Returns no bytes for an event the child should not see at all — a bare +/// modifier press, or a release while the child has not asked for release +/// events. +pub fn key(term: &Terminal, ev: &KeyEvent) -> Vec { + let Ok(mut encoder) = key::Encoder::new() else { + return Vec::new(); + }; + encoder.set_options_from_terminal(term); + let Ok(mut event) = key::Event::new() else { + return Vec::new(); + }; + event + .set_key(ev.key) + .set_mods(ev.mods) + .set_action(ev.action) + .set_consumed_mods(ev.consumed_mods) + .set_composing(ev.composing); + // The encoder wants the unmodified character; a control or a platform + // function-key code has to be withheld so it uses the logical key. + let text = ev + .text + .as_deref() + .filter(|t| !t.chars().any(is_unencodable_text)); + event.set_utf8(text); + if let Some(c) = ev.unshifted { + event.set_unshifted_codepoint(c); + } + let mut out = Vec::new(); + let _ = encoder.encode_to_vec(&event, &mut out); + out +} + +/// A character the key encoder must not be given as associated text. +fn is_unencodable_text(c: char) -> bool { + let c = c as u32; + c < 0x20 || c == 0x7f || (0xf700..=0xf8ff).contains(&c) +} + +/// Encode a mouse event for the child, using `term`'s tracking mode and +/// report format. +/// +/// `None` means this event is not reportable in the mode the child chose, and +/// the surface keeps it: no tracking at all, a wheel notch under X10 (`?9`, +/// which reports button presses only), or an event the format drops. That +/// distinction is the whole point of the return type — a consumer that wants +/// to scroll its own viewport with the wheel needs to know the child would +/// not have heard it. +pub fn mouse(term: &Terminal, modes: &Modes, ev: &MouseEvent, cell: CellSize) -> Option> { + if !modes.mouse_reporting() { + return None; + } + // X10 reports presses of real buttons and nothing else. libghostty's + // encoder is told the mode, but the rule is stated here so the answer + // does not depend on how the encoder happens to treat a wheel button. + if !modes.mouse_tracking() && (ev.is_wheel() || ev.action != MouseAction::Press) { + return None; + } + let cell = if cell.width == 0 || cell.height == 0 { + DEFAULT_CELL + } else { + cell + }; + let cols = term.cols().unwrap_or(0).max(1) as u32; + let rows = term.rows().unwrap_or(0).max(1) as u32; + + let mut encoder = mouse::Encoder::new().ok()?; + encoder.set_options_from_terminal(term); + encoder + .set_size(mouse::EncoderSize { + screen_width: cols * cell.width, + screen_height: rows * cell.height, + cell_width: cell.width, + cell_height: cell.height, + padding_top: 0, + padding_bottom: 0, + padding_right: 0, + padding_left: 0, + }) + .set_any_button_pressed(ev.any_button_pressed); + + let mut event = mouse::Event::new().ok()?; + event + .set_action(ev.action) + .set_button(ev.button) + .set_mods(ev.mods) + // The middle of the cell: a cell-addressed report rounds to the same + // cell from anywhere inside it, and an SGR-pixels report gets a + // position that is actually in the cell it names. + .set_position(mouse::Position { + x: (ev.col as f32 + 0.5) * cell.width as f32, + y: (ev.row as f32 + 0.5) * cell.height as f32, + }); + + let mut out = Vec::new(); + encoder.encode_to_vec(&event, &mut out).ok()?; + (!out.is_empty()).then_some(out) +} + +/// Encode a focus change, or `None` when the child did not ask for focus +/// events (`?1004`). +pub fn focus(modes: &Modes, gained: bool) -> Option> { + if !modes.focus_events { + return None; + } + let event = if gained { + focus::Event::Gained + } else { + focus::Event::Lost + }; + let mut buf = [0u8; 8]; + let n = event.encode(&mut buf).ok()?; + Some(buf[..n].to_vec()) +} + +/// Encode pasted text: bracketed when the child asked for `?2004`, with +/// control bytes stripped and newlines turned into carriage returns when it +/// did not. +pub fn paste(modes: &Modes, text: &str) -> Vec { + let mut data = text.as_bytes().to_vec(); + // Bracketing adds `ESC[200~` and `ESC[201~`; the encoder never grows the + // payload itself. + let mut buf = vec![0u8; data.len() + 16]; + match paste::encode(&mut data, modes.bracketed_paste, &mut buf) { + Ok(n) => { + buf.truncate(n); + buf + } + Err(_) => Vec::new(), + } +} + +/// Whether pasting `text` is safe without asking the user: a paste carrying a +/// newline (or a forged bracketed-paste end) can run a command the user never +/// typed. Not enforced here — a consumer decides whether to confirm — but the +/// judgement belongs with the terminal, not with each surface. +pub fn paste_is_safe(text: &str) -> bool { + paste::is_safe(text) +} diff --git a/crates/pty-terminal/src/lib.rs b/crates/pty-terminal/src/lib.rs index 01eece4..9ea4c52 100644 --- a/crates/pty-terminal/src/lib.rs +++ b/crates/pty-terminal/src/lib.rs @@ -18,12 +18,21 @@ //! - [`serialize`]: the ATTACH/PEEK replay (Node mode prefix + VT) and the //! plain-text screen (viewport or full scrollback). //! - [`snapshot`]: [`CellGrid`], the typed cell grid for renderers. +//! - [`graphics`]: the kitty graphics state — bounded image bytes, +//! placements, source crops, and where each placement sits in the window +//! that was read — plus the replay block that carries it through +//! ATTACH/PEEK. +//! - [`input`]: the child input encoder — keys (kitty keyboard included), +//! mouse, focus, and paste, encoded from the terminal's own state so no +//! consumer needs a second encoder. //! - [`handle`]: [`TerminalHandle`], a `Send + Sync` handle over an actor //! thread, either spawning a child or attaching to a session daemon. //! - [`screenshot`]: the testkit's [`Screenshot`] capture. pub mod actor; +pub mod graphics; pub mod handle; +pub mod input; pub mod queries; pub mod screenshot; pub mod serialize; @@ -31,9 +40,14 @@ pub mod snapshot; pub mod strip; pub use actor::{Modes, Notification, Range, TerminalActor, TerminalEvent}; +pub use graphics::{ + CellSize, Compression, GraphicsOptions, GraphicsState, ImageBytes, ImageDesc, PixelFormat, + PlaceholderRect, Placement, PlacementPosition, SourceRect, +}; pub use handle::{ AttachOptions, AttemptId, HandleEvent, SessionRef, SpawnOptions, TerminalHandle, }; +pub use input::{Key, KeyAction, KeyEvent, KittyKeyFlags, Mods, MouseAction, MouseButton, MouseEvent}; pub use screenshot::{Screenshot, capture, serialize_for_replay}; pub use serialize::SerializeOpts; pub use snapshot::{CellGrid, CellSnap, ColorSnap, Wide}; diff --git a/crates/pty-terminal/src/screenshot.rs b/crates/pty-terminal/src/screenshot.rs index 56d131e..dcd0e83 100644 --- a/crates/pty-terminal/src/screenshot.rs +++ b/crates/pty-terminal/src/screenshot.rs @@ -34,7 +34,7 @@ impl Screenshot { /// This is the body without Node's mode prefix; the daemon gets the full /// payload from [`crate::TerminalActor::serialize`]. pub fn serialize_for_replay(term: &Terminal) -> String { - serialize::vt(term, true) + serialize::vt(term, true, crate::graphics::CellSize::FALLBACK) } /// Capture the current terminal state into a [`Screenshot`]. @@ -48,7 +48,7 @@ pub fn capture(term: &Terminal) -> Screenshot { while lines.last().map(|l| l.trim().is_empty()).unwrap_or(false) { lines.pop(); } - let ansi = serialize::vt(term, true); + let ansi = serialize::vt(term, true, crate::graphics::CellSize::FALLBACK); let text = lines.join("\n"); Screenshot { lines, text, ansi } } diff --git a/crates/pty-terminal/src/serialize.rs b/crates/pty-terminal/src/serialize.rs index 918c1cd..d9c2cde 100644 --- a/crates/pty-terminal/src/serialize.rs +++ b/crates/pty-terminal/src/serialize.rs @@ -19,6 +19,7 @@ use libghostty_vt::selection::Selection; use libghostty_vt::terminal::{Point, PointCoordinate, Terminal}; use crate::actor::{Modes, TerminalActor}; +use crate::graphics::{self, CellSize}; /// What a replay should carry. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -88,14 +89,36 @@ pub fn mode_prefix(modes: &Modes, include_alt_screen: bool) -> String { /// is the copy the actor took when the child left it. Without it a client /// that reconnects gets a blank normal screen the moment the program exits. /// +/// The prefix comes first because its position is the contract (`ESC[?1049h` +/// at byte 0), which leaves the client on its alternate screen just as the +/// normal half arrives. For text that was invisible — the alternate screen is +/// overwritten immediately afterwards — but kitty image storage is per +/// screen, so the normal screen's images landed in the client's alternate +/// storage and were lost the moment the program exited. One `ESC[?1049l` +/// ahead of the normal half puts it where it belongs; the switch back is +/// [`vt`]'s own `ESC[?1049h`, which also clears and homes the alternate +/// screen the way the body that follows it expects. +/// /// node: src/server.ts:962 and 1017 (`serialize.serialize()`, whose addon /// walks both buffers). pub fn serialize_for_replay(actor: &TerminalActor, opts: SerializeOpts) -> String { - let mut out = mode_prefix(&actor.modes(), opts.include_alt_screen_prefix); + let modes = actor.modes(); + let mut out = mode_prefix(&modes, opts.include_alt_screen_prefix); if let Some(normal) = actor.normal_replay() { - out.push_str(normal); + if opts.include_alt_screen_prefix && modes.alt_screen { + // Back to the normal screen for its own half, then to the + // alternate one again with the cursor homed — Node's + // `ESC[?1049h ESC[H` — because the normal half leaves the cursor + // wherever its own trailing CUP put it and the alternate body + // that follows is written from wherever the cursor is. + out.push_str("\x1b[?1049l"); + out.push_str(normal); + out.push_str("\x1b[?1049h\x1b[H"); + } else { + out.push_str(normal); + } } - out.push_str(&vt(actor.terminal(), opts.scrollback)); + out.push_str(&vt(actor.terminal(), opts.scrollback, actor.cell_size())); out } @@ -242,10 +265,19 @@ fn plain_opts<'t, 's>() -> FormatterOptions<'t, 's> { } /// The VT serialization: cells with styles, then cursor position, modes that -/// differ from their defaults, and the kitty keyboard flags. With -/// `scrollback` the history rows come first; without it only the active area -/// is emitted. -pub fn vt(term: &Terminal, scrollback: bool) -> String { +/// differ from their defaults, the kitty keyboard flags, and the kitty +/// graphics storage. With `scrollback` the history rows come first; without +/// it only the active area is emitted. +/// +/// The graphics block comes last, after the cursor move, because it must not +/// change where the cursor ends up: libghostty's formatter keeps the +/// placeholder cells of a virtual placement (they are ordinary text) but +/// neither the images nor the placements, so without the block a client would +/// replay placeholders naming images it never received +/// (docs/decisions/0012-kitty-graphics-replay.md). It is empty for a terminal +/// with no graphics, so a session that never sent an image is byte-identical +/// to before. +pub fn vt(term: &Terminal, scrollback: bool, cell: CellSize) -> String { let mut out = if scrollback { format(term, vt_opts()) } else { @@ -271,6 +303,7 @@ pub fn vt(term: &Terminal, scrollback: bool) -> String { let cx = term.cursor_x().unwrap_or(0); let cy = term.cursor_y().unwrap_or(0); out.push_str(&format!("\x1b[{};{}H", cy + 1, cx + 1)); + out.push_str(&graphics::replay(term, cell)); out } @@ -320,6 +353,11 @@ mod tests { bracketed_paste: true, focus_events: true, kitty_stack: vec![7, 1], + // Neither of these reaches the prefix: Node does not track them, + // and a replay must not tell the client's terminal to turn a mode + // on that the daemon's own prefix never carried. + mouse_9: true, + app_cursor: true, }; assert_eq!( mode_prefix(&m, true), diff --git a/crates/pty-terminal/tests/graphics.rs b/crates/pty-terminal/tests/graphics.rs new file mode 100644 index 0000000..c783906 --- /dev/null +++ b/crates/pty-terminal/tests/graphics.rs @@ -0,0 +1,931 @@ +//! Kitty graphics state: what a compositor reads, and what survives a +//! replay. +//! +//! Every test drives the actor with the exact bytes OMP writes +//! (`packages/tui/src/terminal-capabilities.ts` `encodeKittyTransmit`, +//! `packages/tui/src/kitty-graphics.ts` `encodeKittyVirtualPlacement` and +//! `encodeKittyPlaceholderGrid`): a `f=100` PNG transmission, a `a=p,U=1` +//! virtual placement, and placeholder cells carrying the image id in their +//! foreground colour and the placement id in their underline colour. + +use std::time::{Duration, Instant}; + +use pty_terminal::graphics::PLACEHOLDER; +use pty_terminal::{ + CellSize, GraphicsOptions, HandleEvent, PixelFormat, PlacementPosition, Range, SerializeOpts, + SpawnOptions, TerminalActor, TerminalHandle, +}; + +/// A 16x8 RGBA PNG. Generated once, checked in as bytes: the point is that +/// the terminal decodes a real PNG, not that this test can make one. +const PNG_16X8_BASE64: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAABAAAAAICAYAAADwdn+XAAAAqUlEQVR4nBXMwYBFIQAAwIewCCGEEEIIIYQQQggh", + "hBBCCCFk8Hc6zHW+72u/PwKRRKZQaXQGk8XmcPm+ICAQSWQKlUZnMFlsDje8IAkIRBKZQqXRGUwWm8NNLygCApFE", + "plBpdAaTxeZwywuagEAkkSlUGp3BZLE53PaCISAQSWQKlUZnMFlsDne8YAkIRBKZQqXRGUwWm8NdLzgCApFEplBp", + "dAaTxeZw+Qf7mTOQwddl2QAAAABJRU5ErkJggg==" +); + +const IMAGE_ID: u32 = 4242; +const PLACEMENT_ID: u32 = 7; + +fn actor(rows: u16, cols: u16, scrollback: usize) -> TerminalActor { + let mut a = TerminalActor::new(rows, cols, scrollback); + assert!( + a.enable_graphics(GraphicsOptions { + cell: CellSize { + width: 8, + height: 16 + }, + ..GraphicsOptions::DEFAULT + }), + "graphics must turn on" + ); + a +} + +/// OMP's `encodeKittyTransmit`: `a=t,f=100,q=2,i=;`. +fn omp_transmit(id: u32) -> String { + format!("\x1b_Ga=t,f=100,q=2,i={id};{PNG_16X8_BASE64}\x1b\\") +} + +/// OMP's `encodeKittyVirtualPlacement`: `a=p,U=1,q=2,i=,p=,c=,r=`. +fn omp_virtual_placement(id: u32, pid: u32, cols: u32, rows: u32) -> String { + format!("\x1b_Ga=p,U=1,q=2,i={id},p={pid},c={cols},r={rows}\x1b\\") +} + +/// The row/column diacritics OMP indexes into, first four entries. +const DIACRITICS: [char; 4] = ['\u{305}', '\u{30d}', '\u{30e}', '\u{310}']; + +/// OMP's `encodeKittyPlaceholderGrid`: image id in the foreground colour, +/// placement id in the underline colour, every cell naming its own row and +/// column. One string per row, no cursor movement. +fn omp_placeholder_rows(id: u32, pid: u32, cols: usize, rows: usize) -> Vec { + let fg = format!( + "\x1b[38;2;{};{};{}m", + (id >> 16) & 0xff, + (id >> 8) & 0xff, + id & 0xff + ); + let ul = format!( + "\x1b[58:2::{}:{}:{}m", + (pid >> 16) & 0xff, + (pid >> 8) & 0xff, + pid & 0xff + ); + (0..rows) + .map(|r| { + let mut line = format!("{fg}{ul}"); + for c in 0..cols { + line.push(PLACEHOLDER); + line.push(DIACRITICS[r]); + line.push(DIACRITICS[c]); + } + line.push_str("\x1b[39;59m"); + line + }) + .collect() +} + +/// The whole OMP render: transmit, then the placement APC in front of the +/// first placeholder row, rows separated by CR/LF. +fn omp_image(id: u32, pid: u32, cols: usize, rows: usize) -> String { + let mut out = omp_transmit(id); + let grid = omp_placeholder_rows(id, pid, cols, rows); + for (i, row) in grid.iter().enumerate() { + if i == 0 { + out.push_str(&omp_virtual_placement(id, pid, cols as u32, rows as u32)); + } else { + out.push_str("\r\n"); + } + out.push_str(row); + } + out +} + +#[test] +fn graphics_are_off_until_the_owner_asks() { + let mut a = TerminalActor::new(10, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let state = a.graphics_state(0); + assert!(!state.enabled, "no storage limit, no protocol"); + assert!(state.images.is_empty()); + assert!(state.placements.is_empty()); + assert_eq!(a.graphics_generation(), 0); + assert!(a.image_bytes(IMAGE_ID).is_none()); +} + +#[test] +fn an_omp_write_gives_image_bytes_placement_identity_crop_and_position() { + let mut a = actor(10, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + + let state = a.graphics_state(0); + assert!(state.enabled); + assert_ne!(state.generation, 0, "a transmit mutates the storage"); + + let image = state.image(IMAGE_ID).expect("the image is stored"); + assert_eq!((image.width, image.height), (16, 8)); + // The PNG was decoded: libghostty stores 8-bit RGBA. + assert_eq!(image.format, PixelFormat::Rgba); + assert_eq!(image.len, 16 * 8 * 4); + assert_ne!(image.generation, 0); + + let bytes = a.image_bytes(IMAGE_ID).expect("bytes are readable"); + assert_eq!(bytes.data.len(), 16 * 8 * 4); + assert_eq!(bytes.desc.generation, image.generation); + + assert_eq!(state.placements.len(), 1); + let p = &state.placements[0]; + assert_eq!((p.image_id, p.placement_id), (IMAGE_ID, PLACEMENT_ID)); + assert!(p.is_virtual, "U=1 is a virtual placement"); + assert_eq!(p.requested_cells, (2, 2)); + // No source rect was sent, so the crop is the whole image. + assert_eq!( + (p.source.x, p.source.y, p.source.width, p.source.height), + (0, 0, 16, 8) + ); + match p.position { + PlacementPosition::Placeholder(r) => { + assert_eq!((r.row, r.col), (0, 0), "written at the home position"); + assert_eq!((r.rows, r.cols), (2, 2)); + assert_eq!((r.cell_row, r.cell_col), (0, 0)); + assert_eq!((r.origin_row, r.origin_col), (0, 0)); + } + other => panic!("expected a placeholder position, got {other:?}"), + } +} + +#[test] +fn a_source_rect_and_offsets_come_back_resolved() { + let mut a = actor(10, 20, 0); + a.write(omp_transmit(IMAGE_ID).as_bytes()); + // A crop with a zero height: kitty means "to the bottom edge". + a.write( + format!("\x1b_Ga=p,U=1,q=2,i={IMAGE_ID},x=4,y=2,w=8,h=0,X=3,Y=5,z=-1,c=2,r=1\x1b\\") + .as_bytes(), + ); + for row in omp_placeholder_rows(IMAGE_ID, 0, 2, 1) { + a.write(row.as_bytes()); + } + let state = a.graphics_state(0); + let p = &state.placements[0]; + assert_eq!(p.placement_id, 0, "no p= means no placement id"); + assert_eq!( + (p.source.x, p.source.y, p.source.width, p.source.height), + (4, 2, 8, 6), + "h=0 resolves to the rest of the image" + ); + assert_eq!(p.cell_offset, (3, 5)); + assert_eq!(p.z, -1); +} + +#[test] +fn scrolling_moves_the_placement_and_scrollback_still_finds_it() { + let mut a = actor(6, 20, 100); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let before = a.graphics_state(0).generation; + + // Push the image up by three rows. + a.write(b"\r\n\r\n\r\n\r\n\r\nx"); + + let state = a.graphics_state(0); + assert_eq!( + state.generation, before, + "scrolling does not touch the storage" + ); + let p = &state.placements[0]; + match p.position { + PlacementPosition::Placeholder(r) => { + assert_eq!(r.row, 0, "the image's second row is now the top row"); + assert_eq!(r.rows, 1, "its first row has scrolled into history"); + assert_eq!(r.cell_row, 1, "the visible row is image row 1"); + assert_eq!(r.origin_row, -1, "image row 0 is one row above"); + } + other => panic!("expected a placeholder position, got {other:?}"), + } + + // The same read one row back into history sees the whole image again. + let scrolled = a.graphics_state(1); + match scrolled.placements[0].position { + PlacementPosition::Placeholder(r) => { + assert_eq!((r.row, r.rows, r.cell_row), (0, 2, 0)); + assert_eq!(r.origin_row, 0); + } + other => panic!("expected a placeholder position, got {other:?}"), + } +} + +#[test] +fn a_resize_keeps_the_image_and_reprojects_it() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let generation = a.graphics_state(0).generation; + + a.resize(10, 8); + + let state = a.graphics_state(0); + assert_eq!(state.generation, generation, "a resize keeps the storage"); + assert_eq!(state.cell, CellSize { width: 8, height: 16 }); + assert!( + a.image_bytes(IMAGE_ID).is_some(), + "the pixels survive a resize" + ); + assert!( + matches!( + state.placements[0].position, + PlacementPosition::Placeholder(_) + ), + "the placeholder cells reflowed with the text, and were found again" + ); +} + +#[test] +fn the_alternate_screen_has_its_own_storage() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(a.graphics_state(0).placements.len(), 1); + + a.write(b"\x1b[?1049h"); + assert!( + a.graphics_state(0).placements.is_empty(), + "the alternate screen starts with no images" + ); + assert!( + a.graphics_state(0).enabled, + "the protocol is on for both screens" + ); + + // An image placed on the alternate screen is the alternate screen's. + a.write(omp_image(99, 1, 1, 1).as_bytes()); + let alt = a.graphics_state(0); + assert_eq!(alt.placements.len(), 1); + assert_eq!(alt.placements[0].image_id, 99); + + a.write(b"\x1b[?1049l"); + let back = a.graphics_state(0); + assert_eq!(back.placements.len(), 1, "the primary screen kept its own"); + assert_eq!(back.placements[0].image_id, IMAGE_ID); +} + +#[test] +fn a_delete_from_the_child_drops_the_placement_and_the_bytes() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let before = a.graphics_state(0).generation; + + // OMP's delete-placement, then its delete-image. + a.write(format!("\x1b_Ga=d,d=i,i={IMAGE_ID},p={PLACEMENT_ID},q=2\x1b\\").as_bytes()); + let after_placement = a.graphics_state(0); + assert!(after_placement.placements.is_empty()); + assert_ne!(after_placement.generation, before, "a delete is a mutation"); + assert!( + a.image_bytes(IMAGE_ID).is_some(), + "deleting a placement keeps the image" + ); + + a.write(format!("\x1b_Ga=d,d=I,i={IMAGE_ID},q=2\x1b\\").as_bytes()); + assert!(a.image_bytes(IMAGE_ID).is_none(), "now the image is gone"); +} + +#[test] +fn clear_graphics_drops_everything_and_keeps_the_protocol() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert!(a.image_bytes(IMAGE_ID).is_some()); + + a.clear_graphics(); + + let state = a.graphics_state(0); + assert!(state.enabled, "the protocol stays on"); + assert!(state.images.is_empty()); + assert!(state.placements.is_empty()); + assert!(a.image_bytes(IMAGE_ID).is_none()); + + // And the terminal still accepts a new image. + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(a.graphics_state(0).placements.len(), 1); +} + +#[test] +fn zeroing_the_limit_turns_the_protocol_off() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + + a.set_graphics_storage_limit(0); + + assert!(!a.graphics_state(0).enabled); + assert!(a.graphics_options().is_none()); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert!( + a.graphics_state(0).placements.is_empty(), + "a disabled terminal stores nothing" + ); +} + +/// The case the Node daemon loses: a client that was not connected when the +/// image was written attaches, gets only the SCREEN payload, and must end up +/// with the same image bytes and the same placement. +#[test] +fn a_late_client_reconstructs_the_image_from_the_replay_alone() { + let mut source = actor(8, 20, 100); + source.write(b"before\r\n"); + source.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + source.write(b"\r\nafter"); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains(&format!("i={IMAGE_ID}")), + "the replay carries the image" + ); + assert!( + payload.contains("a=p,U=1"), + "and the virtual placement that binds the placeholder cells" + ); + + // A fresh terminal that has seen none of the original DATA. + let mut late = actor(8, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + + let state = late.graphics_state(0); + let image = state + .image(IMAGE_ID) + .expect("the late client has the image"); + assert_eq!((image.width, image.height), (16, 8)); + assert_eq!(image.format, PixelFormat::Rgba); + assert_eq!( + late.image_bytes(IMAGE_ID).map(|b| b.data), + source.image_bytes(IMAGE_ID).map(|b| b.data), + "byte for byte the same pixels" + ); + + assert_eq!(state.placements.len(), 1); + let p = &state.placements[0]; + assert_eq!(p.image_id, IMAGE_ID); + assert_eq!(p.placement_id, PLACEMENT_ID, "placement identity survives"); + assert!(p.is_virtual); + assert_eq!(p.requested_cells, (2, 2)); + + // And it is in the same place as in the source terminal. + assert_eq!( + p.position, + source.graphics_state(0).placements[0].position, + "same cells" + ); + assert_eq!(late.plain(Range::Full), source.plain(Range::Full)); +} + +#[test] +fn a_cursor_positioned_placement_replays_at_its_cell() { + let mut source = actor(8, 20, 0); + source.write(b"\r\n\r\n "); + source.write(omp_transmit(IMAGE_ID).as_bytes()); + source.write(format!("\x1b_Ga=p,q=2,i={IMAGE_ID},p=3,c=2,r=1\x1b\\").as_bytes()); + let placed = match source.graphics_state(0).placements[0].position { + PlacementPosition::Direct { col, row, .. } => (col, row), + other => panic!("expected a direct placement, got {other:?}"), + }; + assert_eq!(placed, (2, 2), "at the cursor"); + let cursor = source.cursor(); + + let payload = source.serialize(SerializeOpts::ATTACH); + let mut late = actor(8, 20, 0); + late.reset(); + late.write(payload.as_bytes()); + + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!(state.placements[0].placement_id, 3); + assert_eq!( + state.placements[0].position, + PlacementPosition::Direct { + col: 2, + row: 2, + cols: 2, + rows: 1 + } + ); + assert_eq!( + late.cursor(), + cursor, + "the graphics block leaves the cursor where the replay put it" + ); +} + +#[test] +fn a_replay_without_graphics_is_unchanged() { + let mut plain = TerminalActor::new(6, 20, 0); + plain.write(b"hello\r\nworld"); + let without = plain.serialize(SerializeOpts::ATTACH); + + let mut enabled = actor(6, 20, 0); + enabled.write(b"hello\r\nworld"); + assert_eq!( + enabled.serialize(SerializeOpts::ATTACH), + without, + "a session that never sent an image serializes exactly as before" + ); +} + +/// The handle path: a real child in a real PTY, the state read from another +/// thread. Kitty graphics need a per-thread PNG decoder and an `!Send` +/// terminal, so this is the case that proves the actor thread set both up. +#[test] +fn a_spawned_child_that_draws_an_image_is_queryable_through_the_handle() { + let sequence = omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2); + let h = TerminalHandle::spawn( + "cat", + &[], + SpawnOptions { + rows: 10, + cols: 20, + graphics: Some(GraphicsOptions::DEFAULT), + ..SpawnOptions::default() + }, + ) + .expect("spawn"); + assert!(h.wait_ready(Duration::from_secs(2))); + let events = h.subscribe(); + + // `cat` echoes what we write, so the child is the one emitting the + // sequence into the terminal. + h.write(sequence.as_bytes()); + + let deadline = Instant::now() + Duration::from_secs(5); + let state = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!( + Instant::now() < deadline, + "timed out waiting for the image; screen:\n{}", + h.plain(Range::Full) + ); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + + assert!(state.enabled); + let image = state.image(IMAGE_ID).expect("the image is stored"); + assert_eq!((image.width, image.height), (16, 8)); + assert_eq!( + h.image_bytes(IMAGE_ID).map(|b| b.data.len()), + Some(16 * 8 * 4) + ); + assert_eq!(h.graphics_generation(), state.generation); + + let p = &state.placements[0]; + assert_eq!((p.image_id, p.placement_id), (IMAGE_ID, PLACEMENT_ID)); + assert!(matches!(p.position, PlacementPosition::Placeholder(_))); + + let mut saw_graphics = false; + while let Ok(ev) = events.try_recv() { + if matches!(ev, HandleEvent::Graphics(g) if g == state.generation) { + saw_graphics = true; + } + } + assert!(saw_graphics, "the storage change is announced"); + + h.clear_graphics(); + let deadline = Instant::now() + Duration::from_secs(5); + while !h.graphics(0).placements.is_empty() { + assert!(Instant::now() < deadline, "clear_graphics did not take"); + h.wait_rev(h.rev(), Duration::from_millis(100)); + } + assert!(h.image_bytes(IMAGE_ID).is_none()); + h.kill(); +} + +// ── bounds ── + +/// A raw RGBA transmission, chunked at 4096 base64 bytes the way every real +/// sender does it (kitty's own limit per APC command): `m=1` on every chunk +/// but the last. +fn chunked_rgba_transmit(id: u32, width: u32, height: u32, data: &[u8]) -> String { + const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut payload = String::with_capacity(data.len().div_ceil(3) * 4); + for c in data.chunks(3) { + let b = [c[0], c.get(1).copied().unwrap_or(0), c.get(2).copied().unwrap_or(0)]; + let n = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | b[2] as u32; + payload.push(B64[(n >> 18) as usize & 63] as char); + payload.push(B64[(n >> 12) as usize & 63] as char); + payload.push(if c.len() > 1 { B64[(n >> 6) as usize & 63] as char } else { '=' }); + payload.push(if c.len() > 2 { B64[n as usize & 63] as char } else { '=' }); + } + let mut out = String::with_capacity(payload.len() + 4096); + let mut chunks = payload.as_bytes().chunks(4096).peekable(); + let mut first = true; + while let Some(chunk) = chunks.next() { + let more = chunks.peek().is_some(); + let chunk = std::str::from_utf8(chunk).expect("base64 is ascii"); + if first { + let m = if more { ",m=1" } else { "" }; + out.push_str(&format!( + "\x1b_Ga=t,q=2,i={id},f=32,s={width},v={height}{m};{chunk}\x1b\\" + )); + first = false; + } else { + let m = if more { 1 } else { 0 }; + out.push_str(&format!("\x1b_Gq=2,m={m};{chunk}\x1b\\")); + } + } + out +} + +/// The bound on a replay is the bound on the state. An image well past the +/// 3 MiB a former replay cap allowed is still stored *and* still replayed: +/// otherwise a supported terminal state would be one a late client can never +/// be given, which is the exact failure this module exists to prevent. +#[test] +fn an_image_far_larger_than_three_mib_still_replays() { + // 1024 x 1024 RGBA = 4 MiB, over any per-image cap and under + // MAX_STORAGE_BYTES. + let (w, h) = (1024u32, 1024u32); + let data: Vec = (0..(w * h * 4)) + .map(|i| (i % 251) as u8) + .collect(); + assert!(data.len() > 3 * 1024 * 1024); + + let mut source = actor(8, 20, 0); + source.write(chunked_rgba_transmit(IMAGE_ID, w, h, &data).as_bytes()); + source.write(omp_virtual_placement(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + for row in omp_placeholder_rows(IMAGE_ID, PLACEMENT_ID, 2, 2) { + source.write(row.as_bytes()); + } + let stored = source + .image_bytes(IMAGE_ID) + .expect("the terminal accepted a 4 MiB image"); + assert_eq!(stored.data.len(), data.len()); + assert_eq!(stored.data, data, "chunked transmission arrived intact"); + + let payload = source.serialize(SerializeOpts::ATTACH); + let mut late = actor(8, 20, 0); + late.reset(); + late.write(payload.as_bytes()); + + assert_eq!( + late.image_bytes(IMAGE_ID).map(|b| b.data), + Some(data), + "a late client gets every byte of it" + ); + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!(state.placements[0].placement_id, PLACEMENT_ID); +} + +#[test] +fn the_storage_limit_is_clamped_to_what_a_replay_can_carry() { + let mut a = TerminalActor::new(8, 20, 0); + assert!(a.enable_graphics(GraphicsOptions { + storage_bytes: 8 * 1024 * 1024 * 1024, + ..GraphicsOptions::DEFAULT + })); + assert_eq!( + a.graphics_options().map(|o| o.storage_bytes), + Some(pty_terminal::graphics::MAX_STORAGE_BYTES), + "asking for more than can be replayed gets the supported bound" + ); + assert_eq!( + a.graphics_state(0).storage_bytes, + pty_terminal::graphics::MAX_STORAGE_BYTES + ); + + a.set_graphics_storage_limit(u64::MAX); + assert_eq!( + a.graphics_state(0).storage_bytes, + pty_terminal::graphics::MAX_STORAGE_BYTES + ); +} + +// ── cell metrics ── + +/// A placement that named neither `c=` nor `r=` gets its cell extent from the +/// image's pixel size and the cell size — so the cell size has to be the +/// client's real one, not a guess about its font. +#[test] +fn an_implicit_placement_takes_its_extent_from_the_declared_cell() { + let mut a = TerminalActor::new(8, 40, 0); + assert!(a.enable_graphics(GraphicsOptions { + cell: CellSize::default(), // undeclared + ..GraphicsOptions::DEFAULT + })); + let state = a.graphics_state(0); + assert!(!state.cell_declared, "nobody has said how big a cell is"); + assert_eq!(state.cell, CellSize::FALLBACK); + + // A 32x32 image placed with no c=/r=: 4x2 cells at the 8x16 fallback. + let data = vec![0u8; 32 * 32 * 4]; + a.write(chunked_rgba_transmit(IMAGE_ID, 32, 32, &data).as_bytes()); + a.write(format!("\x1b_Ga=p,q=2,i={IMAGE_ID},p=1\x1b\\").as_bytes()); + assert_eq!(a.graphics_state(0).placements[0].cell_size, (4, 2)); + + // The client says its cells are 16x16: the same image is now 2x2 cells. + a.set_cell_size(CellSize { + width: 16, + height: 16, + }); + let state = a.graphics_state(0); + assert!(state.cell_declared); + assert_eq!(state.cell, CellSize { width: 16, height: 16 }); + assert_eq!( + state.placements[0].cell_size, + (2, 2), + "geometry follows the declared cell" + ); + assert_eq!( + state.placements[0].requested_cells, + (0, 0), + "the child never named a size; only the derived extent moved" + ); + + // And it survives a grid resize, which is a change of grid, not of font. + a.resize(20, 6); + let state = a.graphics_state(0); + assert!(state.cell_declared); + assert_eq!(state.placements[0].cell_size, (2, 2)); +} + +#[test] +fn a_declared_cell_does_not_move_an_explicit_placement() { + let mut a = actor(8, 40, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(a.graphics_state(0).placements[0].cell_size, (2, 2)); + a.set_cell_size(CellSize { + width: 20, + height: 40, + }); + assert_eq!( + a.graphics_state(0).placements[0].cell_size, + (2, 2), + "c=2,r=2 is the child's own answer and no cell size changes it" + ); +} + +// ── replay fidelity (regressions) ── + +/// `w=`/`h=` default to 0 in the protocol, which means "the whole image", so +/// a replay that omits them turns a cropped placement into an uncropped one +/// squeezed into the cropped placement's cell box: the wrong pixels at the +/// right size, on every reattaching client. +#[test] +fn a_cropped_placement_keeps_its_crop_through_a_replay() { + let mut source = actor(8, 20, 0); + source.write(omp_transmit(IMAGE_ID).as_bytes()); + source.write( + format!("\x1b_Ga=p,U=1,q=2,i={IMAGE_ID},p={PLACEMENT_ID},x=4,y=2,w=8,h=6,c=2,r=1\x1b\\") + .as_bytes(), + ); + for row in omp_placeholder_rows(IMAGE_ID, PLACEMENT_ID, 2, 1) { + source.write(row.as_bytes()); + } + let crop = source.graphics_state(0).placements[0].source; + assert_eq!((crop.x, crop.y, crop.width, crop.height), (4, 2, 8, 6)); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains("w=8,h=6"), + "the replay names the crop: {payload:?}" + ); + + let mut late = actor(8, 20, 0); + late.reset(); + late.write(payload.as_bytes()); + let replayed = late.graphics_state(0).placements[0].source; + assert_eq!( + (replayed.x, replayed.y, replayed.width, replayed.height), + (4, 2, 8, 6), + "same pixels, not the whole image measured from (4, 2)" + ); +} + +/// A virtual placement's command carries no position, and its placeholder +/// cells are serialized wherever they are — including the scrollback. So a +/// replay must carry it even when nothing of it is in the viewport, or a +/// late client holds placeholder cells naming an image it never received. +#[test] +fn a_virtual_placement_scrolled_into_history_still_replays() { + let mut source = actor(4, 20, 100); + source.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + // Push it entirely out of the viewport. + source.write(b"\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\nbottom"); + assert_eq!( + source.graphics_state(0).placements[0].position, + PlacementPosition::Offscreen, + "nothing of it is in the live viewport any more" + ); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains("a=p,U=1"), + "the placement is still replayed: {payload:?}" + ); + + let mut late = actor(4, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!( + (state.placements[0].image_id, state.placements[0].placement_id), + (IMAGE_ID, PLACEMENT_ID) + ); + assert!(late.image_bytes(IMAGE_ID).is_some()); +} + +/// A cursor-positioned placement whose top has scrolled above the viewport is +/// still partly on screen. libghostty reports a negative row for exactly this +/// case, so dropping every negative row loses an image the source terminal is +/// showing. +#[test] +fn a_partially_scrolled_direct_placement_replays_clipped() { + let mut source = actor(6, 20, 100); + source.write(omp_transmit(IMAGE_ID).as_bytes()); + source.write(format!("\x1b_Ga=p,q=2,i={IMAGE_ID},p=3,c=2,r=4\x1b\\").as_bytes()); + // Scroll it up by two rows: its top two cell rows are gone, its bottom + // two are still on screen. + source.write(b"\r\n\r\n\r\nx"); + let position = source.graphics_state(0).placements[0].position; + let PlacementPosition::Direct { row, .. } = position else { + panic!("expected a direct placement, got {position:?}"); + }; + assert!(row < 0, "its top has scrolled above the viewport: {row}"); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains(&format!("i={IMAGE_ID},p=3")), + "the visible part is still replayed: {payload:?}" + ); + let mut late = actor(6, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1, "not dropped"); + assert_eq!(state.placements[0].placement_id, 3); + assert!( + state.placements[0].requested_cells.1 < 4, + "the rows that scrolled off are not re-placed: {:?}", + state.placements[0].requested_cells + ); +} + +/// A grayscale PNG is an ordinary PNG. Rejecting it (which is what checking +/// the decoder's output for RGBA did) drops every monochrome plot a child +/// sends, with no diagnostic. +#[test] +fn a_grayscale_png_is_stored_as_rgba() { + const GRAY_4X2_PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAACCAAAAABawyK/AAAAEklEQVR4nGNgsKnYwsDl1rQPAAwmAvkL8nz8AAAAAElFTkSuQmCC"; + let mut a = actor(8, 20, 0); + a.write(format!("\x1b_Ga=t,f=100,q=2,i=55;{GRAY_4X2_PNG}\x1b\\").as_bytes()); + a.write("\x1b_Ga=p,U=1,q=2,i=55,p=1,c=1,r=1\x1b\\".as_bytes()); + for row in omp_placeholder_rows(55, 1, 1, 1) { + a.write(row.as_bytes()); + } + let bytes = a.image_bytes(55).expect("the grayscale PNG was accepted"); + assert_eq!((bytes.desc.width, bytes.desc.height), (4, 2)); + assert_eq!(bytes.desc.format, PixelFormat::Rgba); + assert_eq!(bytes.data.len(), 4 * 2 * 4); + for px in bytes.data.chunks_exact(4) { + assert_eq!(px[0], px[1], "grey expands to r == g == b"); + assert_eq!(px[1], px[2]); + assert_eq!(px[3], 0xff, "with full alpha"); + } +} + +/// Kitty allows a placeholder cell to name its image with a 256-colour +/// foreground, and its own worked example uses one. A reader that insists on +/// truecolor does not see such a cell at all. +#[test] +fn a_palette_foreground_names_a_placeholder_image() { + let mut a = actor(8, 20, 0); + a.write(omp_transmit(42).as_bytes()); + a.write("\x1b_Ga=p,U=1,q=2,i=42,c=1,r=1\x1b\\".as_bytes()); + a.write(format!("\x1b[38;5;42m{PLACEHOLDER}\u{305}\u{305}\x1b[39m").as_bytes()); + let state = a.graphics_state(0); + assert_eq!(state.placements.len(), 1); + match state.placements[0].position { + PlacementPosition::Placeholder(r) => assert_eq!((r.row, r.col), (0, 0)), + other => panic!("a palette foreground must name the image, got {other:?}"), + } +} + +/// Kitty's compact placeholder form leaves the diacritics off every cell but +/// the first: those cells inherit the row, the column plus one, and the image +/// id's high byte from the cell to their left. +#[test] +fn a_bare_continuation_cell_inherits_row_column_and_id_high_byte() { + let id: u32 = 0x0100_0009; + let mut a = actor(8, 20, 0); + a.write(format!("\x1b_Ga=t,f=100,q=2,i={id};{PNG_16X8_BASE64}\x1b\\").as_bytes()); + a.write(format!("\x1b_Ga=p,U=1,q=2,i={id},c=3,r=1\x1b\\").as_bytes()); + // Row 0, column 0, id high byte 1 — then two bare continuation cells. + let first = format!("{PLACEHOLDER}\u{305}\u{305}\u{30d}"); + a.write( + format!("\x1b[38;2;0;0;9m{first}{PLACEHOLDER}{PLACEHOLDER}\x1b[39m").as_bytes(), + ); + + let state = a.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!(state.images[0].id, id, "the high byte reached the image id"); + match state.placements[0].position { + PlacementPosition::Placeholder(r) => { + assert_eq!((r.row, r.col), (0, 0)); + assert_eq!( + (r.rows, r.cols), + (1, 3), + "all three cells belong to the same placement" + ); + } + other => panic!("expected a placeholder position, got {other:?}"), + } +} + +/// Raising the limit is the one way graphics get turned on, so it has to turn +/// them on completely: the cell metrics the terminal already had, and a PNG +/// decoder. The old shape rebuilt the options from the defaults, which +/// silently replaced a declared cell size with 8x16. +#[test] +fn raising_the_storage_limit_keeps_the_cell_size_and_decodes_png() { + let mut a = TerminalActor::new(8, 20, 0); + assert!(a.enable_graphics(GraphicsOptions { + cell: CellSize { + width: 10, + height: 21 + }, + ..GraphicsOptions::DEFAULT + })); + a.set_graphics_storage_limit(0); + assert!(!a.graphics_state(0).enabled); + + a.set_graphics_storage_limit(4 * 1024 * 1024); + let state = a.graphics_state(0); + assert!(state.enabled); + assert!(state.cell_declared); + assert_eq!( + state.cell, + CellSize { + width: 10, + height: 21 + }, + "the declared cell size survived" + ); + + // And PNG still decodes, which it would not with no decoder installed. + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert!(a.image_bytes(IMAGE_ID).is_some()); +} + +/// An actor that never enabled graphics and then has its limit raised must +/// end up in the same complete state, not one that stores images it cannot +/// decode. +#[test] +fn the_storage_limit_alone_turns_graphics_fully_on() { + let mut a = TerminalActor::new(8, 20, 0); + a.set_graphics_storage_limit(4 * 1024 * 1024); + assert!(a.graphics_state(0).enabled); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let bytes = a.image_bytes(IMAGE_ID).expect("a PNG transmission decodes"); + assert_eq!((bytes.desc.width, bytes.desc.height), (16, 8)); +} + +/// The normal screen's images have to land in the client's normal screen. +/// Kitty storage is per screen and the replay's alt-screen prefix is at byte +/// 0, so the normal half has to say where it belongs. +#[test] +fn a_replay_from_the_alt_screen_puts_the_normal_screens_images_on_the_normal_screen() { + let mut source = actor(6, 20, 100); + source.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(source.graphics_state(0).placements.len(), 1); + source.write(b"\x1b[?1049h"); + source.write(b"full-screen program"); + assert!(source.graphics_state(0).placements.is_empty()); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.starts_with("\x1b[?1049h"), + "the prefix position is the contract" + ); + + let mut late = actor(6, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + assert!( + late.graphics_state(0).placements.is_empty(), + "the alternate screen has no images, as in the source" + ); + assert_eq!( + late.plain(Range::Viewport), + source.plain(Range::Viewport), + "the alternate screen replays as it was" + ); + + // The program exits: the normal screen comes back, and so do its images. + late.write(b"\x1b[?1049l"); + let state = late.graphics_state(0); + assert_eq!( + state.placements.len(), + 1, + "the normal screen's image survived the replay" + ); + assert_eq!(state.placements[0].placement_id, PLACEMENT_ID); + assert!(late.image_bytes(IMAGE_ID).is_some()); +} diff --git a/crates/pty-terminal/tests/handle.rs b/crates/pty-terminal/tests/handle.rs index ddf2b57..d050224 100644 --- a/crates/pty-terminal/tests/handle.rs +++ b/crates/pty-terminal/tests/handle.rs @@ -279,6 +279,234 @@ fn attach_identity_reconnect_reaches_the_replacement() { rig.kill("a"); } +/// The case a Node daemon cannot serve: the child draws a kitty image, and a +/// client attaches only afterwards. The image was in `DATA` that this client +/// never saw, so everything it knows comes from the daemon's `SCREEN` — which +/// carries the image because the session's own terminal holds it +/// (docs/decisions/0012-kitty-graphics-replay.md). +/// +/// The bytes are OMP's (`packages/tui/src/terminal-capabilities.ts` +/// `encodeKittyTransmit`, `packages/tui/src/kitty-graphics.ts` +/// `encodeKittyVirtualPlacement` / `encodeKittyPlaceholderGrid`): a `f=100` +/// PNG transmission, a virtual placement, and a placeholder cell carrying the +/// image id in its foreground colour and the placement id in its underline +/// colour. +#[test] +fn a_late_attach_gets_the_image_the_child_drew_before_it_connected() { + let Some(rig) = Rig::new() else { + eprintln!("skipping: no pty binary"); + return; + }; + // A 1x1 red PNG (the kitty protocol's own example image), image id 4242, + // placement id 7, one placeholder cell at image row 0, column 0. + let png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; + let script = format!( + "printf 'drawn\\n'; \ + printf '\\033_Ga=t,f=100,q=2,i=4242;{png}\\033\\\\'; \ + printf '\\033_Ga=p,U=1,q=2,i=4242,p=7,c=1,r=1\\033\\\\'; \ + printf '\\033[38;2;0;16;146m\\033[58:2::0:0:7m\u{10eeee}\u{305}\u{305}\\033[39;59m'; \ + exec cat" + ); + rig.run("g", &script); + // Let the child finish drawing before anyone attaches: this client must + // learn the image from the replay, not from live DATA. + std::thread::sleep(Duration::from_millis(400)); + + let h = TerminalHandle::attach( + rig.session("g"), + AttachOptions { + graphics: Some(pty_terminal::GraphicsOptions::DEFAULT), + ..Default::default() + }, + ) + .expect("attach"); + assert!(h.wait_ready(Duration::from_secs(5)), "first SCREEN"); + assert!(h.plain(Range::Full).contains("drawn")); + + let deadline = Instant::now() + Duration::from_secs(5); + let state = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!( + Instant::now() < deadline, + "the replay carried no placement; screen:\n{}", + h.plain(Range::Full) + ); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + + let image = state.image(4242).expect("the image came with the replay"); + assert_eq!((image.width, image.height), (1, 1)); + assert_eq!( + h.image_bytes(4242).map(|b| b.data), + Some(vec![255, 0, 0, 255]), + "the pixels themselves, decoded from the PNG" + ); + let p = &state.placements[0]; + assert_eq!((p.image_id, p.placement_id), (4242, 7)); + assert!(p.is_virtual); + assert!( + matches!(p.position, pty_terminal::PlacementPosition::Placeholder(_)), + "located by its placeholder cell, at {:?}", + p.position + ); + + // A reconnect resets the terminal and replays a fresh SCREEN. The image + // has to come back with it: the reset must not take the storage away, or + // the replay's own transmission would be rejected. + let position = p.position; + h.reconnect().expect("reconnect"); + assert!(h.wait_ready(Duration::from_secs(5)), "SCREEN after reconnect"); + let deadline = Instant::now() + Duration::from_secs(5); + let after = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!(Instant::now() < deadline, "reconnect lost the image"); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + assert_eq!( + h.image_bytes(4242).map(|b| b.data), + Some(vec![255, 0, 0, 255]) + ); + assert_eq!(after.placements[0].placement_id, 7); + assert_eq!(after.placements[0].position, position, "same cell"); + + h.kill(); + rig.kill("g"); +} + +/// A daemon-side detach is a state a consumer has to be able to enter, so it +/// is an event, not something to poll `connected()` for. A reconnect is the +/// symmetric one: `Connected` means the new socket is up, and the attempt's +/// first SCREEN follows. +#[test] +fn a_lost_socket_and_a_reconnect_are_both_announced() { + let Some(rig) = Rig::new() else { + eprintln!("skipping: no pty binary"); + return; + }; + rig.run("d", "printf 'first\\n'; exec sleep 60"); + let h = TerminalHandle::attach(rig.session("d"), AttachOptions::default()).expect("attach"); + assert!(h.wait_ready(Duration::from_secs(5))); + let events = h.subscribe(); + + rig.kill("d"); + let deadline = Instant::now() + Duration::from_secs(5); + while h.connected() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + assert!(!h.connected()); + let mut saw_disconnected = false; + while let Ok(ev) = events.try_recv() { + if ev == HandleEvent::Disconnected { + saw_disconnected = true; + } + } + assert!(saw_disconnected, "the lost socket was announced"); + + rig.run("d", "printf 'second\\n'; exec sleep 60"); + h.reconnect().expect("reconnect"); + assert!(h.wait_ready(Duration::from_secs(5))); + let mut saw_connected = false; + while let Ok(ev) = events.try_recv() { + if ev == HandleEvent::Connected(h.attempt()) { + saw_connected = true; + } + } + assert!(saw_connected, "the new socket was announced"); + h.kill(); + rig.kill("d"); +} + +/// Cell metrics live on the client's host — its font — but the session's +/// terminal is what answers geometry for a placement that left `c=`/`r=` +/// implicit, including in the replay every other client gets. So the client +/// declares them on ATTACH and RESIZE, and the daemon adopts them. +#[test] +fn a_client_declares_its_cell_size_and_the_session_geometry_follows() { + let Some(rig) = Rig::new() else { + eprintln!("skipping: no pty binary"); + return; + }; + // A 16x16 RGBA image (f=32, 1024 pixel bytes) placed with no c=/r=, so + // its cell extent is purely derived from the cell size. + let px = format!("{}==", "A".repeat(1366)); + let script = format!( + "printf 'drawn\\n'; \ + printf '\\033_Ga=t,q=2,i=77,f=32,s=16,v=16;{px}\\033\\\\'; \ + printf '\\033_Ga=p,q=2,i=77,p=1\\033\\\\'; \ + exec cat" + ); + rig.run("c", &script); + std::thread::sleep(Duration::from_millis(400)); + + let graphics = pty_terminal::GraphicsOptions { + cell: pty_terminal::CellSize { + width: 16, + height: 16, + }, + ..pty_terminal::GraphicsOptions::DEFAULT + }; + let h = TerminalHandle::attach( + rig.session("c"), + AttachOptions { + graphics: Some(graphics), + ..Default::default() + }, + ) + .expect("attach"); + assert!(h.wait_ready(Duration::from_secs(5))); + + // 16x16 pixels at a declared 16x16 cell is 1x1 cells, on both sides: the + // client's own terminal and the daemon's, which was told on ATTACH. + let deadline = Instant::now() + Duration::from_secs(5); + let state = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!( + Instant::now() < deadline, + "no placement; screen:\n{}", + h.plain(Range::Full) + ); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + assert!(state.cell_declared, "the client declared its cell size"); + assert_eq!(state.placements[0].cell_size, (1, 1)); + assert_eq!(state.placements[0].requested_cells, (0, 0)); + + // The daemon's own answer, via a PEEK-based read-only attach that never + // declares anything: the replay it gets carries the image, and the + // session's terminal is what resolved the geometry. + let (code, out, err) = rig.pty(&["peek", "c"]); + assert_eq!(code, 0, "peek failed: {out}{err}"); + assert!( + out.contains("i=77"), + "the session's replay carries the image: {out:?}" + ); + + // A later declaration travels on RESIZE and moves the derived extent. + h.set_cell_size(8, 16); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let state = h.graphics(0); + if state.placements.first().map(|p| p.cell_size) == Some((2, 1)) { + assert_eq!(state.cell, pty_terminal::CellSize { width: 8, height: 16 }); + break; + } + assert!(Instant::now() < deadline, "the new cell size never took"); + h.wait_rev(h.rev(), Duration::from_millis(100)); + } + + h.kill(); + rig.kill("c"); +} + #[test] fn attach_to_missing_session_is_an_error() { let root = std::env::temp_dir(); diff --git a/crates/pty-terminal/tests/input.rs b/crates/pty-terminal/tests/input.rs new file mode 100644 index 0000000..b74aa61 --- /dev/null +++ b/crates/pty-terminal/tests/input.rs @@ -0,0 +1,242 @@ +//! The child input encoder: the bytes a key, a mouse event, a focus change, +//! or a paste turns into depend on modes the *child* set, so every case here +//! sets that mode the way a child would and then encodes. + +use std::time::Duration; + +use pty_terminal::input::{self, Key, KeyAction, KeyEvent, Mods, MouseAction, MouseButton, MouseEvent}; +use pty_terminal::{SpawnOptions, TerminalActor, TerminalHandle}; + +fn actor() -> TerminalActor { + TerminalActor::new(10, 20, 0) +} + +// ── keys ── + +#[test] +fn decckm_decides_whether_an_arrow_is_csi_or_ss3() { + let mut a = actor(); + assert!(!a.modes().app_cursor); + assert_eq!(a.encode_key(&KeyEvent::press(Key::ArrowUp)), b"\x1b[A"); + + a.write(b"\x1b[?1h"); + assert!(a.modes().app_cursor, "?1 is tracked"); + assert_eq!( + a.encode_key(&KeyEvent::press(Key::ArrowUp)), + b"\x1bOA", + "application cursor keys" + ); + + a.write(b"\x1b[?1l"); + assert!(!a.modes().app_cursor); + assert_eq!(a.encode_key(&KeyEvent::press(Key::ArrowUp)), b"\x1b[A"); +} + +#[test] +fn a_plain_character_is_itself() { + let a = actor(); + assert_eq!(a.encode_key(&KeyEvent::typed(Key::A, "a", Some('a'))), b"a"); + assert_eq!( + a.encode_key(&KeyEvent::press(Key::C).with_mods(Mods::CTRL)), + b"\x03", + "ctrl+c is a control byte until the child asks for more" + ); +} + +/// The kitty keyboard protocol's associated text: the child asked for it +/// (`CSI > 17 u` = disambiguate + report associated), so the shifted text has +/// to reach it alongside the base key. A `KeyEvent` that folded shift into one +/// character could not express this. +#[test] +fn kitty_associated_text_carries_the_shifted_character() { + let mut a = actor(); + a.write(b"\x1b[>17u"); + assert_eq!(a.kitty_flags(), 17); + assert_eq!(a.modes().kitty_stack, vec![17]); + + let shift_a = KeyEvent::typed(Key::A, "A", Some('a')).with_mods(Mods::SHIFT); + assert_eq!( + a.encode_key(&shift_a), + b"\x1b[97;2;65u", + "base key 97 ('a'), shift modifier, associated text 65 ('A')" + ); + assert_eq!( + a.encode_key(&KeyEvent::typed(Key::A, "a", Some('a'))), + b"a", + "an unmodified key still goes through as text" + ); +} + +/// The alternate-key form (`CSI > 5 u` = disambiguate + report alternates): +/// the shifted key travels in the key field itself, `base:shifted`. +#[test] +fn kitty_alternates_carry_the_shifted_key() { + let mut a = actor(); + a.write(b"\x1b[>5u"); + let shift_a = KeyEvent::typed(Key::A, "A", Some('a')).with_mods(Mods::SHIFT); + let out = String::from_utf8(a.encode_key(&shift_a)).expect("utf8"); + assert!( + out.contains("97:65"), + "expected base:shifted in {out:?} (flags {})", + a.kitty_flags() + ); +} + +#[test] +fn a_release_reaches_the_child_only_when_it_asked_for_events() { + let mut a = actor(); + let release = KeyEvent { + action: KeyAction::Release, + ..KeyEvent::typed(Key::A, "a", Some('a')) + }; + assert!( + a.encode_key(&release).is_empty(), + "no release reporting by default" + ); + + // `CSI > 3 u` = disambiguate + report events. + a.write(b"\x1b[>3u"); + assert_eq!( + a.encode_key(&release), + b"\x1b[97;1:3u", + "key 97, no modifiers, event type 3 = release" + ); + + // A character key with neither text nor an unshifted codepoint has no + // identity the protocol can report, so it encodes to nothing. A + // consumer that sees an empty encoding for a character key is missing + // `KeyEvent::unshifted`. + let anonymous = KeyEvent { + action: KeyAction::Release, + ..KeyEvent::press(Key::A) + }; + assert!(a.encode_key(&anonymous).is_empty()); +} + +// ── mouse ── + +fn press() -> MouseEvent { + MouseEvent::press(MouseButton::Left, 3, 4) +} + +#[test] +fn no_tracking_reports_nothing_so_the_surface_keeps_the_event() { + let a = actor(); + assert_eq!(a.encode_mouse(&press()), None); + assert_eq!(a.encode_mouse(&MouseEvent::wheel(true, 3, 4)), None); + assert!(!a.modes().mouse_reporting()); +} + +/// X10 (`?9`) reports button presses and nothing else. A consumer that folded +/// it into "mouse tracking is on" would forward the wheel to a child that +/// never hears it — and lose its own scrolling. +#[test] +fn x10_reports_presses_but_never_the_wheel() { + let mut a = actor(); + a.write(b"\x1b[?9h"); + assert!(a.modes().mouse_9); + assert!(!a.modes().mouse_tracking(), "?9 is not wheel-reporting"); + assert!(a.modes().mouse_reporting()); + + assert_eq!(a.encode_mouse(&press()), Some(b"\x1b[M \x24\x25".to_vec())); + assert_eq!( + a.encode_mouse(&MouseEvent::wheel(true, 3, 4)), + None, + "the surface keeps the wheel" + ); + assert_eq!( + a.encode_mouse(&MouseEvent { + action: MouseAction::Release, + ..press() + }), + None, + "X10 has no release report" + ); +} + +#[test] +fn normal_tracking_reports_the_wheel() { + let mut a = actor(); + a.write(b"\x1b[?1000h"); + assert!(a.modes().mouse_tracking()); + assert!(a.encode_mouse(&press()).is_some()); + assert_eq!( + a.encode_mouse(&MouseEvent::wheel(true, 3, 4)), + Some(b"\x1b[M`\x24\x25".to_vec()), + "button 64 = wheel up" + ); +} + +#[test] +fn sgr_mode_reports_the_cell_the_event_names() { + let mut a = actor(); + a.write(b"\x1b[?1000h\x1b[?1006h"); + assert_eq!( + a.encode_mouse(&press()), + Some(b"\x1b[<0;4;5M".to_vec()), + "col 3 / row 4, 1-based in the report" + ); +} + +// ── focus and paste ── + +#[test] +fn focus_is_reported_only_under_1004() { + let mut a = actor(); + assert_eq!(a.encode_focus(true), None); + a.write(b"\x1b[?1004h"); + assert_eq!(a.encode_focus(true), Some(b"\x1b[I".to_vec())); + assert_eq!(a.encode_focus(false), Some(b"\x1b[O".to_vec())); + a.write(b"\x1b[?1004l"); + assert_eq!(a.encode_focus(false), None); +} + +#[test] +fn paste_is_bracketed_only_under_2004() { + let mut a = actor(); + assert_eq!( + a.encode_paste("a\nb"), + b"a\rb", + "without bracketing a newline becomes a carriage return" + ); + a.write(b"\x1b[?2004h"); + assert_eq!(a.encode_paste("a\nb"), b"\x1b[200~a\nb\x1b[201~"); +} + +#[test] +fn a_multi_line_paste_is_flagged_unsafe() { + assert!(input::paste_is_safe("just text")); + assert!(!input::paste_is_safe("rm -rf /\n")); + assert!( + !input::paste_is_safe("a\x1b[201~b"), + "a forged bracketed-paste end escapes the brackets" + ); +} + +// ── through the handle ── + +/// The handle path: the events are `Send`, the encoding happens on the actor +/// thread against the live terminal, and `send_*` is ordered with `write`. +#[test] +fn send_key_reaches_the_child_and_encode_key_agrees() { + let h = TerminalHandle::spawn("cat", &[], SpawnOptions::default()).expect("spawn"); + assert!(h.wait_ready(Duration::from_secs(2))); + + assert_eq!(h.encode_key(&KeyEvent::press(Key::ArrowUp)), b"\x1b[A"); + assert_eq!(h.encode_mouse(&press()), None, "no tracking, no report"); + + // `cat` echoes: what the child received comes back on the screen. + h.send_key(&KeyEvent::typed(Key::A, "a", Some('a'))); + h.send_key(&KeyEvent::press(Key::Enter)); + let grid = h + .wait_for(Duration::from_secs(5), |g| g.text().starts_with('a')) + .expect("the child got the key"); + assert!(grid.text().starts_with('a')); + + h.send_paste("pasted"); + let grid = h + .wait_for(Duration::from_secs(5), |g| g.text().contains("pasted")) + .expect("the child got the paste"); + assert!(grid.text().contains("pasted")); + h.kill(); +} diff --git a/crates/pty-terminal/tests/replay.rs b/crates/pty-terminal/tests/replay.rs index 09da72a..95da551 100644 --- a/crates/pty-terminal/tests/replay.rs +++ b/crates/pty-terminal/tests/replay.rs @@ -66,7 +66,7 @@ fn peek_never_prefixes_1049h() { for opts in [SerializeOpts::PEEK, SerializeOpts::PEEK_FULL] { let screen = a.serialize(opts); let normal = a.normal_replay().unwrap_or(""); - let body = pty_terminal::serialize::vt(a.terminal(), opts.scrollback); + let body = pty_terminal::serialize::vt(a.terminal(), opts.scrollback, a.cell_size()); // The normal screen comes first and the alternate one after it, but // no mode prefix is added on top. assert_eq!(screen, format!("{normal}{body}"), "a prefix was added"); diff --git a/crates/pty/src/daemon/clients.rs b/crates/pty/src/daemon/clients.rs index 5c5829f..6b4beed 100644 --- a/crates/pty/src/daemon/clients.rs +++ b/crates/pty/src/daemon/clients.rs @@ -15,7 +15,7 @@ use std::sync::mpsc::Sender; use std::time::{Duration, Instant}; use pty_core::protocol::{ - Packet, MessageType, decode_peek, decode_size, encode_exit, encode_geometry, encode_screen, + Packet, MessageType, decode_cell, decode_peek, decode_size, encode_exit, encode_geometry, encode_screen, encode_status_response, }; use pty_core::registry::{self, MutateOptions}; @@ -128,6 +128,7 @@ impl Daemon { return; } let (rows, cols) = decode_size(payload); + self.adopt_cell_size(payload); // Read before negotiation: a smaller client shrinks the session to // its own size, which would then look like it had matched. let size_matched = rows == self.actor.rows() && cols == self.actor.cols(); @@ -220,9 +221,33 @@ impl Daemon { c.cols = cols; self.attach_counter += 1; c.attach_seq = self.attach_counter; + self.adopt_cell_size(payload); self.negotiate_size(); } + /// Take the cell pixel metrics a client declared on ATTACH or RESIZE. + /// + /// Only a client knows how big a cell is — it comes from a font on the + /// client's host, which this process may never see — and the session's + /// terminal needs them to answer the cell extent of a kitty placement + /// that did not name `c=`/`r=` itself. A payload without them (every + /// older client, the Node one included) changes nothing, and the terminal + /// keeps its deterministic fallback. + /// + /// The most recent declaration wins. Clients that draw cells of different + /// sizes cannot all be right about an implicit placement, and unlike rows + /// and cols there is nothing to negotiate: the metrics change no bytes and + /// no client's screen, only what this session reports as derived + /// geometry. + fn adopt_cell_size(&mut self, payload: &[u8]) { + if let Some((width, height)) = decode_cell(payload) { + self.actor.set_cell_size(pty_terminal::CellSize { + width: width as u32, + height: height as u32, + }); + } + } + /// node: src/server.ts:1040-1043 /// A client asked to leave: close its socket and take it off the books /// at once. diff --git a/crates/pty/src/daemon/lifecycle.rs b/crates/pty/src/daemon/lifecycle.rs index bb219a7..d1c3977 100644 --- a/crates/pty/src/daemon/lifecycle.rs +++ b/crates/pty/src/daemon/lifecycle.rs @@ -305,7 +305,7 @@ pub fn run(cfg: DaemonConfig) -> Result { name, generation, cfg, - actor: TerminalActor::new(rows, cols, pty_terminal::actor::DEFAULT_SCROLLBACK), + actor: terminal_actor(rows, cols), master: pair.master, writer, child_pid, @@ -331,6 +331,26 @@ pub fn run(cfg: DaemonConfig) -> Result { Ok(daemon.serve()) } +/// The session's terminal, with kitty graphics on. +/// +/// A session is the durable owner of the child's screen, and since libghostty +/// keeps image state per screen, that includes the child's images: without it +/// the `SCREEN` a late client replays would carry placeholder cells naming +/// images nobody has (docs/decisions/0012-kitty-graphics-replay.md). The +/// storage limit is a cap, not an allocation — a session whose child never +/// transmits an image holds nothing and serializes exactly as before. +fn terminal_actor(rows: u16, cols: u16) -> TerminalActor { + let mut actor = TerminalActor::new(rows, cols, pty_terminal::actor::DEFAULT_SCROLLBACK); + if !actor.enable_graphics(pty_terminal::GraphicsOptions::DEFAULT) { + // The session still runs: text is unaffected and the actor rolled the + // storage limit back, so the only loss is images. Say so rather than + // leaving a client to wonder why its replay carries placeholder cells + // and no pictures. + daemon_warn!("pty: kitty graphics unavailable for this session"); + } + actor +} + fn spawn_pty_reader(mut reader: Box, tx: Sender) { std::thread::spawn(move || { let mut buf = [0u8; 16384]; diff --git a/docs/decisions/0012-kitty-graphics-replay.md b/docs/decisions/0012-kitty-graphics-replay.md new file mode 100644 index 0000000..3cb88e5 --- /dev/null +++ b/docs/decisions/0012-kitty-graphics-replay.md @@ -0,0 +1,210 @@ +# 0012 — kitty graphics are terminal state, and a replay carries them + +**Status:** accepted + +**Node behavior.** The Node daemon has no image state. `xterm-headless` has no +kitty graphics handler, so a `ESC _G ... ESC \` transmission is parsed and +discarded; the placeholder cells of a virtual placement survive only because +they are ordinary text with a foreground colour. A client that was connected +when the child wrote the image saw the raw bytes in `DATA` and its own terminal +kept them; a client that attaches afterwards gets a `SCREEN` +(`src/server.ts:962`, `@xterm/addon-serialize`) with the placeholder cells and +no image and no placement. Nothing in the Node daemon can answer "which images +does this session hold, and where are they". + +**Rust behavior.** Graphics are terminal state, off by default and bounded when +on: + +- `TerminalActor::enable_graphics` (`SpawnOptions::graphics` / + `AttachOptions::graphics`) sets a byte limit on libghostty's image storage + and installs a PNG decoder for `f=100`. Without it the storage limit stays + zero, which is libghostty's own "protocol disabled": a child cannot make a + terminal hold images its owner never asked for. The file, temporary-file, + and shared-memory transmission media stay disabled, so a child cannot name a + path the owner did not authorize; inline transmission only. +- One number bounds the state and the wire: `graphics::MAX_STORAGE_BYTES` + (32 MiB), which `enable_graphics` and `set_graphics_storage_limit` clamp to + and which a replay carries in full. A smaller replay cap would have made a + supported state — an image the terminal accepted and reports — one that a + late client can never be given, which is the exact failure this record is + about. A caller asking for more gets the bound and can see it in + `graphics_options()`. +- Cell pixel metrics come from whoever draws the cells and travel on the wire. + A placement that named neither `c=` nor `r=` gets its cell extent from the + image's pixel size divided by the cell size, so that size has to be the + client's real one: it comes from a font on the client's host, which a + session daemon may never see. `AttachOptions::graphics`'s cell size is + appended to ATTACH, `TerminalHandle::set_cell_size` sends it on RESIZE + (`encode_attach_with_cell` / `encode_resize_with_cell`: four optional bytes + after the existing rows/cols, so every older reader — the Node daemon + included — takes the size it always took and ignores the rest), and the + daemon adopts it (`clients.rs`, `adopt_cell_size`). Undeclared is explicit, + not silent: geometry uses `CellSize::FALLBACK` (8x16) and + `GraphicsState::cell_declared` is false. +- `TerminalActor::graphics_state(scroll_offset)` / + `TerminalHandle::graphics(scroll_offset)` answer with owned values + (`GraphicsState`, `ImageDesc`, `Placement`, `SourceRect`, + `PlacementPosition`) for the same window `snapshot(scroll_offset)` reads, so + a grid and a graphics state taken with the same offset line up cell for cell. + `image_bytes(id)` copies the pixels once, on request, keyed by + `ImageDesc::generation`. +- `serialize::vt` appends a graphics block after the cursor move: one + `a=t` transmission per image (chunked at 4096 base64 bytes, `s=`/`v=` for raw + formats, `o=z` for a compressed one), then `a=p,U=1` per virtual placement + and `DECSC` + `CUP` + `a=p` + `DECRC` per cursor-positioned one. The source + rectangle is always emitted in full (`x=`,`y=`,`w=`,`h=`): `w=`/`h=` default + to "the whole image", so omitting them replays a cropped placement as the + wrong pixels at the right size. A virtual placement is emitted wherever its + cells are, history included, because its command carries no position at + all; a cursor-positioned one needs a cell in the active area, and one whose + top has scrolled above it is anchored at row 0 with its crop advanced by + the rows that are gone. It is empty + for a terminal with no graphics, so a session that never sent an image + serializes exactly as it did before + (`crates/pty-terminal/tests/graphics.rs::a_replay_without_graphics_is_unchanged`). +- The daemon's own terminal has graphics on + (`crates/pty/src/daemon/lifecycle.rs`, `terminal_actor`). The session is the + durable owner of the child's screen, so it is the durable owner of the + child's images; a daemon without them would serve a `SCREEN` whose + placeholder cells name images no client can have. The limit is a cap, not an + allocation: a session whose child never transmits holds nothing. + +**Why.** Two other shapes were available and are worse. Passing the child's +graphics bytes through to an outer terminal cannot work for an embedder that +draws a sub-rectangle: the child's coordinates are its own, and the embedder +clips, pans, and draws chrome around it. Keeping the state in each client +cannot work either: a client that attaches later never sees the `DATA` that +carried the image, so the state has to be reconstructible from the replay the +daemon already sends. Re-emitting the storage in the protocol the child used is +the smallest thing that makes a late client and a live client hold the same +images. + +Two deviations from libghostty's own API are deliberate: + +- Positions are resolved against the window that was asked for, not the live + viewport. A cursor-positioned placement's own rectangle + (`PlacementIteration::rect` + `point_from_grid_ref(PointSpace::Screen)`) + answers for any window and reports a negative row for a placement whose top + has scrolled above it; `viewport_pos` answers only for the live viewport and + reports nothing at all for a placement above it, which would make a + scrolled-back reader lose exactly the images it wants. +- libghostty reports no viewport position for a virtual placement — correctly, + because a virtual placement has none: it is wherever its placeholder cells + are. Those cells each name their own image row and column, so + `graphics::scan_placeholders` decodes them from the grid (one pass over the + window, never the scrollback) and reports both the visible cell box and the + image cell indices it covers. That is what makes a partially scrolled image + answerable: a two-row image scrolled by one row reports one visible row, + `cell_row = 1`, and `origin_row = -1`. +- `TerminalActor::reset` re-enables graphics after `RIS`. libghostty's reset + restores its defaults, which include no storage and no cell metrics; since + `reset` is what precedes a `SCREEN` replay, a terminal that lost its storage + there would reject the very images the replay carries. + +**Client effect.** A consumer that opts in can ask, at any time and from any +client, for the image bytes, the placement identity (`image_id` + +`placement_id`), the resolved source crop, the rendered pixel and cell size, +and the placement's position in a chosen window; `HandleEvent::Graphics(gen)` +says when the storage content changed, and `graphics_generation()` keys a +texture cache. Scrolling and resizing move placements without changing the +generation, so a dirty frame still re-reads positions. The alternate screen has +its own storage, as in the protocol: a full-screen program's images are not the +primary screen's, and neither set is lost when the child switches. + +Residual differences a consumer can observe: + +1. `GraphicsState::images` lists the images that have at least one placement. + libghostty exposes lookup by id, not enumeration, so an image transmitted + and never placed is not listed (it is also not drawable). +2. Grayscale images (`Gray`, `GrayAlpha`) have no kitty `f=` value and are not + re-emitted into a replay. Every PNG this terminal accepts is expanded to + 8-bit RGBA before libghostty stores it (a grayscale PNG decodes as + `Grayscale`/`GrayscaleAlpha`, which `PngDecoder` widens itself — rejecting + it instead would drop every monochrome plot a child sends), so the + grayscale variants are unreachable in practice and kept as a typed dead end + rather than a silent conversion. +3. Cell metrics are the newest declaration, not a negotiation. Two clients + whose fonts differ cannot both be right about a placement that left its + size implicit, and unlike rows and cols there is nothing to reconcile: the + metrics change no bytes and no client's screen, only the derived geometry + this session reports. A client that draws pixels should declare its own and + read `GraphicsState::cell_declared` rather than trust the fallback. +4. A cursor-positioned placement that has scrolled *entirely* above the + active area is not re-emitted: there is no cell left to put the cursor + on. One that is partly on screen is re-emitted clipped, so what a late + client gets is what the source terminal is showing. Virtual placements + have no such limit at all, which is one more reason they are the preferred + form. +5. A replay of the alternate screen brackets the normal half in + `ESC[?1049l` / `ESC[?1049h` + `ESC[H`. Node's payload writes its normal + half after `ESC[?1049h`, so a Node client's normal buffer takes it while + already switched; that is invisible for text but would put the normal + screen's images in the client's alternate storage, since kitty storage is + per screen. +6. `SIXEL` and the iTerm2 protocol are not read at all. Neither offers a + placement contract an embedder can reproject into a sub-rectangle. + +**Test.** `crates/pty-terminal/tests/graphics.rs` — twenty-six cases driven with +the exact bytes OMP writes (`packages/tui/src/terminal-capabilities.ts` +`encodeKittyTransmit`, `packages/tui/src/kitty-graphics.ts` +`encodeKittyVirtualPlacement` / `encodeKittyPlaceholderGrid`): opt-in, live +write, source crop, scroll, scrollback window, resize, alternate screen, +child-sent delete of a placement and of an image, `clear_graphics`, zeroed +limit, the replay cases +(`a_late_client_reconstructs_the_image_from_the_replay_alone`, +`a_cursor_positioned_placement_replays_at_its_cell`), the handle path +(`a_spawned_child_that_draws_an_image_is_queryable_through_the_handle`, which +also pins `HandleEvent::Graphics`), the bound +(`an_image_far_larger_than_three_mib_still_replays` — a 4 MiB image +transmitted in 4096-byte chunks, stored and then recovered byte for byte from +the replay alone — and +`the_storage_limit_is_clamped_to_what_a_replay_can_carry`), and the cell +metrics (`an_implicit_placement_takes_its_extent_from_the_declared_cell`, +`a_declared_cell_does_not_move_an_explicit_placement`). + +Replay fidelity has a case per failure it can have, each of which fails on the +shape that preceded it: `a_cropped_placement_keeps_its_crop_through_a_replay` +(the crop survives, not the whole image measured from its origin), +`a_virtual_placement_scrolled_into_history_still_replays`, +`a_partially_scrolled_direct_placement_replays_clipped`, +`a_grayscale_png_is_stored_as_rgba`, +`a_palette_foreground_names_a_placeholder_image`, +`a_bare_continuation_cell_inherits_row_column_and_id_high_byte`, +`raising_the_storage_limit_keeps_the_cell_size_and_decodes_png`, +`the_storage_limit_alone_turns_graphics_fully_on`, and +`a_replay_from_the_alt_screen_puts_the_normal_screens_images_on_the_normal_screen` +(the normal screen's image is there after the full-screen program exits, and +the alternate screen replays character for character). Unit tests for the +base64 encoder and the diacritic table live in `graphics.rs`; the wire form is +pinned in `crates/pty-core/tests/protocol.rs` +(`attach_can_declare_a_cell_size_without_changing_the_size_it_carries`, +`resize_can_declare_a_cell_size`, `a_plain_size_payload_declares_no_cell`). + +End to end, against a real session daemon: +`crates/pty-terminal/tests/handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` +— a child transmits a PNG, places it virtually, and writes a placeholder cell; +a `TerminalHandle` attaches only afterwards, so it never sees that `DATA`, and +still reads the decoded pixels (`[255, 0, 0, 255]`), the placement identity +(image 4242, placement 7), and its cell from the `SCREEN` replay alone. The +same test then reconnects and asserts the image and its cell come back, which +is what the graphics-preserving `reset` is for. Its sibling +`a_client_declares_its_cell_size_and_the_session_geometry_follows` attaches +with a declared 16x16 cell to a session whose child placed a 16x16 image with +no `c=`/`r=`, asserts the extent is 1x1 cell, then declares 8x16 and asserts +it becomes 2x1. + +No gated `_node` / `_rust` conformance pair exists, and none can: the Node side +has no graphics state to compare against. The difference is not "Rust renders +this differently" but "Rust has state Node does not", so the record stands on +the Rust tests plus the Node source above. + +**Migration / negotiation.** None. A client's graphics are off unless it asks +(`SpawnOptions::graphics` / `AttachOptions::graphics`); the daemon's are on, +and a session whose child never transmits an image serves the same `SCREEN` +bytes it did before. A client that does not opt in is unaffected by a session +that holds images: the extra `ESC _G ... ESC \` in its replay is an APC string, +which a terminal that does not implement the protocol ignores. A Node daemon +feeding a Rust client still works — the client then has only what the Node +`SCREEN` carries (placeholder cells, no images), which is exactly what a Node +client would have. Full replay fidelity needs the daemon side to be the Rust +one. From 180c09bc9d8588eaa2053b5fc93a3e5c694671db Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:16:40 +0200 Subject: [PATCH 2/3] fix(terminal): make the 10 000-line scrollback promise true MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libghostty's `Options::max_scrollback` is documented as a line count and is a byte budget for the history page list. Passing 10 000 for "10 000 lines" bought one page: 745 rows at 80 columns, 456 at 200, 3 310 at 20, and doubling the number changed nothing because both values are smaller than a page. A session that promised a 10 000-line replay window delivered 7% of it, and `scrollback_capacity()` reported a number that described nothing. The line count is now converted to a byte budget that scales with the width (256 + 16 bytes per column per line, against a measured ~838 at 80 columns and ~1 804 at 200), capped at 64 MiB. `scrollback()` reports what is retainable, `scrollback_request()` what was asked for, and `scrollback_bytes()` the budget libghostty holds it in — so the reported numbers describe the terminal rather than the request. Two consequences are recorded rather than hidden: capacity is a guaranteed minimum instead of Node's ceiling, because libghostty never holds less than one page; and widening a terminal lowers the line count it can retain, because libghostty takes the budget in `Options` and exposes no setter. docs/decisions/0013-scrollback-is-a-line-promise.md has the measurements, the memory cost, and the remaining gap. Refs #3 agent-identity: dev3.direct.omp.2gz9tcpa agent-persona: generalist agent-supervisor: unavailable agent-tool: OMP agent-tool-version: 18.1.2 agent-runtime: OMP 18.1.2 tooling-profile: dotfiles@7534055 --- crates/pty-terminal/src/actor.rs | 91 ++++++++++- crates/pty-terminal/tests/scrollback.rs | 147 ++++++++++++++++++ .../0013-scrollback-is-a-line-promise.md | 110 +++++++++++++ 3 files changed, 341 insertions(+), 7 deletions(-) create mode 100644 crates/pty-terminal/tests/scrollback.rs create mode 100644 docs/decisions/0013-scrollback-is-a-line-promise.md diff --git a/crates/pty-terminal/src/actor.rs b/crates/pty-terminal/src/actor.rs index ee948f0..84b4c8f 100644 --- a/crates/pty-terminal/src/actor.rs +++ b/crates/pty-terminal/src/actor.rs @@ -24,9 +24,50 @@ use crate::serialize::{self, SerializeOpts}; use crate::snapshot::{self, CellGrid}; use crate::strip::{OutputScanner, Osc, Token}; -/// Node's scrollback (`src/server.ts:333-338`). +/// Node's scrollback (`src/server.ts:333-338`), in lines. pub const DEFAULT_SCROLLBACK: usize = 10_000; +/// The most memory one terminal's history may be given, whatever scrollback +/// it was asked for: 64 MiB, which is 10 000 lines of a 400-column terminal. +/// +/// A session that asks for more history than this gets what fits and says so +/// ([`TerminalActor::scrollback`] reports what is actually retainable, not +/// what was requested), because a silently unmet promise is what this whole +/// conversion exists to remove. +pub const MAX_SCROLLBACK_BYTES: usize = 64 * 1024 * 1024; + +/// libghostty's `max_scrollback` is a **byte** budget for the history page +/// list, not a line count — its own doc comment says "lines", but the +/// behaviour is bytes and the page list evicts whole pages. Passing 10 000 +/// for "10 000 lines" buys one page: 745 rows at 80 columns, 456 at 200, +/// 3 310 at 20. A line promise therefore has to be converted, and the cost +/// of a row depends on how wide it is. +/// +/// These two numbers are the conversion, measured against libghostty-vt +/// 0.2.1: a plain row costs ~838 bytes at 80 columns and ~1 804 at 200, so +/// ~9-10 bytes per column plus per-row overhead. They are deliberately +/// generous — roughly 1.8x the measured cost — because the page list rounds +/// up to whole pages and because a row of styled or multi-codepoint cells +/// costs more than a plain one. Under-budgeting loses history; over-budgeting +/// costs address space that is only touched when the history actually fills. +const SCROLLBACK_BYTES_PER_COL: usize = 16; +/// Fixed per-row cost, independent of width. See +/// [`SCROLLBACK_BYTES_PER_COL`]. +const SCROLLBACK_ROW_OVERHEAD: usize = 256; + +/// What one row of a `cols`-wide terminal costs in the history. +fn scrollback_row_bytes(cols: u16) -> usize { + SCROLLBACK_ROW_OVERHEAD + SCROLLBACK_BYTES_PER_COL * cols.max(1) as usize +} + +/// The byte budget that retains `lines` lines of a `cols`-wide terminal, +/// capped at [`MAX_SCROLLBACK_BYTES`]. +fn scrollback_budget(lines: usize, cols: u16) -> usize { + lines + .saturating_mul(scrollback_row_bytes(cols)) + .min(MAX_SCROLLBACK_BYTES) +} + /// A desktop notification the child asked for (OSC 9, 99, or 777). /// Shapes follow Node (`src/server.ts:421-454`). #[derive(Debug, Clone, PartialEq, Eq)] @@ -156,7 +197,12 @@ pub struct TerminalActor { modes: Modes, events: Vec, last_title: Option, - scrollback: usize, + /// Lines of history the owner asked for. + scrollback_request: usize, + /// The byte budget libghostty was given for the history. Fixed at + /// construction: libghostty takes it in `Options` and exposes no setter, + /// so a resize changes how many lines it holds, not how much memory. + scrollback_bytes: usize, /// How big a cell is on the surface that draws this terminal, zero when /// undeclared. It comes from the client (a font on its host), travels on /// ATTACH and RESIZE, and decides the cell extent of any placement that @@ -176,12 +222,17 @@ pub struct TerminalActor { impl TerminalActor { /// A terminal of `rows` x `cols` with `scrollback` lines of history. + /// + /// The line count is converted to the byte budget libghostty actually + /// takes (see [`MAX_SCROLLBACK_BYTES`]); [`TerminalActor::scrollback`] + /// reports how many lines that buys at the current width. pub fn new(rows: u16, cols: u16, scrollback: usize) -> TerminalActor { let shared: Rc> = Rc::new(RefCell::new(Shared::default())); + let scrollback_bytes = scrollback_budget(scrollback, cols); let mut term = Terminal::new(Options { cols: cols.max(1), rows: rows.max(1), - max_scrollback: scrollback, + max_scrollback: scrollback_bytes, }) .expect("libghostty terminal"); { @@ -210,7 +261,8 @@ impl TerminalActor { modes: Modes::default(), events: Vec::new(), last_title: None, - scrollback, + scrollback_request: scrollback, + scrollback_bytes, cell: CellSize::default(), graphics: None, normal_replay: None, @@ -653,13 +705,38 @@ impl TerminalActor { } /// Node's `scrollbackCapacity`: `rows + scrollback`. + /// + /// A guaranteed minimum, not a ceiling. libghostty's history is a list of + /// pages and it never holds less than one, so a terminal asked for a + /// small scrollback retains more than it promised (a 100-line request at + /// 80 columns keeps ~1 000 rows). Node's number is a ceiling because + /// xterm counts lines; this one is a floor because libghostty counts + /// bytes and rounds to pages + /// (docs/decisions/0013-scrollback-is-a-line-promise.md). pub fn scrollback_capacity(&self) -> usize { - self.rows() as usize + self.scrollback + self.rows() as usize + self.scrollback() } - /// Configured scrollback lines. + /// How many lines of history this terminal retains at its current width. + /// + /// Normally the line count the owner asked for. It is less when the byte + /// budget cannot buy that many — either because the request exceeded + /// [`MAX_SCROLLBACK_BYTES`], or because the terminal has since been + /// widened and libghostty's budget is fixed at construction. Reporting + /// the request in that case is what made the promise a lie. pub fn scrollback(&self) -> usize { - self.scrollback + let fits = self.scrollback_bytes / scrollback_row_bytes(self.cols()); + self.scrollback_request.min(fits) + } + + /// Lines of history the owner asked for, whether or not they fit. + pub fn scrollback_request(&self) -> usize { + self.scrollback_request + } + + /// The byte budget libghostty holds the history in. + pub fn scrollback_bytes(&self) -> usize { + self.scrollback_bytes } /// Node's `baseY`: the buffer row where the active area starts. diff --git a/crates/pty-terminal/tests/scrollback.rs b/crates/pty-terminal/tests/scrollback.rs new file mode 100644 index 0000000..049d226 --- /dev/null +++ b/crates/pty-terminal/tests/scrollback.rs @@ -0,0 +1,147 @@ +//! The scrollback promise: a terminal asked for N lines of history retains N +//! lines, and says so. +//! +//! libghostty's `max_scrollback` is a byte budget for the history page list, +//! not a line count — its own doc comment says "lines" — and the page list +//! evicts whole pages. Passing 10 000 for "10 000 lines" buys one page: 745 +//! rows at 80 columns, 456 at 200, 3 310 at 20. Every case here is about the +//! conversion that makes the declared line count true, and about the reported +//! numbers being the truth rather than the request. + +use pty_terminal::actor::{DEFAULT_SCROLLBACK, MAX_SCROLLBACK_BYTES}; +use pty_terminal::{Range, TerminalActor}; + +/// Feed `n` numbered lines, each short enough that it cannot wrap, so a row +/// written is a row of history and nothing here depends on reflow. +fn fill(a: &mut TerminalActor, n: usize) { + for i in 0..n { + a.write(format!("L{i}\r\n").as_bytes()); + } +} + +/// The oldest of 10 000 lines is still reachable, at the geometry the +/// terminal was built with. This is the substrate's promised replay window; +/// before the byte conversion it retained 745 of them. +#[test] +fn ten_thousand_lines_of_history_are_all_retained() { + let mut a = TerminalActor::new(24, 80, DEFAULT_SCROLLBACK); + fill(&mut a, 10_008); + + // 10 008 written lines plus the row the cursor sits on, all within the + // 10 024-row capacity of a 24-row terminal with 10 000 lines of history. + assert!( + a.buffer_length() >= 10_008, + "expected every line retained, got {} rows", + a.buffer_length() + ); + let text = a.plain(Range::Full); + let first = text.lines().next().unwrap_or(""); + assert_eq!(first, "L0", "the oldest line is still the oldest line"); + assert!(text.contains("\nL9999\n"), "and the newest are there too"); +} + +/// The same promise at widths where the byte cost of a row differs by 5x. A +/// budget derived from the line count has to scale with the width, or a wide +/// terminal keeps a fraction of the history a narrow one does. +#[test] +fn the_promise_holds_at_every_width() { + for cols in [20u16, 80, 200, 400] { + let mut a = TerminalActor::new(24, cols, 10_000); + fill(&mut a, 10_008); + assert!( + a.buffer_length() >= 10_008, + "{cols} columns: expected 10 008 rows, got {}", + a.buffer_length() + ); + assert_eq!( + a.plain(Range::Full).lines().next().unwrap_or(""), + "L0", + "{cols} columns: the oldest line was evicted" + ); + } +} + +/// A small scrollback is a line promise too, and the promise is a floor: the +/// history is a list of pages and libghostty never holds less than one, so a +/// terminal asked for 100 lines keeps at least 100 and in practice more. +/// Bounded, though — a terminal asked for 100 lines does not keep 5 000. +#[test] +fn a_small_scrollback_keeps_at_least_what_it_promised() { + let mut a = TerminalActor::new(24, 80, 100); + fill(&mut a, 5_000); + let text = a.plain(Range::Full); + for i in 4_900..4_999 { + assert!( + text.contains(&format!("L{i}\n")), + "the promised window must be there: L{i} is missing" + ); + } + assert!(text.contains("L4999"), "including the newest line"); + assert!(!text.contains("L0\n"), "and the far past is evicted"); + assert!( + a.buffer_length() < 5_000, + "a 100-line request must not keep 5 000 rows, got {}", + a.buffer_length() + ); +} + +/// `scrollback_used` and `scrollback_capacity` are the numbers a consumer +/// budgets against, so they have to describe the terminal rather than the +/// request. +#[test] +fn used_and_capacity_are_honest() { + let mut a = TerminalActor::new(24, 80, DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback(), DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback_capacity(), 24 + DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback_used(), 24, "an empty terminal is its viewport"); + + fill(&mut a, 10_008); + assert_eq!(a.scrollback_used(), a.buffer_length()); + // Capacity is a floor, not a ceiling (see its doc comment): what it + // promises has to actually be there. + assert!( + a.scrollback_used() >= 10_008, + "capacity is not honest if the rows are not there: {}", + a.scrollback_used() + ); +} + +/// A request beyond the memory bound is reported as what it is. libghostty +/// takes the budget at construction and exposes no setter, so this cannot be +/// fixed by asking for more later — it can only be told truthfully. +#[test] +fn a_request_beyond_the_memory_bound_reports_what_fits() { + let a = TerminalActor::new(24, 80, 10_000_000); + assert_eq!(a.scrollback_request(), 10_000_000); + assert_eq!(a.scrollback_bytes(), MAX_SCROLLBACK_BYTES); + assert!( + a.scrollback() < 10_000_000, + "the request cannot be met and must not be reported as met" + ); + assert!( + a.scrollback() > DEFAULT_SCROLLBACK, + "but the bound still buys more than the default: {}", + a.scrollback() + ); + assert_eq!(a.scrollback_capacity(), 24 + a.scrollback()); +} + +/// Widening the terminal makes each row cost more out of a budget fixed at +/// construction, so the retainable line count drops. The reported number +/// follows it instead of repeating the original promise. +#[test] +fn a_widened_terminal_reports_the_history_it_can_still_hold() { + let mut a = TerminalActor::new(24, 80, DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback(), DEFAULT_SCROLLBACK); + let budget = a.scrollback_bytes(); + + a.resize(400, 24); + assert_eq!(a.scrollback_bytes(), budget, "the budget does not move"); + assert!( + a.scrollback() < DEFAULT_SCROLLBACK, + "a five-times wider row cannot hold the same line count: {}", + a.scrollback() + ); + assert_eq!(a.scrollback_capacity(), 24 + a.scrollback()); + assert_eq!(a.scrollback_request(), DEFAULT_SCROLLBACK); +} diff --git a/docs/decisions/0013-scrollback-is-a-line-promise.md b/docs/decisions/0013-scrollback-is-a-line-promise.md new file mode 100644 index 0000000..662099a --- /dev/null +++ b/docs/decisions/0013-scrollback-is-a-line-promise.md @@ -0,0 +1,110 @@ +# 0013 — scrollback is a line promise, and libghostty counts bytes + +**Status:** accepted + +**Node behavior.** Node's daemon passes `scrollback: 10000` to +`xterm-headless` (`src/server.ts:333-338`), where it is a line count: the +buffer holds 10 000 lines of history and evicts the 10 001st line. `pty stats` +reports `scrollbackUsed` (`buffer.active.length`, `src/server.ts:1128`) against +`scrollbackCapacity` (`rows + scrollback`), and that capacity is a ceiling the +buffer never exceeds. + +**Rust behavior.** `TerminalActor::new(rows, cols, scrollback)` still takes a +line count, and now converts it to the byte budget libghostty actually wants. + +libghostty's `Options::max_scrollback` is documented as "maximum number of +lines to keep in scrollback history" and is not: it is a byte budget for the +history page list, and the list evicts whole pages. Measured against +libghostty-vt 0.2.1, a terminal given `max_scrollback: 10_000` and fed 10 008 +short lines keeps: + +| columns | rows retained | oldest line surviving | +| --- | --- | --- | +| 20 | 3 310 | `L6698` | +| 80 | 745 | `L9263` | +| 200 | 456 | `L9552` | +| 400 | 149 | `L9859` | + +The retained count scales inversely with the width, which a line count cannot +do, and doubling the number to 20 000 changes nothing at 80 columns — both are +smaller than one page, and one page is the floor. Passing a line count straight +through therefore delivered 7% of the promised history at 80 columns, and the +number `scrollback_capacity()` reported (`24 + 10_000`) described nothing that +existed. + +The conversion is `SCROLLBACK_ROW_OVERHEAD + SCROLLBACK_BYTES_PER_COL * cols` +per line (256 + 16), against a measured cost of ~838 bytes per row at 80 +columns and ~1 804 at 200 — roughly 1.8x headroom, because the page list rounds +up to whole pages and because a row of styled or multi-codepoint cells costs +more than a plain one. The total is capped at `MAX_SCROLLBACK_BYTES` (64 MiB, +which is 10 000 lines of a 400-column terminal). + +Three reads describe the result instead of the request: + +- `scrollback()` — lines actually retainable at the current width. +- `scrollback_request()` — what the owner asked for, met or not. +- `scrollback_bytes()` — the budget libghostty holds the history in. + +**Why.** The alternative was to leave the number as libghostty takes it and +weaken the promise to "up to N lines", which is what the first Fractal test did +when it asserted `used <= capacity` — an assertion that passes when 6 398 of +10 008 lines have been thrown away. A replay window is a product promise: a +consumer that shows history decides what to keep on the basis of that number, +and the honest options were to meet it or to publish a smaller one. Meeting it +costs memory that is bounded, documented, and only touched when the history +actually fills; publishing a smaller one would have meant every consumer +carrying its own conversion from lines to whatever libghostty's number means +this release. + +libghostty exposes the budget only in `Options`, with no setter, so it is fixed +at construction. That is the source of the one remaining gap below. + +**Client effect.** A session asked for 10 000 lines retains 10 000 lines at the +width it was created with, and `stats` reports numbers that are true. Memory +per session with the default 10 000 lines: 14.6 MiB of budget at 80 columns +(33 MiB at 200, 63.5 MiB at 400), against ~1 MiB before — the budget is address +space the page list fills only as history accumulates, so an idle session pays +nothing. + +Residual differences a consumer can observe: + +1. `scrollback_capacity()` is a guaranteed minimum, where Node's is a ceiling. + libghostty never holds less than one page, so a terminal asked for a small + scrollback keeps more than it promised: a 100-line request at 80 columns + retains about 1 000 rows. The promise is met and then some; code that + treated the number as an upper bound on `scrollback_used` has to stop. +2. Widening a terminal reduces the line count it can retain, because the byte + budget is fixed at construction and a wider row costs more. `scrollback()` + and `scrollback_capacity()` follow the width down; `scrollback_request()` + keeps saying what was asked for. A terminal created at 80 columns and + widened to 400 holds about a fifth of the lines. Fixing this needs either a + libghostty setter for the budget (upstream) or budgeting for the widest + plausible width at construction (10 000 lines at 1 000 columns is 154 MiB + per session, which is not worth it). +3. A request whose budget exceeds `MAX_SCROLLBACK_BYTES` is clamped, and + `scrollback()` reports the clamped line count rather than the request. + +**Test.** `crates/pty-terminal/tests/scrollback.rs` — six cases, all of which +fail on the pass-through: +`ten_thousand_lines_of_history_are_all_retained` (the oldest of 10 008 lines is +still `L0`, at 24x80 with the default scrollback), +`the_promise_holds_at_every_width` (the same at 20, 80, 200 and 400 columns, +where the per-row cost differs by 5x), +`a_small_scrollback_keeps_at_least_what_it_promised` (the promised window is +present and the far past is gone), +`used_and_capacity_are_honest`, +`a_request_beyond_the_memory_bound_reports_what_fits`, and +`a_widened_terminal_reports_the_history_it_can_still_hold`. + +Every line written by these tests is short enough that it cannot wrap, so no +result here depends on reflow: a row written is a row of history. + +No gated `_node` / `_rust` conformance pair exists for the byte conversion +itself — it is an implementation detail of reaching Node's behaviour, not a +deviation from it. The observable deviations are the three above. + +**Migration / negotiation.** None. `TerminalActor::new` and +`SpawnOptions`/`AttachOptions` still take a line count and now honour it; a +consumer reading `scrollback_capacity()` as a ceiling should read it as a floor +(residual 1), and one that resizes should re-read `scrollback()` rather than +assume the original number (residual 2). From 16fe0da41e063fc252b166535cb984356a92b25f Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sat, 5 Sep 2026 09:09:06 +0200 Subject: [PATCH 3/3] docs: add terminal images VRS agent-identity: dev3.direct.omp.2gz9tcpa agent-persona: generalist agent-supervisor: unavailable agent-tool: OMP agent-tool-version: 18.1.2 agent-runtime: OMP 18.1.2 tooling-profile: dotfiles@7534055 --- README.md | 6 + .../pty-conformance/tests/output_activity.rs | 5 +- crates/pty-core/src/registry/metadata.rs | 3 +- crates/pty/src/daemon/lifecycle.rs | 3 +- docs/parity.md | 3 +- docs/vrs/01-images/requirements.md | 44 ++++ docs/vrs/01-images/spec.md | 207 ++++++++++++++++++ 7 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 docs/vrs/01-images/requirements.md create mode 100644 docs/vrs/01-images/spec.md diff --git a/README.md b/README.md index 97edc7a..ffee244 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,12 @@ Where the port stands against the Node `pty`, surface by surface, is in [docs/parity.md](docs/parity.md); the work packages that close the gap are in [docs/parity-plan.md](docs/parity-plan.md). +What must stay true of terminal images — the Kitty graphics protocol state a +session holds and replays — is in +[docs/vrs/01-images/requirements.md](docs/vrs/01-images/requirements.md); how +the session meets it is in +[docs/vrs/01-images/spec.md](docs/vrs/01-images/spec.md). + ## Install With Nix (flakes), from a checkout or straight from GitHub: diff --git a/crates/pty-conformance/tests/output_activity.rs b/crates/pty-conformance/tests/output_activity.rs index c1a469d..25c511a 100644 --- a/crates/pty-conformance/tests/output_activity.rs +++ b/crates/pty-conformance/tests/output_activity.rs @@ -1,7 +1,8 @@ //! `lastOutputAtMs`: the daemon records when the child last printed. //! -//! The Node tool grew this field in its PR #168, merged on 2026-08-29 -//! (`docs/vrs/requirements.md` R14, `docs/disk-layout.md`). The contract is: +//! The Node tool grew this field in its PR #168, merged on 2026-08-29 (the +//! Node pty repository's `docs/vrs/requirements.md` R14 and +//! `docs/disk-layout.md`, not this repository's `docs/`). The contract is: //! //! - absent until the session produces output, and absent on a record an //! older daemon wrote — never zero, and never a claim of idleness; diff --git a/crates/pty-core/src/registry/metadata.rs b/crates/pty-core/src/registry/metadata.rs index 6ed68db..8eabe79 100644 --- a/crates/pty-core/src/registry/metadata.rs +++ b/crates/pty-core/src/registry/metadata.rs @@ -89,7 +89,8 @@ pub struct SessionMetadata { /// nothing to take; it is persisted at most once a second while output /// flows. /// - /// node: src/sessions.ts (`lastOutputAtMs`), docs/vrs/requirements.md R14 + /// node: src/sessions.ts (`lastOutputAtMs`), the Node pty repository's + /// `docs/vrs/requirements.md` R14 (not this repository's `docs/vrs`) #[serde(default, skip_serializing_if = "Option::is_none")] pub last_output_at_ms: Option, /// Every field this version does not model, round-tripped verbatim. diff --git a/crates/pty/src/daemon/lifecycle.rs b/crates/pty/src/daemon/lifecycle.rs index d1c3977..e3f904c 100644 --- a/crates/pty/src/daemon/lifecycle.rs +++ b/crates/pty/src/daemon/lifecycle.rs @@ -104,7 +104,8 @@ pub(crate) struct Daemon { /// How long the activity write waits after the first chunk of a burst. /// -/// node: src/server.ts `scheduleActivityPersist` (1 s), docs/vrs R14. +/// node: src/server.ts `scheduleActivityPersist` (1 s), the Node pty +/// repository's `docs/vrs` R14 (not this repository's `docs/vrs`). const ACTIVITY_PERSIST_DEBOUNCE: Duration = Duration::from_secs(1); /// 32 hex characters, Node's `randomBytes(16).toString("hex")`. diff --git a/docs/parity.md b/docs/parity.md index cd60328..ab8c64b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -489,7 +489,8 @@ looked. | Mixed-fleet rig (section 11) | M | | Packaging and completions (section 14) | M | -Sources for this map: the Node source and its tests at `500eab2`, `docs/vrs`, +Sources for this map: the Node source and its tests at `500eab2`, and — all in +the `compoundingtech/pty` (Node) repository — its `docs/vrs`, `docs/disk-layout.md`, `docs/client.md`, `docs/testing.md`; this repository at `e4d6cda`; the `st2`, `pty-relay`, `deskset`, `ding`, `smalltalk`, and `evals` call sites; issues #1, #3, #4 here and the open issues and PRs on the Node diff --git a/docs/vrs/01-images/requirements.md b/docs/vrs/01-images/requirements.md new file mode 100644 index 0000000..af56ca8 --- /dev/null +++ b/docs/vrs/01-images/requirements.md @@ -0,0 +1,44 @@ +# Terminal images — Requirements + +## Context + +This node defines the durable terminal-image contract for pty-rust. The implementation uses the Kitty graphics protocol and libghostty. The rationale for the protocol and replay shape remains in [decision 0012](../../decisions/0012-kitty-graphics-replay.md). + +## Assumptions + +- **PTY.IMG-A01 Typed terminal ownership:** The terminal actor is the single owner of parsed image state. Clients read typed image descriptions, bytes, placements, and generations rather than untrusted escape sequences. +- **PTY.IMG-A02 Cell-relative composition:** An embedding client composes terminal images in character-cell coordinates and needs the same effective cell metrics that the terminal used. + +## Constraints + +- **PTY.IMG-C01 Library enumeration:** libghostty does not enumerate an image that was transmitted but never placed; it exposes image lookup by identifier. +- **PTY.IMG-C02 Pixel formats:** Kitty has no `f=` value for grayscale pixels, so a grayscale raw-pixel transmission cannot be represented as a typed replay image without conversion. +- **PTY.IMG-C03 Scrolled direct placements:** A cursor-positioned placement that is entirely above the active area cannot be re-emitted because restoring its position requires a cursor cell inside the active area. +- **PTY.IMG-C04 Protocol scope:** SIXEL and iTerm2 output do not provide the typed placement contract required for client-side reprojection and are outside this node. +- **PTY.IMG-C05 Cross-runtime conformance:** The Node implementation has no equivalent typed image state, so graphics behavior has no Node/Rust conformance pair. + +## Acceptable Tradeoffs + +- **PTY.IMG-T01 Complete bounded replay:** A session may retain up to 32 MiB of image state, and a replay may carry all retained bytes. The limit is a cap, not an eager allocation. +- **PTY.IMG-T02 Latest cell declaration wins:** Cell metrics use the newest client declaration rather than multi-client negotiation. + +## Requirements + +### Must bound accepted image state + +- **PTY.IMG-R01 Opt-in bounded storage:** Image storage is disabled until an owner enables it. The effective storage limit must not exceed 32 MiB and must be observable through the typed terminal API. +- **PTY.IMG-R08 Inline-only transmission:** The terminal must accept image bytes only from inline transmissions. File, temporary-file, and shared-memory media must remain disabled so a child cannot name an owner-unapproved path. +- **PTY.IMG-R09 Bounded PNG normalization:** PNG (`f=100`) input must be bounded before decoding and normalized to 8-bit RGBA before it enters retained image storage. + +### Must survive replay + +- **PTY.IMG-R02 Complete late-client replay:** Every retained image reported for a placement must be carried in full by ATTACH and PEEK replay. The replay must not impose a smaller byte cap than the reported retained state. +- **PTY.IMG-R04 Crop fidelity:** Replay must preserve the resolved, image-bounded source rectangle and offsets for each placement. +- **PTY.IMG-R06 Image-free compatibility:** A session that has accepted no image must retain the pre-image replay byte shape; its image replay block is empty. +- **PTY.IMG-R07 Screen and reset lifecycle:** Normal and alternate screens must retain separate image state. Terminal reset must preserve the configured graphics capability needed to replay retained state into its owning screen. + +### Must support cell-relative composition + +- **PTY.IMG-R03 Window-relative placements:** A graphics read and a grid read taken at the same scroll offset must describe the same terminal window and align cell for cell. Virtual placements follow bounded placeholder cells; direct placements use their screen-space rectangle. Placements outside the requested window must not be reported as visible. +- **PTY.IMG-R05 Declared cell metrics:** ATTACH and RESIZE may append backward-compatible cell width and height fields. An absent declaration must stay explicit and use a deterministic 8×16 fallback. Explicit placement coordinates must not move when cell metrics change. +- **PTY.IMG-R10 Observable content generation:** Image-content changes must advance an observable generation used to fence cached pixels. Scroll and resize may reproject placements without advancing the content generation. diff --git a/docs/vrs/01-images/spec.md b/docs/vrs/01-images/spec.md new file mode 100644 index 0000000..c6f73c0 --- /dev/null +++ b/docs/vrs/01-images/spec.md @@ -0,0 +1,207 @@ +# Terminal images — Specification + +## Status + +Implemented. The session daemon's terminal holds images, `serialize::vt` +replays them, and the embedding handle reads them. This node is lazy — it +stands on [./requirements.md](./requirements.md) alone and requires no parent +`docs/vrs` artifacts. The rationale for the shape below is +[decision 0012](../../decisions/0012-kitty-graphics-replay.md); this +specification states the mechanism and does not restate the argument. + +## Scope + +This specification defines how a pty session holds, bounds, reports, and +replays the terminal images a child draws with the Kitty graphics protocol: +storage admission and its byte bound, placement and crop geometry resolved +against a requested window, the cell pixel metrics the geometry needs, the +replay wire form, and the storage lifecycle across reset and screen switches. + +It does not define rendering — a consumer decides how, where, and whether to +draw what it reads. It does not define the SIXEL or iTerm2 protocols, which +are not read. It does not define Node parity for images: the Node daemon +holds no image state at all, so there is nothing to compare against +(see the Node parity map in [docs/parity.md](../../parity.md) for the +surfaces that do compare). + +Requirements `PTY.IMG-R01` through `PTY.IMG-R10` are in +[./requirements.md](./requirements.md) and are cited inline below. + +## Module map + +| Concern | Source | +| --- | --- | +| Admission, bound, cell metrics, generation | `crates/pty-terminal/src/actor.rs` — `TerminalActor::enable_graphics`, `set_graphics_storage_limit`, `graphics_options`, `set_cell_size`, `graphics_generation`, `reset` | +| Storage bound and options | `crates/pty-terminal/src/graphics.rs` — `MAX_STORAGE_BYTES` (32 MiB), `GraphicsOptions` | +| Window read | `crates/pty-terminal/src/graphics.rs` — `read`, `screen_origin`, `scan_placeholders`, `placeholder_cell`, `PLACEHOLDER` (U+10EEEE), `ROWCOLUMN_DIACRITICS` | +| Owned reported values | `crates/pty-terminal/src/graphics.rs` — `GraphicsState`, `ImageDesc`, `Placement`, `PlacementPosition`, `PlaceholderRect`, `SourceRect`, `image_bytes` | +| Replay emission | `crates/pty-terminal/src/graphics.rs` — `replay`, `transmit`, `place_virtual`, `place_direct`, `clip_top`, `placement_params`; called from `crates/pty-terminal/src/serialize.rs` — `vt` | +| PNG normalization | `crates/pty-terminal/src/graphics.rs` — `PngDecoder`, `expand`, `install_png_decoder`, `MAX_DECODED_PNG_BYTES` (64 MiB) | +| Cell metric wire | `crates/pty-core/src/protocol.rs` — `encode_attach_with_cell`, `encode_resize_with_cell`, `decode_cell` | +| Client and daemon adoption | `crates/pty/src/daemon/clients.rs` — `adopt_cell_size`; `crates/pty/src/daemon/lifecycle.rs` — `terminal_actor` | +| Embedding surface | `crates/pty-terminal/src/handle.rs` — `TerminalHandle::graphics`, `image_bytes`, `set_cell_size`, `graphics_generation`, `HandleEvent::Graphics` | + +## Admission and bound + +`TerminalActor::enable_graphics(GraphicsOptions)` sets libghostty's image +storage limit and installs the `f=100` PNG decoder. Until an owner asks — via +`SpawnOptions::graphics`, `AttachOptions::graphics`, or a non-zero +`set_graphics_storage_limit` — the limit stays zero, which is libghostty's own +"protocol disabled" (`PTY.IMG-R01`). Any requested limit is clamped to +`graphics::MAX_STORAGE_BYTES` and the effective value is readable through +`graphics_options()`, so a caller that asked for more can see what it got. +Zeroing the limit turns the protocol back off. + +File, temporary-file, and shared-memory transmission media are left disabled, +so a child can only send bytes and never name a path (`PTY.IMG-R08`). PNG +(`f=100`) payloads are expanded to 8-bit RGBA before storage sees them, and +rejected above `MAX_DECODED_PNG_BYTES` (`PTY.IMG-R09`). + +The same number bounds the state and the wire: there is no second, smaller +replay cap, so a retained image that `graphics_state` reports for a placement +is carried in full to a late client, whatever its size up to the bound +(`PTY.IMG-R02`). Reachability is scoped to placed images and to formats the +protocol can name — an image transmitted and never placed is not enumerated +(`PTY.IMG-C01`), and the grayscale pixel formats have no kitty `f=` value +(`PTY.IMG-C02`). The limit is a cap, not an allocation — a session whose +child never transmits holds nothing. + +## Window read + +`TerminalActor::graphics_state(scroll_offset)` (and +`TerminalHandle::graphics(scroll_offset)`) answers for the same window +`snapshot(scroll_offset)` reads, so a grid and a graphics state taken with one +offset line up cell for cell (`PTY.IMG-R03`). A cursor-positioned placement is +located by its own screen-space rectangle, which answers for any window and +reports a negative origin row when its top has scrolled above the active area. +A virtual placement has no position of its own, so `scan_placeholders` decodes +the `U+10EEEE` placeholder cells of the window — one pass, never the +scrollback — and reports both the visible cell box and the image cell indices +covered; a bare continuation cell inherits row, column, and image-id high byte +from its predecessor. + +Source rectangles are resolved on read and on replay: the protocol's `w=`/`h=` +default of "the whole dimension" is expanded and then clamped to the image, so +a crop is a concrete rectangle everywhere it is reported (`PTY.IMG-R04`). + +`image_bytes(id)` copies the pixels once, on request, keyed by +`ImageDesc::generation`. + +## Replay wire form + +`serialize::vt(term, scrollback, cell)` appends a graphics block after the +cursor move. The block is: + +```text +first transmission frame ESC _G a=t,q=2,i=,f=[,s=,v=][,o=z][,m=1]; ESC \ +continuation frame ESC _G q=2,m=<1|0>; ESC \ +virtual placement ESC _G a=p,U=1,q=2 ESC \ +cursor-positioned ESC 7 ESC [;H ESC _G a=p,q=2 ESC \ ESC 8 + + ,i=[,p=][,c=][,r=],x=,y=,w=,h=[,X=][,Y=][,z=] +``` + +`q=2` suppresses the terminal's replies on every command, so a replay is +silent. One `a=t` transmission per image, its base64 payload chunked at 4096 +bytes: the first frame carries the full parameter list and gains `,m=1` only +when another chunk follows, and each continuation frame carries just `q=2` +and `m=`, ending with `m=0`. A payload that fits one chunk therefore has no +`m=` at all. `s=`/`v=` carry the pixel dimensions of a raw format and `o=z` +marks a zlib-deflated payload. + +Placement commands carry no payload, so they have no `;` and no terminating +data: the parameter list runs straight into `ESC \`. `p=`, `c=`, `r=`, `X=`, +`Y=`, and `z=` are emitted only when nonzero, since zero is the protocol's +own default for each. The source rectangle is the exception and is always +emitted in full (`x=`, `y=`, `w=`, `h=`), because `w=`/`h=` default to "the +whole image", so omitting them would replay a cropped placement as the wrong +pixels at the right size (`PTY.IMG-R04`). + +A virtual placement is emitted wherever its cells are, history included, since +its command carries no position. A cursor-positioned one needs a cell in the +active area, so it is bracketed by `ESC 7` / `ESC 8` around a `CUP` to that +cell: one whose top has scrolled partly above it is anchored at row 0 with its +crop advanced by the rows that are gone (`clip_top`, which also shrinks `r=`), +and one that has scrolled entirely above it is not re-emitted +(`PTY.IMG-C03`). For a terminal holding no images the block is empty, so a +session that never sent one serializes exactly the bytes it did before +(`PTY.IMG-R06`). + +## Cell metric wire form + +Cell pixel metrics come from the font on the client's host, which a session +daemon may never see, so they are declared rather than guessed +(`PTY.IMG-R05`). The size payload of ATTACH and RESIZE grows an optional +4-byte suffix: + +```text +byte 0..1 rows u16 big-endian +byte 2..3 cols u16 big-endian +byte 4..5 cell_width u16 big-endian (optional) +byte 6..7 cell_height u16 big-endian (optional) +``` + +`encode_attach_with_cell` and `encode_resize_with_cell` write the 8-byte form; +`encode_attach` and `encode_resize` write the plain 4-byte one. Every reader +of a size payload takes rows and cols from the first four bytes and the frame +carries its own length, so a daemon that predates the suffix — the Node one +included — reads the size it always read and ignores the rest. `decode_cell` +returns `None` for a payload shorter than 8 bytes or a degenerate zero. + +The daemon adopts a declaration on either message (`adopt_cell_size` → +`TerminalActor::set_cell_size`); the newest declaration wins. Undeclared is +explicit, not silent: geometry falls back to `CellSize::FALLBACK` (8x16) and +`GraphicsState::cell_declared` is false. A declared cell changes only derived +geometry — a placement that named neither `c=` nor `r=` takes its cell extent +from the image's pixel size divided by the cell size — and never moves a +placement that declared its own extent. + +## Lifecycle + +The session daemon's own terminal has images on +(`lifecycle.rs`, `terminal_actor`, `GraphicsOptions::DEFAULT`): the session is +the durable owner of the child's screen, so it is the durable owner of the +child's images; a daemon without them would serve a replay whose placeholder +cells name images no client can have. When enabling fails the daemon warns and +serves text as before. + +`TerminalActor::reset` re-enables images after `RIS` (`PTY.IMG-R07`). +libghostty's reset restores its own defaults, which include no storage and no +cell metrics, and `reset` is what precedes a replay — a terminal that lost its +storage there would reject the very images the replay is about to carry. + +Storage is per screen, as in the protocol: the alternate screen holds its own +images, so a full-screen program's are not the primary screen's and neither +set is lost when the child switches. A replay taken from the alternate screen +brackets the normal half so that the normal screen's images land in the normal +screen's storage (`PTY.IMG-R07`). + +Content change is signalled once, by a counter: `HandleEvent::Graphics(gen)` +and `graphics_generation()` key a texture cache (`PTY.IMG-R10`). Scrolling and +resizing move placements without bumping the generation, so a dirty frame +re-reads positions but does not re-upload pixels; a new transmission and a +child-sent delete both bump it. + +## Validation + +Test names are function names in the files given. `graphics.rs` and +`handle.rs` are under `crates/pty-terminal/tests/`, `protocol.rs` under +`crates/pty-core/tests/`. + +| Requirement | Owning source | Executable evidence | +| --- | --- | --- | +| `PTY.IMG-R01` | `actor.rs`, `graphics.rs` | `graphics.rs::graphics_are_off_until_the_owner_asks`, `zeroing_the_limit_turns_the_protocol_off`, `the_storage_limit_alone_turns_graphics_fully_on`, `the_storage_limit_is_clamped_to_what_a_replay_can_carry`, `raising_the_storage_limit_keeps_the_cell_size_and_decodes_png` | +| `PTY.IMG-R02` | `graphics.rs` (`replay`, `transmit`), `serialize.rs` | `graphics.rs::a_late_client_reconstructs_the_image_from_the_replay_alone`, `an_image_far_larger_than_three_mib_still_replays`, `a_cursor_positioned_placement_replays_at_its_cell`, `handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` | +| `PTY.IMG-R03` | `graphics.rs` (`read`, `screen_origin`, `scan_placeholders`) | `graphics.rs::an_omp_write_gives_image_bytes_placement_identity_crop_and_position`, `scrolling_moves_the_placement_and_scrollback_still_finds_it`, `a_resize_keeps_the_image_and_reprojects_it`, `a_virtual_placement_scrolled_into_history_still_replays`, `a_partially_scrolled_direct_placement_replays_clipped`, `a_palette_foreground_names_a_placeholder_image`, `a_bare_continuation_cell_inherits_row_column_and_id_high_byte` | +| `PTY.IMG-R04` | `graphics.rs` (`placement_params`, `clip_top`) | `graphics.rs::a_source_rect_and_offsets_come_back_resolved`, `a_cropped_placement_keeps_its_crop_through_a_replay` | +| `PTY.IMG-R05` | `protocol.rs` (`decode_cell`), `clients.rs`, `graphics.rs` (`CellSize`) | `protocol.rs::attach_can_declare_a_cell_size_without_changing_the_size_it_carries`, `resize_can_declare_a_cell_size`, `a_plain_size_payload_declares_no_cell`, `graphics.rs::an_implicit_placement_takes_its_extent_from_the_declared_cell`, `a_declared_cell_does_not_move_an_explicit_placement`, `handle.rs::a_client_declares_its_cell_size_and_the_session_geometry_follows` | +| `PTY.IMG-R06` | `graphics.rs` (`replay`), `serialize.rs` (`vt`) | `graphics.rs::a_replay_without_graphics_is_unchanged` | +| `PTY.IMG-R07` | `actor.rs` (`reset`), `lifecycle.rs` | `graphics.rs::the_alternate_screen_has_its_own_storage`, `a_replay_from_the_alt_screen_puts_the_normal_screens_images_on_the_normal_screen`, `handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` (reconnect half) | +| `PTY.IMG-R08` | `actor.rs` (`enable_graphics`), `graphics.rs` (`GraphicsOptions`) | None. Enforced by leaving the file, temporary-file, and shared-memory media disabled; backed by [decision 0012](../../decisions/0012-kitty-graphics-replay.md) only. A regression here would not fail the suite. | +| `PTY.IMG-R09` | `graphics.rs` (`PngDecoder`, `expand`, `MAX_DECODED_PNG_BYTES`) | `graphics.rs::a_grayscale_png_is_stored_as_rgba`, `raising_the_storage_limit_keeps_the_cell_size_and_decodes_png`, `handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` | +| `PTY.IMG-R10` | `graphics.rs` (`generation`), `handle.rs` (`HandleEvent::Graphics`) | `graphics.rs::a_spawned_child_that_draws_an_image_is_queryable_through_the_handle`, `a_delete_from_the_child_drops_the_placement_and_the_bytes`, `scrolling_moves_the_placement_and_scrollback_still_finds_it`, `a_resize_keeps_the_image_and_reprojects_it` | + +Run the whole map with `cargo test -p pty-terminal -p pty-core`; the image +cases alone with `cargo test -p pty-terminal --test graphics`. No gated +`_node` / `_rust` conformance pair exists or can: the Node side has no image +state to compare against (`PTY.IMG-C05`).