diff --git a/Cargo.lock b/Cargo.lock index c79e032..839e1d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -172,6 +178,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -360,6 +375,15 @@ dependencies = [ "regex", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filedescriptor" version = "0.8.3" @@ -383,6 +407,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -747,6 +782,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -1027,6 +1082,19 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -1130,6 +1198,7 @@ version = "0.13.0-rust" dependencies = [ "libc", "libghostty-vt", + "png", "portable-pty", "pty-core", "serde_json", @@ -1498,6 +1567,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.3" @@ -2151,6 +2226,12 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 0f1b9e7..bf40fcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1.0.229", features = ["derive"] } serde_json = { version = "1.0.151", features = ["preserve_order"] } indexmap = { version = "2.14.0", features = ["serde"] } notify = "8.2.0" +png = "0.18.1" toml = { version = "1.1.3", features = ["preserve_order"] } # Workspace crates. diff --git a/README.md b/README.md index 97edc7a..ffee244 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,12 @@ Where the port stands against the Node `pty`, surface by surface, is in [docs/parity.md](docs/parity.md); the work packages that close the gap are in [docs/parity-plan.md](docs/parity-plan.md). +What must stay true of terminal images — the Kitty graphics protocol state a +session holds and replays — is in +[docs/vrs/01-images/requirements.md](docs/vrs/01-images/requirements.md); how +the session meets it is in +[docs/vrs/01-images/spec.md](docs/vrs/01-images/spec.md). + ## Install With Nix (flakes), from a checkout or straight from GitHub: diff --git a/crates/pty-conformance/tests/output_activity.rs b/crates/pty-conformance/tests/output_activity.rs index c1a469d..25c511a 100644 --- a/crates/pty-conformance/tests/output_activity.rs +++ b/crates/pty-conformance/tests/output_activity.rs @@ -1,7 +1,8 @@ //! `lastOutputAtMs`: the daemon records when the child last printed. //! -//! The Node tool grew this field in its PR #168, merged on 2026-08-29 -//! (`docs/vrs/requirements.md` R14, `docs/disk-layout.md`). The contract is: +//! The Node tool grew this field in its PR #168, merged on 2026-08-29 (the +//! Node pty repository's `docs/vrs/requirements.md` R14 and +//! `docs/disk-layout.md`, not this repository's `docs/`). The contract is: //! //! - absent until the session produces output, and absent on a record an //! older daemon wrote — never zero, and never a claim of idleness; diff --git a/crates/pty-core/src/protocol.rs b/crates/pty-core/src/protocol.rs index 748378c..2164843 100644 --- a/crates/pty-core/src/protocol.rs +++ b/crates/pty-core/src/protocol.rs @@ -114,11 +114,35 @@ fn size_payload(rows: u16, cols: u16) -> [u8; 4] { [r[0], r[1], c[0], c[1]] } +/// `rows u16BE, cols u16BE, cell_width u16BE, cell_height u16BE`: the size +/// payload with the client's cell pixel metrics appended. +/// +/// The four extra bytes are an *optional suffix*, which is what makes this +/// safe to send to any peer: every reader of a size payload takes rows and +/// cols from the first four bytes and the frame carries its own length, so a +/// daemon that predates this (the Node one included) reads the size it +/// always read and ignores the rest. +fn size_cell_payload(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> [u8; 8] { + let s = size_payload(rows, cols); + let w = cell_width.to_be_bytes(); + let h = cell_height.to_be_bytes(); + [s[0], s[1], s[2], s[3], w[0], w[1], h[0], h[1]] +} + /// Encode an ATTACH with a terminal size (4-byte payload). pub fn encode_attach(rows: u16, cols: u16) -> Vec { encode_packet(MessageType::Attach, &size_payload(rows, cols)) } +/// Encode an ATTACH that also declares the client's cell pixel size +/// (8-byte payload; see [`decode_cell`]). +pub fn encode_attach_with_cell(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> Vec { + encode_packet( + MessageType::Attach, + &size_cell_payload(rows, cols, cell_width, cell_height), + ) +} + /// Encode a DETACH. pub fn encode_detach() -> Vec { encode_packet(MessageType::Detach, &[]) @@ -129,6 +153,14 @@ pub fn encode_resize(rows: u16, cols: u16) -> Vec { encode_packet(MessageType::Resize, &size_payload(rows, cols)) } +/// Encode a RESIZE that also declares the client's cell pixel size. +pub fn encode_resize_with_cell(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> Vec { + encode_packet( + MessageType::Resize, + &size_cell_payload(rows, cols, cell_width, cell_height), + ) +} + /// Encode a GEOMETRY (effective shared rows/cols, server → client). pub fn encode_geometry(rows: u16, cols: u16) -> Vec { encode_packet(MessageType::Geometry, &size_payload(rows, cols)) @@ -177,6 +209,21 @@ pub fn decode_size(payload: &[u8]) -> (u16, u16) { (rows, cols) } +/// The cell pixel size a client appended to an ATTACH or RESIZE payload, or +/// `None` when it sent the plain 4-byte size or a degenerate zero. +/// +/// Cell metrics are the client's to know — they come from its font, on its +/// host — so a session daemon can only be told. `None` means nobody has, and +/// the reader keeps its own deterministic fallback. +pub fn decode_cell(payload: &[u8]) -> Option<(u16, u16)> { + if payload.len() < 8 { + return None; + } + let width = u16::from_be_bytes([payload[4], payload[5]]); + let height = u16::from_be_bytes([payload[6], payload[7]]); + (width > 0 && height > 0).then_some((width, height)) +} + /// Decode a GEOMETRY payload (rows, cols); same layout and fallback as /// [`decode_size`]. pub fn decode_geometry(payload: &[u8]) -> (u16, u16) { diff --git a/crates/pty-core/src/registry/metadata.rs b/crates/pty-core/src/registry/metadata.rs index 6ed68db..8eabe79 100644 --- a/crates/pty-core/src/registry/metadata.rs +++ b/crates/pty-core/src/registry/metadata.rs @@ -89,7 +89,8 @@ pub struct SessionMetadata { /// nothing to take; it is persisted at most once a second while output /// flows. /// - /// node: src/sessions.ts (`lastOutputAtMs`), docs/vrs/requirements.md R14 + /// node: src/sessions.ts (`lastOutputAtMs`), the Node pty repository's + /// `docs/vrs/requirements.md` R14 (not this repository's `docs/vrs`) #[serde(default, skip_serializing_if = "Option::is_none")] pub last_output_at_ms: Option, /// Every field this version does not model, round-tripped verbatim. diff --git a/crates/pty-core/tests/protocol.rs b/crates/pty-core/tests/protocol.rs index 3fada09..2754343 100644 --- a/crates/pty-core/tests/protocol.rs +++ b/crates/pty-core/tests/protocol.rs @@ -1,9 +1,10 @@ //! Port of the pty project's `tests/protocol.test.ts`. use pty_core::protocol::{ - MAX_PACKET_LENGTH, MessageType, PacketReader, decode_exit, decode_geometry, decode_size, - encode_attach, encode_data, encode_detach, encode_exit, encode_geometry, encode_packet, - encode_resize, encode_screen, encode_status, encode_status_response, + MAX_PACKET_LENGTH, MessageType, PacketReader, decode_cell, decode_exit, decode_geometry, + decode_size, encode_attach, encode_attach_with_cell, encode_data, encode_detach, encode_exit, + encode_geometry, encode_packet, encode_resize, encode_resize_with_cell, encode_screen, + encode_status, encode_status_response, }; use pty_core::stats::{ClientStats, ConnectionStats, Constrains, StatsResult}; @@ -36,6 +37,50 @@ fn attach_byte_identical_to_hand_built_packet() { ); } +/// The cell pixel size is an optional suffix on a size payload: a reader that +/// does not know about it takes the same rows and cols it always did, which +/// is what makes this safe to send to any daemon. +#[test] +fn attach_can_declare_a_cell_size_without_changing_the_size_it_carries() { + let with_cell = encode_attach_with_cell(24, 80, 9, 18); + assert_eq!( + with_cell, + encode_packet(MessageType::Attach, &[0, 24, 0, 80, 0, 9, 0, 18]) + ); + + let mut reader = PacketReader::new(); + let packets = reader.feed(&with_cell).unwrap(); + assert_eq!(packets[0].type_, MessageType::Attach); + assert_eq!( + decode_size(&packets[0].payload), + (24, 80), + "the size is where it always was" + ); + assert_eq!(decode_cell(&packets[0].payload), Some((9, 18))); +} + +#[test] +fn resize_can_declare_a_cell_size() { + let mut reader = PacketReader::new(); + let packets = reader.feed(&encode_resize_with_cell(30, 100, 7, 15)).unwrap(); + assert_eq!(packets[0].type_, MessageType::Resize); + assert_eq!(decode_size(&packets[0].payload), (30, 100)); + assert_eq!(decode_cell(&packets[0].payload), Some((7, 15))); +} + +/// No declaration is the normal case, and it must not be mistaken for one. +#[test] +fn a_plain_size_payload_declares_no_cell() { + assert_eq!(decode_cell(&encode_attach(24, 80)[5..]), None); + assert_eq!(decode_cell(&encode_resize(24, 80)[5..]), None); + assert_eq!(decode_cell(&[]), None); + assert_eq!( + decode_cell(&[0, 24, 0, 80, 0, 0, 0, 0]), + None, + "a zero cell is no cell" + ); +} + /// node: tests/protocol.test.ts:288-297 #[test] fn round_trips_geometry() { diff --git a/crates/pty-terminal/Cargo.toml b/crates/pty-terminal/Cargo.toml index 92bf1b2..8850cf4 100644 --- a/crates/pty-terminal/Cargo.toml +++ b/crates/pty-terminal/Cargo.toml @@ -12,6 +12,7 @@ libghostty-vt.workspace = true portable-pty.workspace = true pty-core.workspace = true libc.workspace = true +png.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/crates/pty-terminal/src/actor.rs b/crates/pty-terminal/src/actor.rs index 4cd1ca2..84b4c8f 100644 --- a/crates/pty-terminal/src/actor.rs +++ b/crates/pty-terminal/src/actor.rs @@ -16,15 +16,58 @@ use std::rc::Rc; use libghostty_vt::style::RgbColor; use libghostty_vt::terminal::{Options, Terminal}; +use crate::graphics::{self, CellSize, GraphicsOptions, GraphicsState, ImageBytes}; +use crate::input::{self, KeyEvent, MouseEvent}; use crate::queries; use crate::screenshot::{self, Screenshot}; use crate::serialize::{self, SerializeOpts}; use crate::snapshot::{self, CellGrid}; use crate::strip::{OutputScanner, Osc, Token}; -/// Node's scrollback (`src/server.ts:333-338`). +/// Node's scrollback (`src/server.ts:333-338`), in lines. pub const DEFAULT_SCROLLBACK: usize = 10_000; +/// The most memory one terminal's history may be given, whatever scrollback +/// it was asked for: 64 MiB, which is 10 000 lines of a 400-column terminal. +/// +/// A session that asks for more history than this gets what fits and says so +/// ([`TerminalActor::scrollback`] reports what is actually retainable, not +/// what was requested), because a silently unmet promise is what this whole +/// conversion exists to remove. +pub const MAX_SCROLLBACK_BYTES: usize = 64 * 1024 * 1024; + +/// libghostty's `max_scrollback` is a **byte** budget for the history page +/// list, not a line count — its own doc comment says "lines", but the +/// behaviour is bytes and the page list evicts whole pages. Passing 10 000 +/// for "10 000 lines" buys one page: 745 rows at 80 columns, 456 at 200, +/// 3 310 at 20. A line promise therefore has to be converted, and the cost +/// of a row depends on how wide it is. +/// +/// These two numbers are the conversion, measured against libghostty-vt +/// 0.2.1: a plain row costs ~838 bytes at 80 columns and ~1 804 at 200, so +/// ~9-10 bytes per column plus per-row overhead. They are deliberately +/// generous — roughly 1.8x the measured cost — because the page list rounds +/// up to whole pages and because a row of styled or multi-codepoint cells +/// costs more than a plain one. Under-budgeting loses history; over-budgeting +/// costs address space that is only touched when the history actually fills. +const SCROLLBACK_BYTES_PER_COL: usize = 16; +/// Fixed per-row cost, independent of width. See +/// [`SCROLLBACK_BYTES_PER_COL`]. +const SCROLLBACK_ROW_OVERHEAD: usize = 256; + +/// What one row of a `cols`-wide terminal costs in the history. +fn scrollback_row_bytes(cols: u16) -> usize { + SCROLLBACK_ROW_OVERHEAD + SCROLLBACK_BYTES_PER_COL * cols.max(1) as usize +} + +/// The byte budget that retains `lines` lines of a `cols`-wide terminal, +/// capped at [`MAX_SCROLLBACK_BYTES`]. +fn scrollback_budget(lines: usize, cols: u16) -> usize { + lines + .saturating_mul(scrollback_row_bytes(cols)) + .min(MAX_SCROLLBACK_BYTES) +} + /// A desktop notification the child asked for (OSC 9, 99, or 777). /// Shapes follow Node (`src/server.ts:421-454`). #[derive(Debug, Clone, PartialEq, Eq)] @@ -66,6 +109,15 @@ pub struct Modes { pub mouse_1002: bool, /// `?1003`. pub mouse_1003: bool, + /// `?9` — X10 mouse reporting: button presses only, no releases, no + /// motion, and no wheel. Deliberately *not* part of + /// [`Modes::mouse_tracking`]: an input encoder has to treat it + /// differently from `?1000`/`?1002`/`?1003`, which do report the wheel. + pub mouse_9: bool, + /// `?1` (DECCKM) — application cursor keys: the child wants `SS3 A` for + /// Up, not `CSI A`. Node does not track it; an input encoder cannot + /// encode an arrow key correctly without it. + pub app_cursor: bool, /// `?1049` / `?1047` / `?47`. pub alt_screen: bool, /// `?25 l` seen and not yet undone by `?25 h`. @@ -84,12 +136,20 @@ impl Modes { self.mouse_1000 || self.mouse_1002 || self.mouse_1003 } + /// Whether any mouse reporting at all is on, X10 included. Use + /// [`Modes::mouse_tracking`] for "reports the wheel". + pub fn mouse_reporting(&self) -> bool { + self.mouse_9 || self.mouse_tracking() + } + fn apply_dec(&mut self, mode: u16, set: bool, events: &mut Vec) { match mode { 1006 => self.sgr_mouse = set, 1000 => self.mouse_1000 = set, 1002 => self.mouse_1002 = set, 1003 => self.mouse_1003 = set, + 9 => self.mouse_9 = set, + 1 => self.app_cursor = set, 1049 | 1047 | 47 => self.alt_screen = set, 25 => { if set { @@ -137,7 +197,20 @@ pub struct TerminalActor { modes: Modes, events: Vec, last_title: Option, - scrollback: usize, + /// Lines of history the owner asked for. + scrollback_request: usize, + /// The byte budget libghostty was given for the history. Fixed at + /// construction: libghostty takes it in `Options` and exposes no setter, + /// so a resize changes how many lines it holds, not how much memory. + scrollback_bytes: usize, + /// How big a cell is on the surface that draws this terminal, zero when + /// undeclared. It comes from the client (a font on its host), travels on + /// ATTACH and RESIZE, and decides the cell extent of any placement that + /// did not name `c=`/`r=` itself. + cell: CellSize, + /// Whether kitty graphics are on, and with what bounds. Off means the + /// storage limit is zero, which is libghostty's own "protocol disabled". + graphics: Option, /// The normal screen, serialized at the moment the child switched to the /// alternate one. libghostty gives no reader for the screen that is not /// active, and a replay has to carry both: a client that reconnects while @@ -149,12 +222,17 @@ pub struct TerminalActor { impl TerminalActor { /// A terminal of `rows` x `cols` with `scrollback` lines of history. + /// + /// The line count is converted to the byte budget libghostty actually + /// takes (see [`MAX_SCROLLBACK_BYTES`]); [`TerminalActor::scrollback`] + /// reports how many lines that buys at the current width. pub fn new(rows: u16, cols: u16, scrollback: usize) -> TerminalActor { let shared: Rc> = Rc::new(RefCell::new(Shared::default())); + let scrollback_bytes = scrollback_budget(scrollback, cols); let mut term = Terminal::new(Options { cols: cols.max(1), rows: rows.max(1), - max_scrollback: scrollback, + max_scrollback: scrollback_bytes, }) .expect("libghostty terminal"); { @@ -183,7 +261,10 @@ impl TerminalActor { modes: Modes::default(), events: Vec::new(), last_title: None, - scrollback, + scrollback_request: scrollback, + scrollback_bytes, + cell: CellSize::default(), + graphics: None, normal_replay: None, } } @@ -193,6 +274,198 @@ impl TerminalActor { TerminalActor::new(24, 80, DEFAULT_SCROLLBACK) } + /// Turn kitty graphics on: a bounded image storage, the cell metrics + /// every placement question is answered in, and a PNG decoder for + /// `f=100` transmissions (installed on this thread, where the terminal + /// lives). Returns false when libghostty refused the options. + /// + /// `storage_bytes` is clamped to [`graphics::MAX_STORAGE_BYTES`] so + /// everything the terminal accepts can also be replayed; read back what + /// took effect with [`TerminalActor::graphics_options`]. A zero + /// `opts.cell` leaves the metrics undeclared — see + /// [`TerminalActor::set_cell_size`]. + /// + /// Off by default: an owner that never reads images should not let a + /// child make the terminal hold any. + pub fn enable_graphics(&mut self, opts: GraphicsOptions) -> bool { + let opts = GraphicsOptions { + storage_bytes: opts.storage_bytes.min(graphics::MAX_STORAGE_BYTES), + ..opts + }; + let previous = self.graphics; + if opts.cell.is_declared() { + self.cell = opts.cell; + } + if self.apply_cell().is_err() { + self.rollback_graphics(previous); + return false; + } + if self.term.set_apc_max_bytes_kitty(opts.apc_max_bytes).is_err() + || self + .term + .set_kitty_image_storage_limit(opts.storage_bytes) + .is_err() + { + self.rollback_graphics(previous); + return false; + } + // The child must not be able to name a path or a shared-memory + // segment the owner never authorized: inline transmission only. + let _ = self.term.set_kitty_image_from_file_allowed(false); + let _ = self.term.set_kitty_image_from_temp_file_allowed(false); + let _ = self.term.set_kitty_image_from_shared_mem_allowed(false); + if !graphics::install_png_decoder() { + self.rollback_graphics(previous); + return false; + } + self.graphics = Some(GraphicsOptions { + cell: self.cell, + ..opts + }); + true + } + + /// Put the storage limit back where a failed [`TerminalActor::enable_graphics`] + /// found it. + /// + /// Leaving a non-zero limit behind after a failure is the worst of both + /// worlds: `self.graphics` is `None`, so a child can fill the storage + /// with images that no read can reach and `clear_graphics` will not + /// free. + fn rollback_graphics(&mut self, previous: Option) { + let limit = previous.map(|o| o.storage_bytes).unwrap_or(0); + let _ = self.term.set_kitty_image_storage_limit(limit); + self.graphics = previous; + } + + /// Push the current grid and cell metrics into libghostty. + fn apply_cell(&mut self) -> Result<(), libghostty_vt::error::Error> { + let cell = self.cell.or_fallback(); + self.term + .resize(self.cols().max(1), self.rows().max(1), cell.width, cell.height) + .map(|_| ()) + } + + /// Declare how big a cell is on the surface that draws this terminal. + /// + /// Nothing else can know: the metrics come from a font on the client's + /// host, which a session daemon may never see. They change no bytes — + /// only derived geometry, which is why a placement that named `c=`/`r=` + /// itself is unaffected and one that did not now gets a cell extent that + /// matches what the client will actually draw. + /// + /// A zero width or height clears the declaration and returns the + /// terminal to [`CellSize::FALLBACK`]. + pub fn set_cell_size(&mut self, cell: CellSize) { + if self.cell == cell { + return; + } + self.cell = cell; + let _ = self.apply_cell(); + if let Some(opts) = &mut self.graphics { + opts.cell = cell; + } + } + + /// The graphics options in effect, or `None` when graphics are off. + pub fn graphics_options(&self) -> Option { + self.graphics + } + + /// The image-storage generation: 0 when graphics are off or nothing has + /// ever been stored. Unchanged means the images and the set of placements + /// are identical; geometry can still have moved. + pub fn graphics_generation(&self) -> u64 { + if self.graphics.is_none() { + return 0; + } + graphics::generation(&self.term) + } + + /// The images and placements for the window `scroll_offset` rows above + /// the live viewport — the same window [`TerminalActor::snapshot`] reads, + /// so cell positions in both line up. + pub fn graphics_state(&self, scroll_offset: usize) -> GraphicsState { + if self.graphics.is_none() { + return GraphicsState::default(); + } + graphics::read(&self.term, self.cell, scroll_offset) + } + + /// The pixels of one image, copied out of the storage. `None` when it is + /// not there (a delete won the race). + pub fn image_bytes(&self, id: u32) -> Option { + self.graphics.and_then(|_| graphics::image_bytes(&self.term, id)) + } + + /// Drop every image and placement, keeping the protocol on: what a pane + /// does when it closes or is reused. Zeroing the limit is libghostty's + /// own delete-everything path, so this needs no synthesized escape. + pub fn clear_graphics(&mut self) { + let Some(opts) = self.graphics else { return }; + let _ = self.term.set_kitty_image_storage_limit(0); + let _ = self.term.set_kitty_image_storage_limit(opts.storage_bytes); + } + + /// Raise or lower the storage limit, clamped to + /// [`graphics::MAX_STORAGE_BYTES`] so everything stored can still be + /// replayed. Zero turns the protocol off and deletes everything stored. + /// + /// A non-zero limit goes through [`TerminalActor::enable_graphics`], so + /// there is one path that turns graphics on: raising the limit on an + /// actor that never enabled them installs the PNG decoder and the cell + /// metrics too, instead of leaving a terminal that stores images it + /// cannot decode or measure. + pub fn set_graphics_storage_limit(&mut self, bytes: u64) { + if bytes == 0 { + let _ = self.term.set_kitty_image_storage_limit(0); + self.graphics = None; + return; + } + let opts = self.graphics.unwrap_or(GraphicsOptions { + cell: self.cell, + ..GraphicsOptions::DEFAULT + }); + self.enable_graphics(GraphicsOptions { + storage_bytes: bytes, + ..opts + }); + } + + /// The declared cell metrics, or a zero size when nobody has declared + /// them. [`CellSize::or_fallback`] gives what geometry actually used. + pub fn cell_size(&self) -> CellSize { + self.cell + } + + /// The bytes the child expects for one key event, given the keyboard + /// state it asked for (DECCKM, keypad, `modifyOtherKeys`, kitty + /// keyboard flags). Empty for an event the child should not see. + /// + /// This is the only child key encoder: a consumer that owns a surface + /// sends [`KeyEvent`]s, not bytes (see [`crate::input`]). + pub fn encode_key(&self, ev: &KeyEvent) -> Vec { + input::key(&self.term, ev) + } + + /// The bytes for one mouse event, or `None` when the mode the child + /// chose does not report it (no tracking, or a wheel notch under `?9`) + /// and the surface keeps the event. + pub fn encode_mouse(&self, ev: &MouseEvent) -> Option> { + input::mouse(&self.term, &self.modes, ev, self.cell_size()) + } + + /// The bytes for a focus change, or `None` when the child did not ask + /// for focus events. + pub fn encode_focus(&self, gained: bool) -> Option> { + input::focus(&self.modes, gained) + } + + /// Pasted text, bracketed when the child asked for it. + pub fn encode_paste(&self, text: &str) -> Vec { + input::paste(&self.modes, text) + } + /// The normal screen as it was when the child entered the alternate /// one, or `None` when the normal screen is the active one. pub fn normal_replay(&self) -> Option<&str> { @@ -236,7 +509,7 @@ impl TerminalActor { (false, true) => { self.flush_feed(&mut feed); self.normal_replay = - Some(crate::serialize::vt(&self.term, true)); + Some(crate::serialize::vt(&self.term, true, self.cell)); } // Back on the normal screen: it serializes itself. (true, false) => self.normal_replay = None, @@ -335,9 +608,14 @@ impl TerminalActor { self.events.push(TerminalEvent::Notification(n)); } - /// Resize the terminal (the primary screen reflows). + /// Resize the terminal (the primary screen reflows). The cell metrics + /// stay as they are: a resize is a change of grid, not of font, and + /// dropping them would make every placement's geometry unanswerable. pub fn resize(&mut self, cols: u16, rows: u16) { - let _ = self.term.resize(cols.max(1), rows.max(1), 0, 0); + let cell = self.cell.or_fallback(); + let _ = self + .term + .resize(cols.max(1), rows.max(1), cell.width, cell.height); } /// Full reset (RIS): screen, scrollback, modes, title. The tracked mode @@ -349,6 +627,13 @@ impl TerminalActor { self.modes = Modes::default(); self.shared.borrow_mut().titles.clear(); self.shared.borrow_mut().bells = 0; + // RIS restores libghostty's defaults, which include no image storage + // and no cell metrics. An owner that asked for graphics keeps them + // across the reset that precedes a SCREEN replay — otherwise the + // replay's own images would be rejected. + if let Some(opts) = self.graphics { + self.enable_graphics(opts); + } } /// The plain-text screen: rows right-trimmed of never-written cells @@ -420,13 +705,38 @@ impl TerminalActor { } /// Node's `scrollbackCapacity`: `rows + scrollback`. + /// + /// A guaranteed minimum, not a ceiling. libghostty's history is a list of + /// pages and it never holds less than one, so a terminal asked for a + /// small scrollback retains more than it promised (a 100-line request at + /// 80 columns keeps ~1 000 rows). Node's number is a ceiling because + /// xterm counts lines; this one is a floor because libghostty counts + /// bytes and rounds to pages + /// (docs/decisions/0013-scrollback-is-a-line-promise.md). pub fn scrollback_capacity(&self) -> usize { - self.rows() as usize + self.scrollback + self.rows() as usize + self.scrollback() } - /// Configured scrollback lines. + /// How many lines of history this terminal retains at its current width. + /// + /// Normally the line count the owner asked for. It is less when the byte + /// budget cannot buy that many — either because the request exceeded + /// [`MAX_SCROLLBACK_BYTES`], or because the terminal has since been + /// widened and libghostty's budget is fixed at construction. Reporting + /// the request in that case is what made the promise a lie. pub fn scrollback(&self) -> usize { - self.scrollback + let fits = self.scrollback_bytes / scrollback_row_bytes(self.cols()); + self.scrollback_request.min(fits) + } + + /// Lines of history the owner asked for, whether or not they fit. + pub fn scrollback_request(&self) -> usize { + self.scrollback_request + } + + /// The byte budget libghostty holds the history in. + pub fn scrollback_bytes(&self) -> usize { + self.scrollback_bytes } /// Node's `baseY`: the buffer row where the active area starts. diff --git a/crates/pty-terminal/src/graphics.rs b/crates/pty-terminal/src/graphics.rs new file mode 100644 index 0000000..be42ba2 --- /dev/null +++ b/crates/pty-terminal/src/graphics.rs @@ -0,0 +1,1099 @@ +//! Kitty graphics state: the durable, typed image state behind +//! [`crate::actor::TerminalActor`] and [`crate::handle::TerminalHandle`]. +//! +//! libghostty owns the image storage; this module is the boundary that turns +//! its borrowed handles into owned, `Send` values a compositor can hold across +//! frames, and that puts the storage back on the wire for the ATTACH/PEEK +//! replay (see [`replay`]). +//! +//! What a consumer gets: +//! +//! - [`GraphicsState`]: the storage generation, every [`ImageDesc`] that has a +//! placement, and every [`Placement`] with its resolved source crop, cell +//! extent, and position in the window it was read for. +//! - [`ImageBytes`]: the pixels themselves, copied once, on request. Metadata +//! is cheap enough to read per frame; bytes are not, so they are keyed by +//! [`ImageDesc::generation`] and fetched only when that changes. +//! +//! Positions are typed rather than optional numbers ([`PlacementPosition`]): +//! libghostty resolves a cursor-positioned placement to a viewport cell, but a +//! virtual (Unicode placeholder) placement has no position of its own — it is +//! wherever its placeholder cells are. Those cells are ordinary text, each +//! naming its own image row and column, so this module decodes them from the +//! grid; that is what survives scrolling, reflow, and a windowed read. +//! +//! Kitty only: SIXEL and the iTerm2 protocol have no equivalent +//! arbitrary-pane contract and are not read here. + +use libghostty_vt::alloc::{Allocator, Bytes}; +use libghostty_vt::kitty::graphics as gfx; +use libghostty_vt::style::StyleColor; +use libghostty_vt::terminal::{Point, PointCoordinate, PointSpace, Terminal}; + +/// The Unicode placeholder character (U+10EEEE) a virtual placement is drawn +/// with. +pub const PLACEHOLDER: char = '\u{10eeee}'; + +/// Cell pixel metrics. Kitty graphics geometry is defined in pixels, so a +/// terminal that stores images has to know how big a cell is: a placement +/// that did not say `c=`/`r=` gets its cell extent from the image's pixel +/// size divided by this. +/// +/// The metrics belong to whoever draws the cells — a font, on a host the +/// session daemon may never see — so they travel from the client +/// ([`crate::handle::AttachOptions::graphics`], carried on ATTACH and +/// RESIZE) rather than being assumed. Zero means undeclared, and derived +/// geometry uses [`CellSize::FALLBACK`] until someone says otherwise. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CellSize { + /// Cell width in pixels; zero when undeclared. + pub width: u32, + /// Cell height in pixels; zero when undeclared. + pub height: u32, +} + +impl CellSize { + /// The conventional 8x16 monospace cell, used only when nobody has + /// declared the real one. Deterministic on purpose: two clients reading + /// an undeclared terminal must agree, even though both are guessing. + pub const FALLBACK: CellSize = CellSize { + width: 8, + height: 16, + }; + + /// Whether a real cell size was declared. + pub fn is_declared(&self) -> bool { + self.width > 0 && self.height > 0 + } + + /// This size, or [`CellSize::FALLBACK`] when undeclared. + pub fn or_fallback(self) -> CellSize { + if self.is_declared() { + self + } else { + CellSize::FALLBACK + } + } +} + +/// The largest image storage this module supports, and therefore the largest +/// a replay can have to carry: 32 MiB, four 2048x2048 RGBA images. +/// +/// One number bounds both on purpose. A storage limit above what a replay +/// carries would mean a terminal that accepts an image a late client can +/// never receive — a supported state that silently loses data, which is the +/// failure this module exists to prevent. +/// [`crate::actor::TerminalActor::enable_graphics`] clamps to it, so a caller +/// asking for more gets this and can see that it did +/// ([`crate::actor::TerminalActor::graphics_options`]). +pub const MAX_STORAGE_BYTES: u64 = 32 * 1024 * 1024; + +/// How much graphics state a terminal may hold, and how it measures cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GraphicsOptions { + /// Image storage limit in bytes, clamped to [`MAX_STORAGE_BYTES`]. Zero + /// disables the protocol; this is the only bound on how much a child can + /// make the terminal hold, and it is also the bound on a replay. + pub storage_bytes: u64, + /// Cell metrics used for every pixel/cell conversion. Zero means + /// undeclared: nobody has told the terminal how big a cell is, and + /// derived geometry falls back to [`CellSize::FALLBACK`]. + pub cell: CellSize, + /// Cap on the bytes one APC command may buffer (`None` keeps + /// libghostty's default). + pub apc_max_bytes: Option, +} + +impl GraphicsOptions { + /// The full supported storage, a conventional 8x16 cell, libghostty's + /// APC cap. + pub const DEFAULT: GraphicsOptions = GraphicsOptions { + storage_bytes: MAX_STORAGE_BYTES, + cell: CellSize { + width: 8, + height: 16, + }, + apc_max_bytes: None, + }; +} + +impl Default for GraphicsOptions { + fn default() -> Self { + GraphicsOptions::DEFAULT + } +} + +/// Pixel format of stored image data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PixelFormat { + /// 24-bit RGB (kitty `f=24`). + Rgb, + /// 32-bit RGBA (kitty `f=32`). + Rgba, + /// PNG bytes (kitty `f=100`), still encoded. + Png, + /// 8-bit grayscale. + Gray, + /// 8-bit grayscale + alpha. + GrayAlpha, +} + +impl PixelFormat { + /// The kitty `f=` value, or `None` for a format the protocol cannot + /// express (grayscale). + pub fn kitty_format(self) -> Option { + match self { + PixelFormat::Rgb => Some(24), + PixelFormat::Rgba => Some(32), + PixelFormat::Png => Some(100), + PixelFormat::Gray | PixelFormat::GrayAlpha => None, + } + } + + /// Whether the data is raw pixels (and therefore needs `s=`/`v=` when + /// transmitted). + pub fn is_raw(self) -> bool { + !matches!(self, PixelFormat::Png) + } +} + +/// Compression applied to stored image data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Compression { + /// Stored uncompressed. + None, + /// zlib deflate (kitty `o=z`). + ZlibDeflate, +} + +/// One stored image, without its pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ImageDesc { + /// Kitty image id (`i=`). + pub id: u32, + /// Kitty image number (`I=`), zero when the child used none. + pub number: u32, + /// The stamp libghostty assigned when these pixels entered the storage. + /// A cache keyed on `(id, generation)` is never stale: the same id with + /// new pixels gets a new stamp. + pub generation: u64, + /// Width in pixels. + pub width: u32, + /// Height in pixels. + pub height: u32, + /// Pixel format. + pub format: PixelFormat, + /// Compression. + pub compression: Compression, + /// Length of the stored data in bytes. + pub len: usize, +} + +/// A stored image with its pixels copied out. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageBytes { + /// What the pixels describe. + pub desc: ImageDesc, + /// The bytes, exactly as libghostty stores them (`desc.format`). + pub data: Vec, +} + +/// The part of the image a placement shows, in image pixels, already resolved +/// (kitty's "0 means the whole dimension") and clamped to the image. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SourceRect { + /// Left edge in image pixels. + pub x: u32, + /// Top edge in image pixels. + pub y: u32, + /// Width in image pixels. + pub width: u32, + /// Height in image pixels. + pub height: u32, +} + +/// Where a virtual placement's placeholder cells are in the window that was +/// read. +/// +/// A placeholder cell names its own image row and column, so a window that +/// shows only part of an image still says exactly which part: `cell_row` / +/// `cell_col` are the image cell indices of the top-left visible cell, and +/// `origin_row` / `origin_col` are where the image's own cell (0, 0) would be +/// — negative when it has scrolled above the window. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PlaceholderRect { + /// Window row of the first placeholder row. + pub row: u16, + /// Window column of the first placeholder column. + pub col: u16, + /// Placeholder rows present in the window. + pub rows: u16, + /// Placeholder columns present in the window. + pub cols: u16, + /// Image cell row of the cell at (`row`, `col`). + pub cell_row: u16, + /// Image cell column of the cell at (`row`, `col`). + pub cell_col: u16, + /// Window row of the placement's own row 0; negative when scrolled above. + pub origin_row: i32, + /// Window column of the placement's own column 0. + pub origin_col: i32, +} + +/// Where a placement is, in the window it was read for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PlacementPosition { + /// Stored, but nothing of it is in this window: scrolled out, or a + /// virtual placement whose placeholder cells are elsewhere (or gone). + Offscreen, + /// A cursor-positioned placement (`a=p` without `U=1`). `row` is negative + /// when the top of the image has scrolled above the window. + Direct { + /// Window column of the top-left corner. + col: i32, + /// Window row of the top-left corner. + row: i32, + /// Cells wide. + cols: u32, + /// Cells tall. + rows: u32, + }, + /// A virtual placement (`U=1`), located by its placeholder cells. + Placeholder(PlaceholderRect), +} + +/// One placement: an image, where it is, and which part of it shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Placement { + /// The image it draws. + pub image_id: u32, + /// Kitty placement id (`p=`), zero when the child used none. With the + /// image id this is the placement's identity. + pub placement_id: u32, + /// Whether it is a virtual (Unicode placeholder) placement. + pub is_virtual: bool, + /// Kitty `z=`. + pub z: i32, + /// The image generation this placement was resolved against. + pub image_generation: u64, + /// The source crop, resolved and clamped. + pub source: SourceRect, + /// Pixel offset inside the first cell (`X=`, `Y=`). + pub cell_offset: (u32, u32), + /// Rendered size in pixels, after crop and aspect ratio. + pub pixel_size: (u32, u32), + /// Rendered size in cells. + pub cell_size: (u32, u32), + /// The raw `c=` and `r=` the child asked for; zero means "natural size". + pub requested_cells: (u32, u32), + /// Where it is in this window. + pub position: PlacementPosition, +} + +/// Everything a compositor needs for one frame. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GraphicsState { + /// Whether the protocol is on (a non-zero storage limit). + pub enabled: bool, + /// The storage generation. Unchanged means the images and the set of + /// placements are byte-for-byte what they were; geometry may still have + /// moved (scroll, resize), so a dirty frame still re-reads positions. + pub generation: u64, + /// The storage limit in bytes. + pub storage_bytes: u64, + /// The cell metrics the geometry was computed with, always a usable + /// size. + pub cell: CellSize, + /// Whether `cell` is what a client declared, or + /// [`CellSize::FALLBACK`] because nobody has. A consumer that draws + /// pixels should declare its own + /// ([`crate::handle::TerminalHandle::set_cell_size`]) rather than + /// trust a fallback: `c=`/`r=` placements are exact either way, but a + /// placement that left its size implicit is only as right as this. + pub cell_declared: bool, + /// Every image that has at least one placement, by id. + pub images: Vec, + /// Every placement, in libghostty's iteration order. + pub placements: Vec, +} + +impl Default for GraphicsState { + fn default() -> Self { + GraphicsState { + enabled: false, + generation: 0, + storage_bytes: 0, + cell: CellSize::FALLBACK, + cell_declared: false, + images: Vec::new(), + placements: Vec::new(), + } + } +} + +impl GraphicsState { + /// The description of `id`, if it is in this state. + pub fn image(&self, id: u32) -> Option<&ImageDesc> { + self.images.iter().find(|i| i.id == id) + } + + /// The placements of `id`. + pub fn placements_of(&self, id: u32) -> impl Iterator { + self.placements.iter().filter(move |p| p.image_id == id) + } + + /// Placements with anything in the window, in draw order (lowest `z` + /// first, ties in storage order). + pub fn visible(&self) -> Vec<&Placement> { + let mut v: Vec<&Placement> = self + .placements + .iter() + .filter(|p| p.position != PlacementPosition::Offscreen) + .collect(); + v.sort_by_key(|p| p.z); + v + } +} + +// --------------------------------------------------------------------------- +// Reading the storage +// --------------------------------------------------------------------------- + +/// Whether the protocol is enabled on `term`. +pub fn enabled(term: &Terminal) -> bool { + term.kitty_image_storage_limit().unwrap_or(0) > 0 +} + +/// The storage generation, or 0 when the storage is empty or disabled. +/// +/// Cheap: one read, no iteration. A caller that sees an unchanged generation +/// can skip [`read`]'s image work, but not its geometry work. +pub fn generation(term: &Terminal) -> u64 { + term.kitty_graphics() + .and_then(|g| g.generation()) + .unwrap_or(0) +} + +/// Read the whole state for the window starting `scroll_offset` rows above +/// the live viewport (0 = the live viewport), the same window +/// [`crate::snapshot::snapshot`] reads. +/// +/// Positions are relative to that window, so a grid and a state read with the +/// same offset line up cell for cell. +pub fn read(term: &Terminal, cell: CellSize, scroll_offset: usize) -> GraphicsState { + let storage_bytes = term.kitty_image_storage_limit().unwrap_or(0); + let mut state = GraphicsState { + enabled: storage_bytes > 0, + generation: 0, + storage_bytes, + cell: cell.or_fallback(), + cell_declared: cell.is_declared(), + images: Vec::new(), + placements: Vec::new(), + }; + if !state.enabled { + return state; + } + let Ok(graphics) = term.kitty_graphics() else { + return state; + }; + state.generation = graphics.generation().unwrap_or(0); + let Ok(mut iter) = gfx::PlacementIterator::new() else { + return state; + }; + let Ok(mut placements) = iter.update(&graphics) else { + return state; + }; + + // The buffer row this window starts at, the same one + // `crate::snapshot::snapshot` uses. Direct placements resolve in screen + // space, so this is what turns them into window rows. + let window_start = term + .scrollback_rows() + .unwrap_or(0) + .saturating_sub(scroll_offset) as i64; + + // The placeholder scan is one pass over the window and only happens once + // a virtual placement is actually there. + let mut placeholders: Option> = None; + + while let Some(p) = placements.next() { + let Ok(image_id) = p.image_id() else { continue }; + let Some(image) = graphics.image(image_id) else { + continue; + }; + let Some(desc) = describe(image_id, &image) else { + continue; + }; + let is_virtual = p.is_virtual().unwrap_or(false); + let placement_id = p.placement_id().unwrap_or(0); + let source = p + .source_rect(&image) + .map(|r| SourceRect { + x: r.x, + y: r.y, + width: r.width, + height: r.height, + }) + .unwrap_or(SourceRect { + x: 0, + y: 0, + width: desc.width, + height: desc.height, + }); + let pixel_size = p + .pixel_size(&image, term) + .map(|s| (s.width, s.height)) + .unwrap_or((0, 0)); + let cell_size = p + .grid_size(&image, term) + .map(|s| (s.cols, s.rows)) + .unwrap_or((0, 0)); + let position = if is_virtual { + let found = placeholders.get_or_insert_with(|| scan_placeholders(term, scroll_offset)); + // A placeholder cell names a placement by its underline colour, + // and a cell that carries none names the image's default + // placement. So an exact match wins, and a placement falls back + // to the cells that named no placement at all rather than + // reporting itself absent. + found + .iter() + .find(|(i, pl, _)| *i == image_id && *pl == placement_id) + .or_else(|| found.iter().find(|(i, pl, _)| *i == image_id && *pl == 0)) + .map(|(_, _, r)| PlacementPosition::Placeholder(*r)) + .unwrap_or(PlacementPosition::Offscreen) + } else { + // Not `viewport_pos`: that answers only for the live viewport and + // reports nothing for a placement above it, which would make a + // scrolled-back window lose exactly the images it should show. + // The placement's own rectangle resolves in screen space, so it + // answers for any window — including a negative row for a + // placement whose top has scrolled above this one. + match screen_origin(&p, &image, term) { + Some((col, screen_row)) => PlacementPosition::Direct { + col, + row: (screen_row - window_start) as i32, + cols: cell_size.0, + rows: cell_size.1, + }, + None => PlacementPosition::Offscreen, + } + }; + if !state.images.iter().any(|i| i.id == desc.id) { + state.images.push(desc); + } + state.placements.push(Placement { + image_id, + placement_id, + is_virtual, + z: p.z().unwrap_or(0), + image_generation: desc.generation, + source, + cell_offset: (p.x_offset().unwrap_or(0), p.y_offset().unwrap_or(0)), + pixel_size, + cell_size, + requested_cells: (p.columns().unwrap_or(0), p.rows().unwrap_or(0)), + position, + }); + } + state +} + +/// The top-left corner of a cursor-positioned placement, as a column and an +/// absolute buffer (screen-space) row. +/// +/// The placement's own rectangle is the only answer that works for a window +/// other than the live viewport: `PlacementIteration::viewport_pos` is +/// defined against the viewport and reports nothing for a placement above +/// it, which is precisely the placement a scrolled-back reader wants. +fn screen_origin( + p: &gfx::PlacementIteration<'_, '_>, + image: &gfx::Image<'_>, + term: &Terminal, +) -> Option<(i32, i64)> { + let rect = p.rect(image, term).ok()?; + let point = term + .point_from_grid_ref(&rect.start(), PointSpace::Screen) + .ok()??; + Some((point.x as i32, point.y as i64)) +} + +/// The pixels of `id`, copied out of the storage. `None` when the image is +/// gone, so a caller that raced a delete learns it here. +pub fn image_bytes(term: &Terminal, id: u32) -> Option { + let graphics = term.kitty_graphics().ok()?; + let image = graphics.image(id)?; + let desc = describe(id, &image)?; + let data = image.data().ok()?; + Some(ImageBytes { + desc, + data: data.to_vec(), + }) +} + +fn describe(id: u32, image: &gfx::Image<'_>) -> Option { + let width = image.width().ok()?; + let height = image.height().ok()?; + if width == 0 || height == 0 { + return None; + } + let format = match image.format().ok()? { + gfx::ImageFormat::Rgb => PixelFormat::Rgb, + gfx::ImageFormat::Rgba => PixelFormat::Rgba, + gfx::ImageFormat::Png => PixelFormat::Png, + gfx::ImageFormat::Gray => PixelFormat::Gray, + gfx::ImageFormat::GrayAlpha => PixelFormat::GrayAlpha, + _ => return None, + }; + let compression = match image.compression().ok()? { + gfx::Compression::ZlibDeflate => Compression::ZlibDeflate, + _ => Compression::None, + }; + Some(ImageDesc { + id, + number: image.number().unwrap_or(0), + generation: image.generation().unwrap_or(0), + width, + height, + format, + compression, + len: image.data().map(|d| d.len()).unwrap_or(0), + }) +} + +// --------------------------------------------------------------------------- +// Placeholder cells +// --------------------------------------------------------------------------- + +/// The row/column diacritics, in index order (kitty's +/// `gen/rowcolumn-diacritics.txt`). Sorted, so a codepoint's index is a +/// binary search. +const ROWCOLUMN_DIACRITICS: [u32; 297] = [ + 0x0305, 0x030d, 0x030e, 0x0310, 0x0312, 0x033d, 0x033e, 0x033f, 0x0346, 0x034a, 0x034b, + 0x034c, 0x0350, 0x0351, 0x0352, 0x0357, 0x035b, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, 0x0483, 0x0484, 0x0485, + 0x0486, 0x0487, 0x0592, 0x0593, 0x0594, 0x0595, 0x0597, 0x0598, 0x0599, 0x059c, 0x059d, + 0x059e, 0x059f, 0x05a0, 0x05a1, 0x05a8, 0x05a9, 0x05ab, 0x05ac, 0x05af, 0x05c4, 0x0610, + 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, 0x0657, 0x0658, 0x0659, 0x065a, + 0x065b, 0x065d, 0x065e, 0x06d6, 0x06d7, 0x06d8, 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06df, + 0x06e0, 0x06e1, 0x06e2, 0x06e4, 0x06e7, 0x06e8, 0x06eb, 0x06ec, 0x0730, 0x0732, 0x0733, + 0x0735, 0x0736, 0x073a, 0x073d, 0x073f, 0x0740, 0x0741, 0x0743, 0x0745, 0x0747, 0x0749, + 0x074a, 0x07eb, 0x07ec, 0x07ed, 0x07ee, 0x07ef, 0x07f0, 0x07f1, 0x07f3, 0x0816, 0x0817, + 0x0818, 0x0819, 0x081b, 0x081c, 0x081d, 0x081e, 0x081f, 0x0820, 0x0821, 0x0822, 0x0823, + 0x0825, 0x0826, 0x0827, 0x0829, 0x082a, 0x082b, 0x082c, 0x082d, 0x0951, 0x0953, 0x0954, + 0x0f82, 0x0f83, 0x0f86, 0x0f87, 0x135d, 0x135e, 0x135f, 0x17dd, 0x193a, 0x1a17, 0x1a75, + 0x1a76, 0x1a77, 0x1a78, 0x1a79, 0x1a7a, 0x1a7b, 0x1a7c, 0x1b6b, 0x1b6d, 0x1b6e, 0x1b6f, + 0x1b70, 0x1b71, 0x1b72, 0x1b73, 0x1cd0, 0x1cd1, 0x1cd2, 0x1cda, 0x1cdb, 0x1ce0, 0x1dc0, + 0x1dc1, 0x1dc3, 0x1dc4, 0x1dc5, 0x1dc6, 0x1dc7, 0x1dc8, 0x1dc9, 0x1dcb, 0x1dcc, 0x1dd1, + 0x1dd2, 0x1dd3, 0x1dd4, 0x1dd5, 0x1dd6, 0x1dd7, 0x1dd8, 0x1dd9, 0x1dda, 0x1ddb, 0x1ddc, + 0x1ddd, 0x1dde, 0x1ddf, 0x1de0, 0x1de1, 0x1de2, 0x1de3, 0x1de4, 0x1de5, 0x1de6, 0x1dfe, + 0x20d0, 0x20d1, 0x20d4, 0x20d5, 0x20d6, 0x20d7, 0x20db, 0x20dc, 0x20e1, 0x20e7, 0x20e9, + 0x20f0, 0x2cef, 0x2cf0, 0x2cf1, 0x2de0, 0x2de1, 0x2de2, 0x2de3, 0x2de4, 0x2de5, 0x2de6, + 0x2de7, 0x2de8, 0x2de9, 0x2dea, 0x2deb, 0x2dec, 0x2ded, 0x2dee, 0x2def, 0x2df0, 0x2df1, + 0x2df2, 0x2df3, 0x2df4, 0x2df5, 0x2df6, 0x2df7, 0x2df8, 0x2df9, 0x2dfa, 0x2dfb, 0x2dfc, + 0x2dfd, 0x2dfe, 0x2dff, 0xa66f, 0xa67c, 0xa67d, 0xa6f0, 0xa6f1, 0xa8e0, 0xa8e1, 0xa8e2, + 0xa8e3, 0xa8e4, 0xa8e5, 0xa8e6, 0xa8e7, 0xa8e8, 0xa8e9, 0xa8ea, 0xa8eb, 0xa8ec, 0xa8ed, + 0xa8ee, 0xa8ef, 0xa8f0, 0xa8f1, 0xaab0, 0xaab2, 0xaab3, 0xaab7, 0xaab8, 0xaabe, 0xaabf, + 0xaac1, 0xfe20, 0xfe21, 0xfe22, 0xfe23, 0xfe24, 0xfe25, 0xfe26, 0x10a0f, 0x10a38, 0x1d185, + 0x1d186, 0x1d187, 0x1d188, 0x1d189, 0x1d1aa, 0x1d1ab, 0x1d1ac, 0x1d1ad, 0x1d242, 0x1d243, + 0x1d244, +]; + +/// The index this diacritic encodes, if it is one. +fn diacritic_index(c: char) -> Option { + ROWCOLUMN_DIACRITICS + .binary_search(&(c as u32)) + .ok() + .map(|i| i as u16) +} + +/// One accumulating placeholder region. +struct Accum { + image_id: u32, + placement_id: u32, + row: u16, + col: u16, + last_row: u16, + last_col: u16, + cell_row: u16, + cell_col: u16, +} + +/// Walk the window once and collect, per (image id, placement id), the +/// placeholder cells that are in it. +/// +/// The scan is bounded by the window: rows x cols cell reads, never the whole +/// scrollback. Kitty's inheritance rules live in [`placeholder_cell`]. +fn scan_placeholders(term: &Terminal, scroll_offset: usize) -> Vec<(u32, u32, PlaceholderRect)> { + let rows_n = term.rows().unwrap_or(0); + let cols = term.cols().unwrap_or(0); + let base_y = term.scrollback_rows().unwrap_or(0); + let len = term.total_rows().unwrap_or(rows_n as usize); + let start = base_y.saturating_sub(scroll_offset); + let live = start == base_y; + + let mut acc: Vec = Vec::new(); + for r in 0..rows_n { + if start + r as usize >= len { + break; + } + // The cell to the left, when it was a placeholder: what an omitted + // diacritic inherits from. + let mut prev: Option = None; + for x in 0..cols { + let point = if live { + Point::Active(PointCoordinate { x, y: r as u32 }) + } else { + Point::Screen(PointCoordinate { + x, + y: (start + r as usize) as u32, + }) + }; + let Some(cell) = placeholder_cell(term, point, prev) else { + prev = None; + continue; + }; + prev = Some(cell); + match acc + .iter_mut() + .find(|a| a.image_id == cell.image_id && a.placement_id == cell.placement_id) + { + Some(a) => { + a.last_row = a.last_row.max(r); + a.last_col = a.last_col.max(x); + a.row = a.row.min(r); + if x < a.col { + a.col = x; + a.cell_col = cell.cell_col; + } + if r < a.row || (r == a.row && x == a.col) { + a.cell_row = cell.cell_row; + } + } + None => acc.push(Accum { + image_id: cell.image_id, + placement_id: cell.placement_id, + row: r, + col: x, + last_row: r, + last_col: x, + cell_row: cell.cell_row, + cell_col: cell.cell_col, + }), + } + } + } + acc.into_iter() + .map(|a| { + ( + a.image_id, + a.placement_id, + PlaceholderRect { + row: a.row, + col: a.col, + rows: a.last_row - a.row + 1, + cols: a.last_col - a.col + 1, + cell_row: a.cell_row, + cell_col: a.cell_col, + origin_row: a.row as i32 - a.cell_row as i32, + origin_col: a.col as i32 - a.cell_col as i32, + }, + ) + }) + .collect() +} + +/// What one placeholder cell says. Also what the cell to its right inherits +/// when it leaves a diacritic out. +#[derive(Clone, Copy)] +struct PlaceholderCell { + image_id: u32, + placement_id: u32, + cell_row: u16, + cell_col: u16, + /// The high byte of the image id, carried separately because a + /// continuation cell inherits it rather than restating it. + id_high: u16, +} + +/// Decode the cell at `point` as a placeholder, or `None` when it is not one. +/// +/// The image id is the cell's foreground colour — 24 bits in truecolor, or a +/// palette index in 256-colour mode, both of which kitty allows — plus an +/// optional high byte from a third diacritic. The placement id is the +/// underline colour, read the same way; no underline colour means the +/// image's default placement. +/// +/// Kitty's inheritance rules apply to a cell that omits diacritics: it takes +/// the row, the column plus one, and the image-id high byte from the cell to +/// its left. That is what makes the compact form (`U+10EEEE` repeated with no +/// diacritics after the first cell) decode to a real rectangle. +fn placeholder_cell( + term: &Terminal, + point: Point, + prev: Option, +) -> Option { + let g = term.grid_ref(point).ok()?; + let mut buf = [char::default(); 8]; + let n = g.graphemes(&mut buf).ok()?; + let chars = &buf[..n]; + if chars.first() != Some(&PLACEHOLDER) { + return None; + } + let style = g.style().ok()?; + let base = color_id(style.fg_color)?; + let placement_id = color_id(style.underline_color).unwrap_or(0); + // Only a cell that names the same image and placement can be continued. + let left = prev.filter(|p| p.image_id & 0x00ff_ffff == base && p.placement_id == placement_id); + let mut diacritics = chars[1..].iter().filter_map(|&c| diacritic_index(c)); + let cell_row = diacritics + .next() + .or_else(|| left.map(|p| p.cell_row)) + .unwrap_or(0); + let cell_col = diacritics + .next() + .or_else(|| left.map(|p| p.cell_col + 1)) + .unwrap_or(0); + let id_high = diacritics + .next() + .or_else(|| left.map(|p| p.id_high)) + .unwrap_or(0); + Some(PlaceholderCell { + image_id: ((id_high as u32) << 24) | base, + placement_id, + cell_row, + cell_col, + id_high, + }) +} + +/// The 24-bit id a placeholder colour names: truecolor components, or a +/// palette index (kitty's 256-colour form, which limits ids to 8 bits). +/// `None` for a default colour, which names nothing. +fn color_id(color: StyleColor) -> Option { + match color { + StyleColor::Rgb(c) => Some(((c.r as u32) << 16) | ((c.g as u32) << 8) | c.b as u32), + StyleColor::Palette(i) => Some(i.0 as u32), + StyleColor::None => None, + } +} + +// --------------------------------------------------------------------------- +// Replay +// --------------------------------------------------------------------------- + +/// Put the storage back on the wire: one kitty transmission per image, then +/// one placement command per placement, in the protocol the child used. +/// +/// This is what makes graphics survive the ATTACH/PEEK replay. libghostty's +/// VT serialization keeps the placeholder cells (ordinary text with a +/// foreground colour) but not the images or the placements, so a client that +/// only replayed the VT body would hold placeholders naming images it does +/// not have. +/// +/// Everything the storage holds is emitted. The bound on a replay is the +/// bound on the state — [`MAX_STORAGE_BYTES`], which +/// [`crate::actor::TerminalActor::enable_graphics`] clamps the storage limit +/// to — so an image the terminal accepted can always be replayed. A second, +/// smaller replay cap would mean a supported image that a late client can +/// never receive, which is exactly the state this whole module exists to +/// prevent. +/// +/// The block carries no cursor movement of its own beyond a save/restore +/// around a cursor-positioned placement, so it can be appended to a replay +/// payload without disturbing it. +pub fn replay(term: &Terminal, cell: CellSize) -> String { + let cell = cell.or_fallback(); + let state = read(term, cell, 0); + if state.placements.is_empty() { + return String::new(); + } + let rows = term.rows().unwrap_or(0) as i32; + let cols = term.cols().unwrap_or(0) as i32; + + let mut ids: Vec = state.images.iter().map(|i| i.id).collect(); + ids.sort_unstable(); + let mut out = String::new(); + let mut sent: Vec = Vec::with_capacity(ids.len()); + for id in ids { + let Some(bytes) = image_bytes(term, id) else { + continue; + }; + let Some(f) = bytes.desc.format.kitty_format() else { + // Grayscale has no kitty `f=`; nothing correct to emit. + continue; + }; + if bytes.data.is_empty() { + continue; + } + transmit(&mut out, &bytes, f); + sent.push(id); + } + for p in &state.placements { + if !sent.contains(&p.image_id) { + continue; + } + if p.is_virtual { + // A virtual placement's command carries no position — its cells + // do, and they are in the VT body wherever they are, including + // in the scrollback. So where it currently shows has nothing to + // do with whether it can be replayed. + place_virtual(&mut out, p); + continue; + } + // A placement the child put at the cursor is restored by putting the + // cursor back, so it needs a cell in the active area. One whose top + // has scrolled above it is still partly on screen: anchor it at row + // 0 and advance the crop by the rows that are gone, which is what + // the source terminal is showing. + let PlacementPosition::Direct { col, row, .. } = p.position else { + continue; + }; + let clipped = (-row).max(0); + if row >= rows || col < 0 || col >= cols || clipped >= p.cell_size.1 as i32 { + continue; + } + place_direct(&mut out, p, col, row.max(0), clipped as u32, cell); + } + out +} + +/// `ESC _G a=t,...; ESC \`, chunked at 4096 base64 bytes (the +/// protocol's limit, and what OMP emits). +fn transmit(out: &mut String, image: &ImageBytes, f: u32) { + let mut params = format!("a=t,q=2,i={},f={}", image.desc.id, f); + if image.desc.format.is_raw() { + params.push_str(&format!(",s={},v={}", image.desc.width, image.desc.height)); + } + if image.desc.compression == Compression::ZlibDeflate { + params.push_str(",o=z"); + } + let payload = base64(&image.data); + let mut chunks = payload.as_bytes().chunks(4096).peekable(); + let mut first = true; + while let Some(chunk) = chunks.next() { + let more = chunks.peek().is_some(); + let chunk = std::str::from_utf8(chunk).unwrap_or_default(); + if first { + out.push_str("\x1b_G"); + out.push_str(¶ms); + if more { + out.push_str(",m=1"); + } + out.push(';'); + first = false; + } else { + out.push_str("\x1b_Gq=2,m="); + out.push(if more { '1' } else { '0' }); + out.push(';'); + } + out.push_str(chunk); + out.push_str("\x1b\\"); + } +} + +/// `ESC _G a=p,U=1,... ESC \`: the virtual placement the placeholder cells in +/// the VT body refer to. +fn place_virtual(out: &mut String, p: &Placement) { + out.push_str("\x1b_Ga=p,U=1,q=2"); + out.push_str(&placement_params(p)); + out.push_str("\x1b\\"); +} + +/// A cursor-positioned placement: save the cursor, put it on the placement's +/// cell, place, restore. +/// +/// `clipped_rows` is how many of its rows have scrolled above the active +/// area; the crop starts that many cells lower so what is emitted is what is +/// still on screen. +fn place_direct(out: &mut String, p: &Placement, col: i32, row: i32, clipped_rows: u32, cell: CellSize) { + out.push_str(&format!("\x1b7\x1b[{};{}H\x1b_Ga=p,q=2", row + 1, col + 1)); + out.push_str(&placement_params(&clip_top(p, clipped_rows, cell))); + out.push_str("\x1b\\\x1b8"); +} + +/// The same placement with its first `rows` cells of image cut off: the crop +/// moves down and shrinks, and the requested row count follows. +fn clip_top(p: &Placement, rows: u32, cell: CellSize) -> Placement { + if rows == 0 { + return *p; + } + let cut = (rows * cell.height.max(1)).min(p.source.height); + let mut clipped = *p; + clipped.source.y += cut; + clipped.source.height -= cut; + clipped.requested_cells.1 = p.requested_cells.1.saturating_sub(rows); + clipped +} + +/// The parameters both placement forms share. +/// +/// The source rectangle is emitted in full, always: `w=`/`h=` default to 0, +/// which the protocol reads as "the whole image", so omitting them turns a +/// cropped placement into an uncropped one squeezed into the cropped +/// placement's cell box — the wrong pixels, at the right size. +fn placement_params(p: &Placement) -> String { + let mut s = format!(",i={}", p.image_id); + if p.placement_id != 0 { + s.push_str(&format!(",p={}", p.placement_id)); + } + if p.requested_cells.0 != 0 { + s.push_str(&format!(",c={}", p.requested_cells.0)); + } + if p.requested_cells.1 != 0 { + s.push_str(&format!(",r={}", p.requested_cells.1)); + } + s.push_str(&format!( + ",x={},y={},w={},h={}", + p.source.x, p.source.y, p.source.width, p.source.height + )); + if p.cell_offset.0 != 0 { + s.push_str(&format!(",X={}", p.cell_offset.0)); + } + if p.cell_offset.1 != 0 { + s.push_str(&format!(",Y={}", p.cell_offset.1)); + } + if p.z != 0 { + s.push_str(&format!(",z={}", p.z)); + } + s +} + +/// Standard base64, no line breaks: what the protocol's payload is. +fn base64(data: &[u8]) -> String { + const ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let b = [ + chunk[0], + chunk.get(1).copied().unwrap_or(0), + chunk.get(2).copied().unwrap_or(0), + ]; + let n = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | b[2] as u32; + out.push(ALPHABET[(n >> 18) as usize & 63] as char); + out.push(ALPHABET[(n >> 12) as usize & 63] as char); + out.push(if chunk.len() > 1 { + ALPHABET[(n >> 6) as usize & 63] as char + } else { + '=' + }); + out.push(if chunk.len() > 2 { + ALPHABET[n as usize & 63] as char + } else { + '=' + }); + } + out +} + +// --------------------------------------------------------------------------- +// PNG +// --------------------------------------------------------------------------- + +/// Largest image a `f=100` transmission may decode to. Storage is bounded +/// too, but the decode buffer exists before storage sees it. +const MAX_DECODED_PNG_BYTES: usize = 64 * 1024 * 1024; + +/// The decoder libghostty calls for `f=100`. Without one it rejects PNG +/// transmissions, which is what most senders (OMP included) use. +#[derive(Default)] +struct PngDecoder { + buf: Vec, +} + +impl gfx::DecodePng for PngDecoder { + fn decode_png<'alloc>( + &mut self, + alloc: &'alloc Allocator<'_>, + data: &[u8], + ) -> Option> { + let mut decoder = png::Decoder::new(std::io::Cursor::new(data)); + // Normalize to 8 bits per channel: expand a palette, expand a low + // bit depth, strip 16-bit. It does NOT make everything RGBA — + // `Transformations::ALPHA` only adds alpha to a paletted image, and + // there is no grayscale-to-RGB rule at all — so a grayscale PNG + // arrives as `Grayscale` or `GrayscaleAlpha` and the expansion to + // RGBA happens below. Rejecting those instead (which is what + // checking for `Rgba` did) silently drops every monochrome plot and + // optipng-converted asset a child sends. + decoder.set_transformations(png::Transformations::normalize_to_color8()); + let mut reader = decoder.read_info().ok()?; + let size = reader.output_buffer_size()?; + let (width, height) = reader.info().size(); + let rgba_len = (width as usize) + .checked_mul(height as usize)? + .checked_mul(4)?; + if size == 0 || rgba_len == 0 || rgba_len > MAX_DECODED_PNG_BYTES { + return None; + } + self.buf.clear(); + self.buf.resize(size, 0); + let info = reader.next_frame(&mut self.buf).ok()?; + if info.bit_depth != png::BitDepth::Eight { + return None; + } + let src = &self.buf[..info.buffer_size()]; + let mut bytes = Bytes::new_with_alloc(alloc, rgba_len).ok()?; + match info.color_type { + png::ColorType::Rgba => bytes.copy_from_slice(src), + png::ColorType::Rgb => expand(src, 3, &mut bytes, |px, out| { + out.copy_from_slice(&[px[0], px[1], px[2], 0xff]) + }), + png::ColorType::GrayscaleAlpha => expand(src, 2, &mut bytes, |px, out| { + out.copy_from_slice(&[px[0], px[0], px[0], px[1]]) + }), + png::ColorType::Grayscale => expand(src, 1, &mut bytes, |px, out| { + out.copy_from_slice(&[px[0], px[0], px[0], 0xff]) + }), + // `normalize_to_color8` leaves no other 8-bit output type. + _ => return None, + } + Some(gfx::DecodedImage { + width: info.width, + height: info.height, + data: bytes, + }) + } +} + +/// Widen `src`, `stride` bytes per pixel, into 8-bit RGBA. +fn expand(src: &[u8], stride: usize, out: &mut [u8], px: impl Fn(&[u8], &mut [u8])) { + for (i, chunk) in src.chunks_exact(stride).enumerate() { + let Some(dst) = out.get_mut(i * 4..i * 4 + 4) else { + return; + }; + px(chunk, dst); + } +} + +/// Install the PNG decoder for this thread's terminals. Idempotent; the +/// decoder is thread-local in libghostty, so it must run on the thread that +/// owns the terminal. +pub fn install_png_decoder() -> bool { + gfx::set_png_decoder(Some(Box::new(PngDecoder::default()))).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn base64_matches_the_reference_alphabet() { + assert_eq!(base64(b""), ""); + assert_eq!(base64(b"f"), "Zg=="); + assert_eq!(base64(b"fo"), "Zm8="); + assert_eq!(base64(b"foo"), "Zm9v"); + assert_eq!(base64(b"foobar"), "Zm9vYmFy"); + assert_eq!(base64(&[0xff, 0xff, 0xff]), "////"); + assert_eq!(base64(&[0xfb, 0xf0]), "+/A="); + } + + #[test] + fn diacritics_are_an_index_table() { + assert_eq!(diacritic_index('\u{305}'), Some(0)); + assert_eq!(diacritic_index('\u{30d}'), Some(1)); + assert_eq!(diacritic_index('\u{1d244}'), Some(296)); + assert_eq!(diacritic_index('a'), None); + } +} diff --git a/crates/pty-terminal/src/handle.rs b/crates/pty-terminal/src/handle.rs index 67ef57a..2af9cb5 100644 --- a/crates/pty-terminal/src/handle.rs +++ b/crates/pty-terminal/src/handle.rs @@ -20,11 +20,14 @@ use std::time::{Duration, Instant}; use portable_pty::{CommandBuilder, MasterPty, PtySize, native_pty_system}; use pty_core::protocol::{ - MessageType, Packet, PacketReader, decode_exit, decode_size, encode_attach, encode_data, encode_peek, - encode_detach, encode_resize, + MessageType, Packet, PacketReader, decode_exit, decode_size, encode_attach, + encode_attach_with_cell, encode_data, encode_detach, encode_peek, encode_resize, + encode_resize_with_cell, }; use crate::actor::{Modes, Notification, Range, TerminalActor, TerminalEvent}; +use crate::graphics::{CellSize, GraphicsOptions, GraphicsState, ImageBytes}; +use crate::input::{KeyEvent, MouseEvent}; use crate::serialize::SerializeOpts; use crate::snapshot::CellGrid; @@ -66,6 +69,10 @@ pub struct SpawnOptions { pub env: Vec<(String, String)>, /// Scrollback lines. pub scrollback: usize, + /// Kitty graphics: a bounded image storage and the cell metrics + /// placements are measured in. `None` leaves the protocol off, so a + /// child cannot make the terminal hold images nobody reads. + pub graphics: Option, } impl Default for SpawnOptions { @@ -76,6 +83,7 @@ impl Default for SpawnOptions { cwd: None, env: Vec::new(), scrollback: 0, + graphics: None, } } } @@ -92,6 +100,9 @@ pub struct AttachOptions { pub readonly: bool, /// Scrollback lines kept locally. pub scrollback: usize, + /// Kitty graphics, as in [`SpawnOptions::graphics`]. An attached client + /// needs it to hold the images the daemon's replay carries. + pub graphics: Option, } impl Default for AttachOptions { @@ -101,6 +112,7 @@ impl Default for AttachOptions { cols: 80, readonly: false, scrollback: 0, + graphics: None, } } } @@ -121,6 +133,19 @@ pub enum HandleEvent { Exited(i32), /// OSC 9 / 99 / 777. Notification(Notification), + /// The image storage changed (transmit, placement, or delete): the new + /// generation. + Graphics(u64), + /// The socket to the session daemon went away. The handle is still + /// alive and [`TerminalHandle::reconnect`] can be called; the session + /// itself did not exit — that is [`HandleEvent::Exited`]. A consumer + /// with a detached state hears this, rather than polling + /// [`TerminalHandle::connected`]. + Disconnected, + /// A new socket is up under this attempt id, after + /// [`TerminalHandle::reconnect`]. The screen follows as the attempt's + /// first SCREEN, so `Connected` means "reattached", not "ready". + Connected(AttemptId), } enum Msg { @@ -133,6 +158,22 @@ enum Msg { Snapshot { offset: usize, reply: Sender<(u64, CellGrid)> }, Plain { range: Range, reply: Sender }, Serialize { opts: SerializeOpts, reply: Sender }, + Graphics { offset: usize, reply: Sender }, + ImageBytes { id: u32, reply: Sender> }, + ClearGraphics, + CellSize(CellSize), + Key(KeyEvent), + Mouse(MouseEvent), + Focus(bool), + Paste(String), + EncodeKey { + ev: KeyEvent, + reply: Sender>, + }, + EncodeMouse { + ev: MouseEvent, + reply: Sender>>, + }, SetPalette(Vec<(u8, u8, u8)>), Reconnect { reply: Sender> }, Close, @@ -154,6 +195,9 @@ struct State { base_y: usize, len: usize, scrollback: usize, + /// The last published image-storage generation, so a consumer can tell + /// an image change from any other dirty frame without asking the actor. + graphics_generation: u64, snap_cache: Option<(u64, CellGrid)>, } @@ -203,7 +247,8 @@ impl Core { /// fan out events. fn publish(&mut self) { let events = self.actor.take_events(); - let rev = { + let generation = self.actor.graphics_generation(); + let (rev, graphics_changed) = { let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner()); st.rev += 1; st.snap_cache = None; @@ -214,7 +259,9 @@ impl Core { st.title = self.actor.title(); st.base_y = self.actor.base_y(); st.len = self.actor.buffer_length(); - st.rev + let changed = st.graphics_generation != generation; + st.graphics_generation = generation; + (st.rev, changed) }; self.shared.cv.notify_all(); for ev in events { @@ -226,6 +273,9 @@ impl Core { }; self.shared.emit(ev); } + if graphics_changed { + self.shared.emit(HandleEvent::Graphics(generation)); + } self.shared.emit(HandleEvent::Dirty(rev)); } @@ -293,6 +343,7 @@ impl Core { *stream = None; } self.shared.cv.notify_all(); + self.shared.emit(HandleEvent::Disconnected); } fn input(&mut self, data: &[u8]) { @@ -330,7 +381,10 @@ impl Core { return; } if let Some(s) = stream { - let _ = s.write_all(&encode_resize(rows, cols)); + let _ = s.write_all(&match cell_pixels(opts) { + Some((w, h)) => encode_resize_with_cell(rows, cols, w, h), + None => encode_resize(rows, cols), + }); let _ = s.flush(); } // Applied locally as well: a daemon that speaks GEOMETRY will @@ -372,6 +426,7 @@ impl Core { st.connected = true; } self.shared.cv.notify_all(); + self.shared.emit(HandleEvent::Connected(self.attempt)); Ok(()) } @@ -408,6 +463,68 @@ impl Core { Msg::Serialize { opts, reply } => { let _ = reply.send(self.actor.serialize(opts)); } + Msg::Graphics { offset, reply } => { + let _ = reply.send(self.actor.graphics_state(offset)); + } + Msg::ImageBytes { id, reply } => { + let _ = reply.send(self.actor.image_bytes(id)); + } + Msg::ClearGraphics => { + self.actor.clear_graphics(); + self.publish(); + } + Msg::CellSize(cell) => { + self.actor.set_cell_size(cell); + if let Backend::Attach { stream, opts, .. } = &mut self.backend + && !opts.readonly + && let Some(s) = stream + { + // The daemon holds the session's terminal, so it needs + // the metrics too: its own replay answers geometry for + // every other client. + if let Some(g) = &mut opts.graphics { + g.cell = cell; + } + if let Some((w, h)) = cell_pixels(opts) { + let _ = s.write_all(&encode_resize_with_cell( + self.actor.rows(), + self.actor.cols(), + w, + h, + )); + let _ = s.flush(); + } + } + self.publish(); + } + Msg::Key(ev) => { + let bytes = self.actor.encode_key(&ev); + if !bytes.is_empty() { + self.input(&bytes); + } + } + Msg::Mouse(ev) => { + if let Some(bytes) = self.actor.encode_mouse(&ev) { + self.input(&bytes); + } + } + Msg::Focus(gained) => { + if let Some(bytes) = self.actor.encode_focus(gained) { + self.input(&bytes); + } + } + Msg::Paste(text) => { + let bytes = self.actor.encode_paste(&text); + if !bytes.is_empty() { + self.input(&bytes); + } + } + Msg::EncodeKey { ev, reply } => { + let _ = reply.send(self.actor.encode_key(&ev)); + } + Msg::EncodeMouse { ev, reply } => { + let _ = reply.send(self.actor.encode_mouse(&ev)); + } Msg::SetPalette(colors) => { self.actor.set_palette(&colors); self.publish(); @@ -446,6 +563,15 @@ fn run(mut core: Core, rx: Receiver) { core.shutdown(); } +/// The cell pixel size this client declares to the daemon, if any. Only a +/// client that asked for graphics has a reason to: the metrics exist so the +/// session can answer geometry for a placement that left its size implicit. +fn cell_pixels(opts: &AttachOptions) -> Option<(u16, u16)> { + let cell = opts.graphics?.cell; + cell.is_declared() + .then(|| (cell.width.min(u16::MAX as u32) as u16, cell.height.min(u16::MAX as u32) as u16)) +} + /// Connect to the daemon, send ATTACH, and start a reader thread that tags /// every packet with `attempt`. fn connect_and_attach( @@ -460,7 +586,10 @@ fn connect_and_attach( let hello = if opts.readonly { encode_peek(false, false) } else { - encode_attach(opts.rows, opts.cols) + match cell_pixels(opts) { + Some((w, h)) => encode_attach_with_cell(opts.rows, opts.cols, w, h), + None => encode_attach(opts.rows, opts.cols), + } }; (&stream).write_all(&hello)?; (&stream).flush()?; @@ -499,6 +628,22 @@ fn exit_code(status: portable_pty::ExitStatus) -> i32 { } } +/// The actor both constructors build. Graphics have to be turned on here, +/// on the actor thread: libghostty's PNG decoder is thread-local and the +/// terminal is `!Send`. +fn new_actor( + rows: u16, + cols: u16, + scrollback: usize, + graphics: Option, +) -> TerminalActor { + let mut actor = TerminalActor::new(rows, cols, scrollback); + if let Some(opts) = graphics { + actor.enable_graphics(opts); + } + actor +} + /// A live terminal you can write to, resize, and read typed cells from. /// Cheap to share (`Send + Sync`); every method is non-blocking except the /// explicit waits and the reads that must ask the actor thread. @@ -587,11 +732,12 @@ impl TerminalHandle { }); } - let (rows, cols, scrollback) = (opts.rows, opts.cols, opts.scrollback); + let (rows, cols, scrollback, gfx) = + (opts.rows, opts.cols, opts.scrollback, opts.graphics); let core_shared = shared.clone(); std::thread::spawn(move || { let core = Core { - actor: TerminalActor::new(rows, cols, scrollback), + actor: new_actor(rows, cols, scrollback, gfx), attempt, shared: core_shared, backend: Backend::Spawn { master, writer }, @@ -634,7 +780,7 @@ impl TerminalHandle { let core_tx = tx.clone(); std::thread::spawn(move || { let core = Core { - actor: TerminalActor::new(opts.rows, opts.cols, opts.scrollback), + actor: new_actor(opts.rows, opts.cols, opts.scrollback, opts.graphics), attempt, shared: core_shared, backend: Backend::Attach { @@ -801,6 +947,138 @@ impl TerminalHandle { reply_rx.recv().unwrap_or_default() } + /// The kitty graphics state for the window `scroll_offset` rows above + /// the live viewport — the same window [`TerminalHandle::snapshot`] + /// reads, so a grid and a graphics state taken with the same offset line + /// up cell for cell. + /// + /// Empty (and `enabled: false`) when the handle was built without + /// [`SpawnOptions::graphics`] / [`AttachOptions::graphics`]. + pub fn graphics(&self, scroll_offset: usize) -> GraphicsState { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::Graphics { + offset: scroll_offset, + reply: reply_tx, + }) + .is_err() + { + return GraphicsState::default(); + } + reply_rx.recv().unwrap_or_default() + } + + /// The pixels of one image. `None` when it is not stored (any more). + /// Cache the result on [`crate::graphics::ImageDesc::generation`]: it + /// changes whenever the pixels behind an id do. + pub fn image_bytes(&self, id: u32) -> Option { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::ImageBytes { id, reply: reply_tx }) + .is_err() + { + return None; + } + reply_rx.recv().unwrap_or(None) + } + + /// The last published image-storage generation, without asking the + /// actor. Unchanged means the images and the set of placements are + /// identical; placement geometry can still have moved, so a dirty frame + /// still re-reads [`TerminalHandle::graphics`]. + pub fn graphics_generation(&self) -> u64 { + self.state(|st| st.graphics_generation) + } + + /// Drop every image and placement, keeping the protocol on: what a pane + /// does when it closes or is reused for something else. + pub fn clear_graphics(&self) { + let _ = self.tx.send(Msg::ClearGraphics); + } + + /// Declare how big a cell is on the surface that draws this terminal. + /// + /// Only the client knows: the metrics come from its font, on its host. An + /// attached handle also tells the daemon (a RESIZE carrying the cell + /// size), because the session's own terminal is what answers geometry for + /// every other client and for the replay. Nothing about the bytes + /// changes; a placement that named `c=`/`r=` is unaffected, and one that + /// left its size implicit now gets the cell extent this surface will + /// actually draw. + /// + /// Until someone declares it, geometry uses + /// [`crate::graphics::CellSize::FALLBACK`] and + /// [`crate::graphics::GraphicsState::cell_declared`] is false. + pub fn set_cell_size(&self, width: u32, height: u32) { + let _ = self.tx.send(Msg::CellSize(CellSize { width, height })); + } + + /// Send one key event to the child, encoded for the keyboard state the + /// child asked for. Ordered with [`TerminalHandle::write`] and the other + /// `send_*` methods; ignored by a read-only attach. + /// + /// A consumer that reserves keys for itself decides that before calling: + /// this encoder never swallows a key. + pub fn send_key(&self, ev: &KeyEvent) { + let _ = self.tx.send(Msg::Key(ev.clone())); + } + + /// Send one mouse event, if the mode the child chose reports it. Use + /// [`TerminalHandle::encode_mouse`] first when the surface wants to keep + /// the event otherwise (a wheel notch the child would not hear). + pub fn send_mouse(&self, ev: &MouseEvent) { + let _ = self.tx.send(Msg::Mouse(*ev)); + } + + /// Report a focus change, if the child asked for focus events. + pub fn send_focus(&self, gained: bool) { + let _ = self.tx.send(Msg::Focus(gained)); + } + + /// Paste text, bracketed when the child asked for it. Check + /// [`crate::input::paste_is_safe`] first if the surface wants to confirm + /// a multi-line paste. + pub fn send_paste(&self, text: &str) { + let _ = self.tx.send(Msg::Paste(text.to_string())); + } + + /// The bytes [`TerminalHandle::send_key`] would write, without writing + /// them. Empty for an event the child should not see. + pub fn encode_key(&self, ev: &KeyEvent) -> Vec { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::EncodeKey { + ev: ev.clone(), + reply: reply_tx, + }) + .is_err() + { + return Vec::new(); + } + reply_rx.recv().unwrap_or_default() + } + + /// The bytes [`TerminalHandle::send_mouse`] would write, or `None` when + /// the child's mode does not report this event — which is how a surface + /// learns it may keep the wheel for its own scrolling. + pub fn encode_mouse(&self, ev: &MouseEvent) -> Option> { + let (reply_tx, reply_rx) = mpsc::channel(); + if self + .tx + .send(Msg::EncodeMouse { + ev: *ev, + reply: reply_tx, + }) + .is_err() + { + return None; + } + reply_rx.recv().unwrap_or(None) + } + /// The current revision; bumps on every change. pub fn rev(&self) -> u64 { self.state(|st| st.rev) diff --git a/crates/pty-terminal/src/input.rs b/crates/pty-terminal/src/input.rs new file mode 100644 index 0000000..18a044d --- /dev/null +++ b/crates/pty-terminal/src/input.rs @@ -0,0 +1,320 @@ +//! Encoding semantic input for the child: keys, mouse, focus, paste. +//! +//! This is the other half of the terminal boundary. A consumer that owns a +//! surface (a TUI pane, a GUI widget) knows *what the user did*; only the +//! terminal knows what bytes the child expects for it, because that depends on +//! state the child itself set: DECCKM, the keypad mode, `modifyOtherKeys`, the +//! kitty keyboard flags, the mouse tracking mode and report format, focus +//! reporting, bracketed paste. Splitting that knowledge — semantic events on +//! one side, a second encoder on the other — means two implementations of the +//! kitty keyboard protocol and a class of parity bug nobody can test. +//! +//! So the events here are deliberately dumb and owned ([`KeyEvent`], +//! [`MouseEvent`]), and the encoding happens inside the terminal: +//! [`crate::actor::TerminalActor::encode_key`] and friends, or +//! [`crate::handle::TerminalHandle::send_key`] to encode and write in one +//! ordered step. +//! +//! The encoders themselves are libghostty's ([`libghostty_vt::key`], +//! [`libghostty_vt::mouse`], [`libghostty_vt::focus`], +//! [`libghostty_vt::paste`]), configured from the live terminal. The key and +//! modifier vocabulary is re-exported rather than re-declared: a second +//! `Key` enum would be a translation table that silently drifts. +//! +//! What this module does *not* do: decide anything. It never swallows a key +//! for its own use and it has no notion of a shortcut, a leader, or a detach +//! sequence. A consumer that reserves keys evaluates them before it calls +//! here. + +use libghostty_vt::terminal::Terminal; +use libghostty_vt::{focus, key, mouse, paste}; + +pub use libghostty_vt::key::{Action as KeyAction, Key, KittyKeyFlags, Mods}; +pub use libghostty_vt::mouse::{Action as MouseAction, Button as MouseButton}; + +use crate::actor::Modes; +use crate::graphics::CellSize; + +/// The cell metrics used for mouse coordinates when the terminal has none +/// (kitty graphics are what otherwise gives a terminal a cell size). Only the +/// ratio of position to cell matters for a cell-addressed report, so any +/// consistent pair works; SGR-pixels reports scale with it. +const DEFAULT_CELL: CellSize = CellSize { + width: 8, + height: 16, +}; + +/// One key event from the surface. +/// +/// The three text-ish fields are separate on purpose, because the kitty +/// keyboard protocol reports them separately: +/// +/// - `key` is the logical key (`Key::KeyA`, `Key::ArrowUp`, `Key::Enter`). +/// - `text` is what the key produced with the user's layout and shift state — +/// kitty's "associated text", reported when the child asked for +/// [`KittyKeyFlags::REPORT_ASSOCIATED`]. +/// - `unshifted` is the same key without shift, which is what the shifted-key +/// alternate is derived against for +/// [`KittyKeyFlags::REPORT_ALTERNATES`]. +/// +/// Folding shift into one character upstream loses the alternate, and with it +/// any way for the child to tell `shift+a` from `A` typed on a layout where +/// they differ. +/// +/// A character key needs at least one of `text` and `unshifted` to have an +/// identity the protocol can report: an event that carries neither encodes to +/// nothing under the kitty protocol, because there is no codepoint to name. +/// Named keys (`Key::Enter`, `Key::ArrowUp`) need neither. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyEvent { + /// The logical key. + pub key: Key, + /// Modifiers held. + pub mods: Mods, + /// Press, repeat, or release. Release only reaches the child when it + /// asked for [`KittyKeyFlags::REPORT_EVENTS`]. + pub action: KeyAction, + /// The text the key produced (shift applied), or `None` for a key that + /// produces none. Must not be a C0 control or a platform function-key + /// code: pass `None` and let the logical key speak. + pub text: Option, + /// The same key with no shift applied. + pub unshifted: Option, + /// Modifiers the surface already consumed (a compose or IME step), which + /// the child should not see again. + pub consumed_mods: Mods, + /// Whether this event is part of an in-progress composition. + pub composing: bool, +} + +impl KeyEvent { + /// A plain press of `key` with no modifiers and no text. + pub fn press(key: Key) -> KeyEvent { + KeyEvent { + key, + mods: Mods::empty(), + action: KeyAction::Press, + text: None, + unshifted: None, + consumed_mods: Mods::empty(), + composing: false, + } + } + + /// A press of a character key: `text` is what it typed, `unshifted` the + /// same key without shift. + pub fn typed(key: Key, text: &str, unshifted: Option) -> KeyEvent { + KeyEvent { + text: Some(text.to_string()), + unshifted, + ..KeyEvent::press(key) + } + } + + /// The same event with `mods` held. + pub fn with_mods(mut self, mods: Mods) -> KeyEvent { + self.mods = mods; + self + } +} + +/// One mouse event from the surface, addressed in terminal cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MouseEvent { + /// Press, release, or motion. A wheel notch is a `Press` of + /// [`MouseButton::Four`]/[`MouseButton::Five`] (vertical) or + /// [`MouseButton::Six`]/[`MouseButton::Seven`] (horizontal), as in the + /// protocol. + pub action: MouseAction, + /// The button, or `None` for motion with no button. + pub button: Option, + /// Modifiers held. + pub mods: Mods, + /// Column in the grid, 0-based. + pub col: u16, + /// Row in the grid, 0-based. + pub row: u16, + /// Whether any button is held, which is what distinguishes a drag from a + /// bare move for `?1002`. + pub any_button_pressed: bool, +} + +impl MouseEvent { + /// A press of `button` at a cell. + pub fn press(button: MouseButton, col: u16, row: u16) -> MouseEvent { + MouseEvent { + action: MouseAction::Press, + button: Some(button), + mods: Mods::empty(), + col, + row, + any_button_pressed: true, + } + } + + /// One wheel notch at a cell: `up` chooses button 4 or 5. + pub fn wheel(up: bool, col: u16, row: u16) -> MouseEvent { + MouseEvent { + any_button_pressed: false, + ..MouseEvent::press( + if up { + MouseButton::Four + } else { + MouseButton::Five + }, + col, + row, + ) + } + } + + /// Whether this is a wheel notch rather than a real button. + pub fn is_wheel(&self) -> bool { + matches!( + self.button, + Some(MouseButton::Four | MouseButton::Five | MouseButton::Six | MouseButton::Seven) + ) + } +} + +/// Encode a key event for the child, using `term`'s own keyboard state +/// (DECCKM, keypad, `modifyOtherKeys`, and the kitty keyboard flags). +/// +/// Returns no bytes for an event the child should not see at all — a bare +/// modifier press, or a release while the child has not asked for release +/// events. +pub fn key(term: &Terminal, ev: &KeyEvent) -> Vec { + let Ok(mut encoder) = key::Encoder::new() else { + return Vec::new(); + }; + encoder.set_options_from_terminal(term); + let Ok(mut event) = key::Event::new() else { + return Vec::new(); + }; + event + .set_key(ev.key) + .set_mods(ev.mods) + .set_action(ev.action) + .set_consumed_mods(ev.consumed_mods) + .set_composing(ev.composing); + // The encoder wants the unmodified character; a control or a platform + // function-key code has to be withheld so it uses the logical key. + let text = ev + .text + .as_deref() + .filter(|t| !t.chars().any(is_unencodable_text)); + event.set_utf8(text); + if let Some(c) = ev.unshifted { + event.set_unshifted_codepoint(c); + } + let mut out = Vec::new(); + let _ = encoder.encode_to_vec(&event, &mut out); + out +} + +/// A character the key encoder must not be given as associated text. +fn is_unencodable_text(c: char) -> bool { + let c = c as u32; + c < 0x20 || c == 0x7f || (0xf700..=0xf8ff).contains(&c) +} + +/// Encode a mouse event for the child, using `term`'s tracking mode and +/// report format. +/// +/// `None` means this event is not reportable in the mode the child chose, and +/// the surface keeps it: no tracking at all, a wheel notch under X10 (`?9`, +/// which reports button presses only), or an event the format drops. That +/// distinction is the whole point of the return type — a consumer that wants +/// to scroll its own viewport with the wheel needs to know the child would +/// not have heard it. +pub fn mouse(term: &Terminal, modes: &Modes, ev: &MouseEvent, cell: CellSize) -> Option> { + if !modes.mouse_reporting() { + return None; + } + // X10 reports presses of real buttons and nothing else. libghostty's + // encoder is told the mode, but the rule is stated here so the answer + // does not depend on how the encoder happens to treat a wheel button. + if !modes.mouse_tracking() && (ev.is_wheel() || ev.action != MouseAction::Press) { + return None; + } + let cell = if cell.width == 0 || cell.height == 0 { + DEFAULT_CELL + } else { + cell + }; + let cols = term.cols().unwrap_or(0).max(1) as u32; + let rows = term.rows().unwrap_or(0).max(1) as u32; + + let mut encoder = mouse::Encoder::new().ok()?; + encoder.set_options_from_terminal(term); + encoder + .set_size(mouse::EncoderSize { + screen_width: cols * cell.width, + screen_height: rows * cell.height, + cell_width: cell.width, + cell_height: cell.height, + padding_top: 0, + padding_bottom: 0, + padding_right: 0, + padding_left: 0, + }) + .set_any_button_pressed(ev.any_button_pressed); + + let mut event = mouse::Event::new().ok()?; + event + .set_action(ev.action) + .set_button(ev.button) + .set_mods(ev.mods) + // The middle of the cell: a cell-addressed report rounds to the same + // cell from anywhere inside it, and an SGR-pixels report gets a + // position that is actually in the cell it names. + .set_position(mouse::Position { + x: (ev.col as f32 + 0.5) * cell.width as f32, + y: (ev.row as f32 + 0.5) * cell.height as f32, + }); + + let mut out = Vec::new(); + encoder.encode_to_vec(&event, &mut out).ok()?; + (!out.is_empty()).then_some(out) +} + +/// Encode a focus change, or `None` when the child did not ask for focus +/// events (`?1004`). +pub fn focus(modes: &Modes, gained: bool) -> Option> { + if !modes.focus_events { + return None; + } + let event = if gained { + focus::Event::Gained + } else { + focus::Event::Lost + }; + let mut buf = [0u8; 8]; + let n = event.encode(&mut buf).ok()?; + Some(buf[..n].to_vec()) +} + +/// Encode pasted text: bracketed when the child asked for `?2004`, with +/// control bytes stripped and newlines turned into carriage returns when it +/// did not. +pub fn paste(modes: &Modes, text: &str) -> Vec { + let mut data = text.as_bytes().to_vec(); + // Bracketing adds `ESC[200~` and `ESC[201~`; the encoder never grows the + // payload itself. + let mut buf = vec![0u8; data.len() + 16]; + match paste::encode(&mut data, modes.bracketed_paste, &mut buf) { + Ok(n) => { + buf.truncate(n); + buf + } + Err(_) => Vec::new(), + } +} + +/// Whether pasting `text` is safe without asking the user: a paste carrying a +/// newline (or a forged bracketed-paste end) can run a command the user never +/// typed. Not enforced here — a consumer decides whether to confirm — but the +/// judgement belongs with the terminal, not with each surface. +pub fn paste_is_safe(text: &str) -> bool { + paste::is_safe(text) +} diff --git a/crates/pty-terminal/src/lib.rs b/crates/pty-terminal/src/lib.rs index 01eece4..9ea4c52 100644 --- a/crates/pty-terminal/src/lib.rs +++ b/crates/pty-terminal/src/lib.rs @@ -18,12 +18,21 @@ //! - [`serialize`]: the ATTACH/PEEK replay (Node mode prefix + VT) and the //! plain-text screen (viewport or full scrollback). //! - [`snapshot`]: [`CellGrid`], the typed cell grid for renderers. +//! - [`graphics`]: the kitty graphics state — bounded image bytes, +//! placements, source crops, and where each placement sits in the window +//! that was read — plus the replay block that carries it through +//! ATTACH/PEEK. +//! - [`input`]: the child input encoder — keys (kitty keyboard included), +//! mouse, focus, and paste, encoded from the terminal's own state so no +//! consumer needs a second encoder. //! - [`handle`]: [`TerminalHandle`], a `Send + Sync` handle over an actor //! thread, either spawning a child or attaching to a session daemon. //! - [`screenshot`]: the testkit's [`Screenshot`] capture. pub mod actor; +pub mod graphics; pub mod handle; +pub mod input; pub mod queries; pub mod screenshot; pub mod serialize; @@ -31,9 +40,14 @@ pub mod snapshot; pub mod strip; pub use actor::{Modes, Notification, Range, TerminalActor, TerminalEvent}; +pub use graphics::{ + CellSize, Compression, GraphicsOptions, GraphicsState, ImageBytes, ImageDesc, PixelFormat, + PlaceholderRect, Placement, PlacementPosition, SourceRect, +}; pub use handle::{ AttachOptions, AttemptId, HandleEvent, SessionRef, SpawnOptions, TerminalHandle, }; +pub use input::{Key, KeyAction, KeyEvent, KittyKeyFlags, Mods, MouseAction, MouseButton, MouseEvent}; pub use screenshot::{Screenshot, capture, serialize_for_replay}; pub use serialize::SerializeOpts; pub use snapshot::{CellGrid, CellSnap, ColorSnap, Wide}; diff --git a/crates/pty-terminal/src/screenshot.rs b/crates/pty-terminal/src/screenshot.rs index 56d131e..dcd0e83 100644 --- a/crates/pty-terminal/src/screenshot.rs +++ b/crates/pty-terminal/src/screenshot.rs @@ -34,7 +34,7 @@ impl Screenshot { /// This is the body without Node's mode prefix; the daemon gets the full /// payload from [`crate::TerminalActor::serialize`]. pub fn serialize_for_replay(term: &Terminal) -> String { - serialize::vt(term, true) + serialize::vt(term, true, crate::graphics::CellSize::FALLBACK) } /// Capture the current terminal state into a [`Screenshot`]. @@ -48,7 +48,7 @@ pub fn capture(term: &Terminal) -> Screenshot { while lines.last().map(|l| l.trim().is_empty()).unwrap_or(false) { lines.pop(); } - let ansi = serialize::vt(term, true); + let ansi = serialize::vt(term, true, crate::graphics::CellSize::FALLBACK); let text = lines.join("\n"); Screenshot { lines, text, ansi } } diff --git a/crates/pty-terminal/src/serialize.rs b/crates/pty-terminal/src/serialize.rs index 918c1cd..d9c2cde 100644 --- a/crates/pty-terminal/src/serialize.rs +++ b/crates/pty-terminal/src/serialize.rs @@ -19,6 +19,7 @@ use libghostty_vt::selection::Selection; use libghostty_vt::terminal::{Point, PointCoordinate, Terminal}; use crate::actor::{Modes, TerminalActor}; +use crate::graphics::{self, CellSize}; /// What a replay should carry. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -88,14 +89,36 @@ pub fn mode_prefix(modes: &Modes, include_alt_screen: bool) -> String { /// is the copy the actor took when the child left it. Without it a client /// that reconnects gets a blank normal screen the moment the program exits. /// +/// The prefix comes first because its position is the contract (`ESC[?1049h` +/// at byte 0), which leaves the client on its alternate screen just as the +/// normal half arrives. For text that was invisible — the alternate screen is +/// overwritten immediately afterwards — but kitty image storage is per +/// screen, so the normal screen's images landed in the client's alternate +/// storage and were lost the moment the program exited. One `ESC[?1049l` +/// ahead of the normal half puts it where it belongs; the switch back is +/// [`vt`]'s own `ESC[?1049h`, which also clears and homes the alternate +/// screen the way the body that follows it expects. +/// /// node: src/server.ts:962 and 1017 (`serialize.serialize()`, whose addon /// walks both buffers). pub fn serialize_for_replay(actor: &TerminalActor, opts: SerializeOpts) -> String { - let mut out = mode_prefix(&actor.modes(), opts.include_alt_screen_prefix); + let modes = actor.modes(); + let mut out = mode_prefix(&modes, opts.include_alt_screen_prefix); if let Some(normal) = actor.normal_replay() { - out.push_str(normal); + if opts.include_alt_screen_prefix && modes.alt_screen { + // Back to the normal screen for its own half, then to the + // alternate one again with the cursor homed — Node's + // `ESC[?1049h ESC[H` — because the normal half leaves the cursor + // wherever its own trailing CUP put it and the alternate body + // that follows is written from wherever the cursor is. + out.push_str("\x1b[?1049l"); + out.push_str(normal); + out.push_str("\x1b[?1049h\x1b[H"); + } else { + out.push_str(normal); + } } - out.push_str(&vt(actor.terminal(), opts.scrollback)); + out.push_str(&vt(actor.terminal(), opts.scrollback, actor.cell_size())); out } @@ -242,10 +265,19 @@ fn plain_opts<'t, 's>() -> FormatterOptions<'t, 's> { } /// The VT serialization: cells with styles, then cursor position, modes that -/// differ from their defaults, and the kitty keyboard flags. With -/// `scrollback` the history rows come first; without it only the active area -/// is emitted. -pub fn vt(term: &Terminal, scrollback: bool) -> String { +/// differ from their defaults, the kitty keyboard flags, and the kitty +/// graphics storage. With `scrollback` the history rows come first; without +/// it only the active area is emitted. +/// +/// The graphics block comes last, after the cursor move, because it must not +/// change where the cursor ends up: libghostty's formatter keeps the +/// placeholder cells of a virtual placement (they are ordinary text) but +/// neither the images nor the placements, so without the block a client would +/// replay placeholders naming images it never received +/// (docs/decisions/0012-kitty-graphics-replay.md). It is empty for a terminal +/// with no graphics, so a session that never sent an image is byte-identical +/// to before. +pub fn vt(term: &Terminal, scrollback: bool, cell: CellSize) -> String { let mut out = if scrollback { format(term, vt_opts()) } else { @@ -271,6 +303,7 @@ pub fn vt(term: &Terminal, scrollback: bool) -> String { let cx = term.cursor_x().unwrap_or(0); let cy = term.cursor_y().unwrap_or(0); out.push_str(&format!("\x1b[{};{}H", cy + 1, cx + 1)); + out.push_str(&graphics::replay(term, cell)); out } @@ -320,6 +353,11 @@ mod tests { bracketed_paste: true, focus_events: true, kitty_stack: vec![7, 1], + // Neither of these reaches the prefix: Node does not track them, + // and a replay must not tell the client's terminal to turn a mode + // on that the daemon's own prefix never carried. + mouse_9: true, + app_cursor: true, }; assert_eq!( mode_prefix(&m, true), diff --git a/crates/pty-terminal/tests/graphics.rs b/crates/pty-terminal/tests/graphics.rs new file mode 100644 index 0000000..c783906 --- /dev/null +++ b/crates/pty-terminal/tests/graphics.rs @@ -0,0 +1,931 @@ +//! Kitty graphics state: what a compositor reads, and what survives a +//! replay. +//! +//! Every test drives the actor with the exact bytes OMP writes +//! (`packages/tui/src/terminal-capabilities.ts` `encodeKittyTransmit`, +//! `packages/tui/src/kitty-graphics.ts` `encodeKittyVirtualPlacement` and +//! `encodeKittyPlaceholderGrid`): a `f=100` PNG transmission, a `a=p,U=1` +//! virtual placement, and placeholder cells carrying the image id in their +//! foreground colour and the placement id in their underline colour. + +use std::time::{Duration, Instant}; + +use pty_terminal::graphics::PLACEHOLDER; +use pty_terminal::{ + CellSize, GraphicsOptions, HandleEvent, PixelFormat, PlacementPosition, Range, SerializeOpts, + SpawnOptions, TerminalActor, TerminalHandle, +}; + +/// A 16x8 RGBA PNG. Generated once, checked in as bytes: the point is that +/// the terminal decodes a real PNG, not that this test can make one. +const PNG_16X8_BASE64: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAABAAAAAICAYAAADwdn+XAAAAqUlEQVR4nBXMwYBFIQAAwIewCCGEEEIIIYQQQggh", + "hBBCCCFk8Hc6zHW+72u/PwKRRKZQaXQGk8XmcPm+ICAQSWQKlUZnMFlsDje8IAkIRBKZQqXRGUwWm8NNLygCApFE", + "plBpdAaTxeZwywuagEAkkSlUGp3BZLE53PaCISAQSWQKlUZnMFlsDne8YAkIRBKZQqXRGUwWm8NdLzgCApFEplBp", + "dAaTxeZw+Qf7mTOQwddl2QAAAABJRU5ErkJggg==" +); + +const IMAGE_ID: u32 = 4242; +const PLACEMENT_ID: u32 = 7; + +fn actor(rows: u16, cols: u16, scrollback: usize) -> TerminalActor { + let mut a = TerminalActor::new(rows, cols, scrollback); + assert!( + a.enable_graphics(GraphicsOptions { + cell: CellSize { + width: 8, + height: 16 + }, + ..GraphicsOptions::DEFAULT + }), + "graphics must turn on" + ); + a +} + +/// OMP's `encodeKittyTransmit`: `a=t,f=100,q=2,i=;`. +fn omp_transmit(id: u32) -> String { + format!("\x1b_Ga=t,f=100,q=2,i={id};{PNG_16X8_BASE64}\x1b\\") +} + +/// OMP's `encodeKittyVirtualPlacement`: `a=p,U=1,q=2,i=,p=,c=,r=`. +fn omp_virtual_placement(id: u32, pid: u32, cols: u32, rows: u32) -> String { + format!("\x1b_Ga=p,U=1,q=2,i={id},p={pid},c={cols},r={rows}\x1b\\") +} + +/// The row/column diacritics OMP indexes into, first four entries. +const DIACRITICS: [char; 4] = ['\u{305}', '\u{30d}', '\u{30e}', '\u{310}']; + +/// OMP's `encodeKittyPlaceholderGrid`: image id in the foreground colour, +/// placement id in the underline colour, every cell naming its own row and +/// column. One string per row, no cursor movement. +fn omp_placeholder_rows(id: u32, pid: u32, cols: usize, rows: usize) -> Vec { + let fg = format!( + "\x1b[38;2;{};{};{}m", + (id >> 16) & 0xff, + (id >> 8) & 0xff, + id & 0xff + ); + let ul = format!( + "\x1b[58:2::{}:{}:{}m", + (pid >> 16) & 0xff, + (pid >> 8) & 0xff, + pid & 0xff + ); + (0..rows) + .map(|r| { + let mut line = format!("{fg}{ul}"); + for c in 0..cols { + line.push(PLACEHOLDER); + line.push(DIACRITICS[r]); + line.push(DIACRITICS[c]); + } + line.push_str("\x1b[39;59m"); + line + }) + .collect() +} + +/// The whole OMP render: transmit, then the placement APC in front of the +/// first placeholder row, rows separated by CR/LF. +fn omp_image(id: u32, pid: u32, cols: usize, rows: usize) -> String { + let mut out = omp_transmit(id); + let grid = omp_placeholder_rows(id, pid, cols, rows); + for (i, row) in grid.iter().enumerate() { + if i == 0 { + out.push_str(&omp_virtual_placement(id, pid, cols as u32, rows as u32)); + } else { + out.push_str("\r\n"); + } + out.push_str(row); + } + out +} + +#[test] +fn graphics_are_off_until_the_owner_asks() { + let mut a = TerminalActor::new(10, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let state = a.graphics_state(0); + assert!(!state.enabled, "no storage limit, no protocol"); + assert!(state.images.is_empty()); + assert!(state.placements.is_empty()); + assert_eq!(a.graphics_generation(), 0); + assert!(a.image_bytes(IMAGE_ID).is_none()); +} + +#[test] +fn an_omp_write_gives_image_bytes_placement_identity_crop_and_position() { + let mut a = actor(10, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + + let state = a.graphics_state(0); + assert!(state.enabled); + assert_ne!(state.generation, 0, "a transmit mutates the storage"); + + let image = state.image(IMAGE_ID).expect("the image is stored"); + assert_eq!((image.width, image.height), (16, 8)); + // The PNG was decoded: libghostty stores 8-bit RGBA. + assert_eq!(image.format, PixelFormat::Rgba); + assert_eq!(image.len, 16 * 8 * 4); + assert_ne!(image.generation, 0); + + let bytes = a.image_bytes(IMAGE_ID).expect("bytes are readable"); + assert_eq!(bytes.data.len(), 16 * 8 * 4); + assert_eq!(bytes.desc.generation, image.generation); + + assert_eq!(state.placements.len(), 1); + let p = &state.placements[0]; + assert_eq!((p.image_id, p.placement_id), (IMAGE_ID, PLACEMENT_ID)); + assert!(p.is_virtual, "U=1 is a virtual placement"); + assert_eq!(p.requested_cells, (2, 2)); + // No source rect was sent, so the crop is the whole image. + assert_eq!( + (p.source.x, p.source.y, p.source.width, p.source.height), + (0, 0, 16, 8) + ); + match p.position { + PlacementPosition::Placeholder(r) => { + assert_eq!((r.row, r.col), (0, 0), "written at the home position"); + assert_eq!((r.rows, r.cols), (2, 2)); + assert_eq!((r.cell_row, r.cell_col), (0, 0)); + assert_eq!((r.origin_row, r.origin_col), (0, 0)); + } + other => panic!("expected a placeholder position, got {other:?}"), + } +} + +#[test] +fn a_source_rect_and_offsets_come_back_resolved() { + let mut a = actor(10, 20, 0); + a.write(omp_transmit(IMAGE_ID).as_bytes()); + // A crop with a zero height: kitty means "to the bottom edge". + a.write( + format!("\x1b_Ga=p,U=1,q=2,i={IMAGE_ID},x=4,y=2,w=8,h=0,X=3,Y=5,z=-1,c=2,r=1\x1b\\") + .as_bytes(), + ); + for row in omp_placeholder_rows(IMAGE_ID, 0, 2, 1) { + a.write(row.as_bytes()); + } + let state = a.graphics_state(0); + let p = &state.placements[0]; + assert_eq!(p.placement_id, 0, "no p= means no placement id"); + assert_eq!( + (p.source.x, p.source.y, p.source.width, p.source.height), + (4, 2, 8, 6), + "h=0 resolves to the rest of the image" + ); + assert_eq!(p.cell_offset, (3, 5)); + assert_eq!(p.z, -1); +} + +#[test] +fn scrolling_moves_the_placement_and_scrollback_still_finds_it() { + let mut a = actor(6, 20, 100); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let before = a.graphics_state(0).generation; + + // Push the image up by three rows. + a.write(b"\r\n\r\n\r\n\r\n\r\nx"); + + let state = a.graphics_state(0); + assert_eq!( + state.generation, before, + "scrolling does not touch the storage" + ); + let p = &state.placements[0]; + match p.position { + PlacementPosition::Placeholder(r) => { + assert_eq!(r.row, 0, "the image's second row is now the top row"); + assert_eq!(r.rows, 1, "its first row has scrolled into history"); + assert_eq!(r.cell_row, 1, "the visible row is image row 1"); + assert_eq!(r.origin_row, -1, "image row 0 is one row above"); + } + other => panic!("expected a placeholder position, got {other:?}"), + } + + // The same read one row back into history sees the whole image again. + let scrolled = a.graphics_state(1); + match scrolled.placements[0].position { + PlacementPosition::Placeholder(r) => { + assert_eq!((r.row, r.rows, r.cell_row), (0, 2, 0)); + assert_eq!(r.origin_row, 0); + } + other => panic!("expected a placeholder position, got {other:?}"), + } +} + +#[test] +fn a_resize_keeps_the_image_and_reprojects_it() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let generation = a.graphics_state(0).generation; + + a.resize(10, 8); + + let state = a.graphics_state(0); + assert_eq!(state.generation, generation, "a resize keeps the storage"); + assert_eq!(state.cell, CellSize { width: 8, height: 16 }); + assert!( + a.image_bytes(IMAGE_ID).is_some(), + "the pixels survive a resize" + ); + assert!( + matches!( + state.placements[0].position, + PlacementPosition::Placeholder(_) + ), + "the placeholder cells reflowed with the text, and were found again" + ); +} + +#[test] +fn the_alternate_screen_has_its_own_storage() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(a.graphics_state(0).placements.len(), 1); + + a.write(b"\x1b[?1049h"); + assert!( + a.graphics_state(0).placements.is_empty(), + "the alternate screen starts with no images" + ); + assert!( + a.graphics_state(0).enabled, + "the protocol is on for both screens" + ); + + // An image placed on the alternate screen is the alternate screen's. + a.write(omp_image(99, 1, 1, 1).as_bytes()); + let alt = a.graphics_state(0); + assert_eq!(alt.placements.len(), 1); + assert_eq!(alt.placements[0].image_id, 99); + + a.write(b"\x1b[?1049l"); + let back = a.graphics_state(0); + assert_eq!(back.placements.len(), 1, "the primary screen kept its own"); + assert_eq!(back.placements[0].image_id, IMAGE_ID); +} + +#[test] +fn a_delete_from_the_child_drops_the_placement_and_the_bytes() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let before = a.graphics_state(0).generation; + + // OMP's delete-placement, then its delete-image. + a.write(format!("\x1b_Ga=d,d=i,i={IMAGE_ID},p={PLACEMENT_ID},q=2\x1b\\").as_bytes()); + let after_placement = a.graphics_state(0); + assert!(after_placement.placements.is_empty()); + assert_ne!(after_placement.generation, before, "a delete is a mutation"); + assert!( + a.image_bytes(IMAGE_ID).is_some(), + "deleting a placement keeps the image" + ); + + a.write(format!("\x1b_Ga=d,d=I,i={IMAGE_ID},q=2\x1b\\").as_bytes()); + assert!(a.image_bytes(IMAGE_ID).is_none(), "now the image is gone"); +} + +#[test] +fn clear_graphics_drops_everything_and_keeps_the_protocol() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert!(a.image_bytes(IMAGE_ID).is_some()); + + a.clear_graphics(); + + let state = a.graphics_state(0); + assert!(state.enabled, "the protocol stays on"); + assert!(state.images.is_empty()); + assert!(state.placements.is_empty()); + assert!(a.image_bytes(IMAGE_ID).is_none()); + + // And the terminal still accepts a new image. + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(a.graphics_state(0).placements.len(), 1); +} + +#[test] +fn zeroing_the_limit_turns_the_protocol_off() { + let mut a = actor(6, 20, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + + a.set_graphics_storage_limit(0); + + assert!(!a.graphics_state(0).enabled); + assert!(a.graphics_options().is_none()); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert!( + a.graphics_state(0).placements.is_empty(), + "a disabled terminal stores nothing" + ); +} + +/// The case the Node daemon loses: a client that was not connected when the +/// image was written attaches, gets only the SCREEN payload, and must end up +/// with the same image bytes and the same placement. +#[test] +fn a_late_client_reconstructs_the_image_from_the_replay_alone() { + let mut source = actor(8, 20, 100); + source.write(b"before\r\n"); + source.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + source.write(b"\r\nafter"); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains(&format!("i={IMAGE_ID}")), + "the replay carries the image" + ); + assert!( + payload.contains("a=p,U=1"), + "and the virtual placement that binds the placeholder cells" + ); + + // A fresh terminal that has seen none of the original DATA. + let mut late = actor(8, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + + let state = late.graphics_state(0); + let image = state + .image(IMAGE_ID) + .expect("the late client has the image"); + assert_eq!((image.width, image.height), (16, 8)); + assert_eq!(image.format, PixelFormat::Rgba); + assert_eq!( + late.image_bytes(IMAGE_ID).map(|b| b.data), + source.image_bytes(IMAGE_ID).map(|b| b.data), + "byte for byte the same pixels" + ); + + assert_eq!(state.placements.len(), 1); + let p = &state.placements[0]; + assert_eq!(p.image_id, IMAGE_ID); + assert_eq!(p.placement_id, PLACEMENT_ID, "placement identity survives"); + assert!(p.is_virtual); + assert_eq!(p.requested_cells, (2, 2)); + + // And it is in the same place as in the source terminal. + assert_eq!( + p.position, + source.graphics_state(0).placements[0].position, + "same cells" + ); + assert_eq!(late.plain(Range::Full), source.plain(Range::Full)); +} + +#[test] +fn a_cursor_positioned_placement_replays_at_its_cell() { + let mut source = actor(8, 20, 0); + source.write(b"\r\n\r\n "); + source.write(omp_transmit(IMAGE_ID).as_bytes()); + source.write(format!("\x1b_Ga=p,q=2,i={IMAGE_ID},p=3,c=2,r=1\x1b\\").as_bytes()); + let placed = match source.graphics_state(0).placements[0].position { + PlacementPosition::Direct { col, row, .. } => (col, row), + other => panic!("expected a direct placement, got {other:?}"), + }; + assert_eq!(placed, (2, 2), "at the cursor"); + let cursor = source.cursor(); + + let payload = source.serialize(SerializeOpts::ATTACH); + let mut late = actor(8, 20, 0); + late.reset(); + late.write(payload.as_bytes()); + + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!(state.placements[0].placement_id, 3); + assert_eq!( + state.placements[0].position, + PlacementPosition::Direct { + col: 2, + row: 2, + cols: 2, + rows: 1 + } + ); + assert_eq!( + late.cursor(), + cursor, + "the graphics block leaves the cursor where the replay put it" + ); +} + +#[test] +fn a_replay_without_graphics_is_unchanged() { + let mut plain = TerminalActor::new(6, 20, 0); + plain.write(b"hello\r\nworld"); + let without = plain.serialize(SerializeOpts::ATTACH); + + let mut enabled = actor(6, 20, 0); + enabled.write(b"hello\r\nworld"); + assert_eq!( + enabled.serialize(SerializeOpts::ATTACH), + without, + "a session that never sent an image serializes exactly as before" + ); +} + +/// The handle path: a real child in a real PTY, the state read from another +/// thread. Kitty graphics need a per-thread PNG decoder and an `!Send` +/// terminal, so this is the case that proves the actor thread set both up. +#[test] +fn a_spawned_child_that_draws_an_image_is_queryable_through_the_handle() { + let sequence = omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2); + let h = TerminalHandle::spawn( + "cat", + &[], + SpawnOptions { + rows: 10, + cols: 20, + graphics: Some(GraphicsOptions::DEFAULT), + ..SpawnOptions::default() + }, + ) + .expect("spawn"); + assert!(h.wait_ready(Duration::from_secs(2))); + let events = h.subscribe(); + + // `cat` echoes what we write, so the child is the one emitting the + // sequence into the terminal. + h.write(sequence.as_bytes()); + + let deadline = Instant::now() + Duration::from_secs(5); + let state = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!( + Instant::now() < deadline, + "timed out waiting for the image; screen:\n{}", + h.plain(Range::Full) + ); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + + assert!(state.enabled); + let image = state.image(IMAGE_ID).expect("the image is stored"); + assert_eq!((image.width, image.height), (16, 8)); + assert_eq!( + h.image_bytes(IMAGE_ID).map(|b| b.data.len()), + Some(16 * 8 * 4) + ); + assert_eq!(h.graphics_generation(), state.generation); + + let p = &state.placements[0]; + assert_eq!((p.image_id, p.placement_id), (IMAGE_ID, PLACEMENT_ID)); + assert!(matches!(p.position, PlacementPosition::Placeholder(_))); + + let mut saw_graphics = false; + while let Ok(ev) = events.try_recv() { + if matches!(ev, HandleEvent::Graphics(g) if g == state.generation) { + saw_graphics = true; + } + } + assert!(saw_graphics, "the storage change is announced"); + + h.clear_graphics(); + let deadline = Instant::now() + Duration::from_secs(5); + while !h.graphics(0).placements.is_empty() { + assert!(Instant::now() < deadline, "clear_graphics did not take"); + h.wait_rev(h.rev(), Duration::from_millis(100)); + } + assert!(h.image_bytes(IMAGE_ID).is_none()); + h.kill(); +} + +// ── bounds ── + +/// A raw RGBA transmission, chunked at 4096 base64 bytes the way every real +/// sender does it (kitty's own limit per APC command): `m=1` on every chunk +/// but the last. +fn chunked_rgba_transmit(id: u32, width: u32, height: u32, data: &[u8]) -> String { + const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut payload = String::with_capacity(data.len().div_ceil(3) * 4); + for c in data.chunks(3) { + let b = [c[0], c.get(1).copied().unwrap_or(0), c.get(2).copied().unwrap_or(0)]; + let n = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | b[2] as u32; + payload.push(B64[(n >> 18) as usize & 63] as char); + payload.push(B64[(n >> 12) as usize & 63] as char); + payload.push(if c.len() > 1 { B64[(n >> 6) as usize & 63] as char } else { '=' }); + payload.push(if c.len() > 2 { B64[n as usize & 63] as char } else { '=' }); + } + let mut out = String::with_capacity(payload.len() + 4096); + let mut chunks = payload.as_bytes().chunks(4096).peekable(); + let mut first = true; + while let Some(chunk) = chunks.next() { + let more = chunks.peek().is_some(); + let chunk = std::str::from_utf8(chunk).expect("base64 is ascii"); + if first { + let m = if more { ",m=1" } else { "" }; + out.push_str(&format!( + "\x1b_Ga=t,q=2,i={id},f=32,s={width},v={height}{m};{chunk}\x1b\\" + )); + first = false; + } else { + let m = if more { 1 } else { 0 }; + out.push_str(&format!("\x1b_Gq=2,m={m};{chunk}\x1b\\")); + } + } + out +} + +/// The bound on a replay is the bound on the state. An image well past the +/// 3 MiB a former replay cap allowed is still stored *and* still replayed: +/// otherwise a supported terminal state would be one a late client can never +/// be given, which is the exact failure this module exists to prevent. +#[test] +fn an_image_far_larger_than_three_mib_still_replays() { + // 1024 x 1024 RGBA = 4 MiB, over any per-image cap and under + // MAX_STORAGE_BYTES. + let (w, h) = (1024u32, 1024u32); + let data: Vec = (0..(w * h * 4)) + .map(|i| (i % 251) as u8) + .collect(); + assert!(data.len() > 3 * 1024 * 1024); + + let mut source = actor(8, 20, 0); + source.write(chunked_rgba_transmit(IMAGE_ID, w, h, &data).as_bytes()); + source.write(omp_virtual_placement(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + for row in omp_placeholder_rows(IMAGE_ID, PLACEMENT_ID, 2, 2) { + source.write(row.as_bytes()); + } + let stored = source + .image_bytes(IMAGE_ID) + .expect("the terminal accepted a 4 MiB image"); + assert_eq!(stored.data.len(), data.len()); + assert_eq!(stored.data, data, "chunked transmission arrived intact"); + + let payload = source.serialize(SerializeOpts::ATTACH); + let mut late = actor(8, 20, 0); + late.reset(); + late.write(payload.as_bytes()); + + assert_eq!( + late.image_bytes(IMAGE_ID).map(|b| b.data), + Some(data), + "a late client gets every byte of it" + ); + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!(state.placements[0].placement_id, PLACEMENT_ID); +} + +#[test] +fn the_storage_limit_is_clamped_to_what_a_replay_can_carry() { + let mut a = TerminalActor::new(8, 20, 0); + assert!(a.enable_graphics(GraphicsOptions { + storage_bytes: 8 * 1024 * 1024 * 1024, + ..GraphicsOptions::DEFAULT + })); + assert_eq!( + a.graphics_options().map(|o| o.storage_bytes), + Some(pty_terminal::graphics::MAX_STORAGE_BYTES), + "asking for more than can be replayed gets the supported bound" + ); + assert_eq!( + a.graphics_state(0).storage_bytes, + pty_terminal::graphics::MAX_STORAGE_BYTES + ); + + a.set_graphics_storage_limit(u64::MAX); + assert_eq!( + a.graphics_state(0).storage_bytes, + pty_terminal::graphics::MAX_STORAGE_BYTES + ); +} + +// ── cell metrics ── + +/// A placement that named neither `c=` nor `r=` gets its cell extent from the +/// image's pixel size and the cell size — so the cell size has to be the +/// client's real one, not a guess about its font. +#[test] +fn an_implicit_placement_takes_its_extent_from_the_declared_cell() { + let mut a = TerminalActor::new(8, 40, 0); + assert!(a.enable_graphics(GraphicsOptions { + cell: CellSize::default(), // undeclared + ..GraphicsOptions::DEFAULT + })); + let state = a.graphics_state(0); + assert!(!state.cell_declared, "nobody has said how big a cell is"); + assert_eq!(state.cell, CellSize::FALLBACK); + + // A 32x32 image placed with no c=/r=: 4x2 cells at the 8x16 fallback. + let data = vec![0u8; 32 * 32 * 4]; + a.write(chunked_rgba_transmit(IMAGE_ID, 32, 32, &data).as_bytes()); + a.write(format!("\x1b_Ga=p,q=2,i={IMAGE_ID},p=1\x1b\\").as_bytes()); + assert_eq!(a.graphics_state(0).placements[0].cell_size, (4, 2)); + + // The client says its cells are 16x16: the same image is now 2x2 cells. + a.set_cell_size(CellSize { + width: 16, + height: 16, + }); + let state = a.graphics_state(0); + assert!(state.cell_declared); + assert_eq!(state.cell, CellSize { width: 16, height: 16 }); + assert_eq!( + state.placements[0].cell_size, + (2, 2), + "geometry follows the declared cell" + ); + assert_eq!( + state.placements[0].requested_cells, + (0, 0), + "the child never named a size; only the derived extent moved" + ); + + // And it survives a grid resize, which is a change of grid, not of font. + a.resize(20, 6); + let state = a.graphics_state(0); + assert!(state.cell_declared); + assert_eq!(state.placements[0].cell_size, (2, 2)); +} + +#[test] +fn a_declared_cell_does_not_move_an_explicit_placement() { + let mut a = actor(8, 40, 0); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(a.graphics_state(0).placements[0].cell_size, (2, 2)); + a.set_cell_size(CellSize { + width: 20, + height: 40, + }); + assert_eq!( + a.graphics_state(0).placements[0].cell_size, + (2, 2), + "c=2,r=2 is the child's own answer and no cell size changes it" + ); +} + +// ── replay fidelity (regressions) ── + +/// `w=`/`h=` default to 0 in the protocol, which means "the whole image", so +/// a replay that omits them turns a cropped placement into an uncropped one +/// squeezed into the cropped placement's cell box: the wrong pixels at the +/// right size, on every reattaching client. +#[test] +fn a_cropped_placement_keeps_its_crop_through_a_replay() { + let mut source = actor(8, 20, 0); + source.write(omp_transmit(IMAGE_ID).as_bytes()); + source.write( + format!("\x1b_Ga=p,U=1,q=2,i={IMAGE_ID},p={PLACEMENT_ID},x=4,y=2,w=8,h=6,c=2,r=1\x1b\\") + .as_bytes(), + ); + for row in omp_placeholder_rows(IMAGE_ID, PLACEMENT_ID, 2, 1) { + source.write(row.as_bytes()); + } + let crop = source.graphics_state(0).placements[0].source; + assert_eq!((crop.x, crop.y, crop.width, crop.height), (4, 2, 8, 6)); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains("w=8,h=6"), + "the replay names the crop: {payload:?}" + ); + + let mut late = actor(8, 20, 0); + late.reset(); + late.write(payload.as_bytes()); + let replayed = late.graphics_state(0).placements[0].source; + assert_eq!( + (replayed.x, replayed.y, replayed.width, replayed.height), + (4, 2, 8, 6), + "same pixels, not the whole image measured from (4, 2)" + ); +} + +/// A virtual placement's command carries no position, and its placeholder +/// cells are serialized wherever they are — including the scrollback. So a +/// replay must carry it even when nothing of it is in the viewport, or a +/// late client holds placeholder cells naming an image it never received. +#[test] +fn a_virtual_placement_scrolled_into_history_still_replays() { + let mut source = actor(4, 20, 100); + source.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + // Push it entirely out of the viewport. + source.write(b"\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\nbottom"); + assert_eq!( + source.graphics_state(0).placements[0].position, + PlacementPosition::Offscreen, + "nothing of it is in the live viewport any more" + ); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains("a=p,U=1"), + "the placement is still replayed: {payload:?}" + ); + + let mut late = actor(4, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!( + (state.placements[0].image_id, state.placements[0].placement_id), + (IMAGE_ID, PLACEMENT_ID) + ); + assert!(late.image_bytes(IMAGE_ID).is_some()); +} + +/// A cursor-positioned placement whose top has scrolled above the viewport is +/// still partly on screen. libghostty reports a negative row for exactly this +/// case, so dropping every negative row loses an image the source terminal is +/// showing. +#[test] +fn a_partially_scrolled_direct_placement_replays_clipped() { + let mut source = actor(6, 20, 100); + source.write(omp_transmit(IMAGE_ID).as_bytes()); + source.write(format!("\x1b_Ga=p,q=2,i={IMAGE_ID},p=3,c=2,r=4\x1b\\").as_bytes()); + // Scroll it up by two rows: its top two cell rows are gone, its bottom + // two are still on screen. + source.write(b"\r\n\r\n\r\nx"); + let position = source.graphics_state(0).placements[0].position; + let PlacementPosition::Direct { row, .. } = position else { + panic!("expected a direct placement, got {position:?}"); + }; + assert!(row < 0, "its top has scrolled above the viewport: {row}"); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.contains(&format!("i={IMAGE_ID},p=3")), + "the visible part is still replayed: {payload:?}" + ); + let mut late = actor(6, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + let state = late.graphics_state(0); + assert_eq!(state.placements.len(), 1, "not dropped"); + assert_eq!(state.placements[0].placement_id, 3); + assert!( + state.placements[0].requested_cells.1 < 4, + "the rows that scrolled off are not re-placed: {:?}", + state.placements[0].requested_cells + ); +} + +/// A grayscale PNG is an ordinary PNG. Rejecting it (which is what checking +/// the decoder's output for RGBA did) drops every monochrome plot a child +/// sends, with no diagnostic. +#[test] +fn a_grayscale_png_is_stored_as_rgba() { + const GRAY_4X2_PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAACCAAAAABawyK/AAAAEklEQVR4nGNgsKnYwsDl1rQPAAwmAvkL8nz8AAAAAElFTkSuQmCC"; + let mut a = actor(8, 20, 0); + a.write(format!("\x1b_Ga=t,f=100,q=2,i=55;{GRAY_4X2_PNG}\x1b\\").as_bytes()); + a.write("\x1b_Ga=p,U=1,q=2,i=55,p=1,c=1,r=1\x1b\\".as_bytes()); + for row in omp_placeholder_rows(55, 1, 1, 1) { + a.write(row.as_bytes()); + } + let bytes = a.image_bytes(55).expect("the grayscale PNG was accepted"); + assert_eq!((bytes.desc.width, bytes.desc.height), (4, 2)); + assert_eq!(bytes.desc.format, PixelFormat::Rgba); + assert_eq!(bytes.data.len(), 4 * 2 * 4); + for px in bytes.data.chunks_exact(4) { + assert_eq!(px[0], px[1], "grey expands to r == g == b"); + assert_eq!(px[1], px[2]); + assert_eq!(px[3], 0xff, "with full alpha"); + } +} + +/// Kitty allows a placeholder cell to name its image with a 256-colour +/// foreground, and its own worked example uses one. A reader that insists on +/// truecolor does not see such a cell at all. +#[test] +fn a_palette_foreground_names_a_placeholder_image() { + let mut a = actor(8, 20, 0); + a.write(omp_transmit(42).as_bytes()); + a.write("\x1b_Ga=p,U=1,q=2,i=42,c=1,r=1\x1b\\".as_bytes()); + a.write(format!("\x1b[38;5;42m{PLACEHOLDER}\u{305}\u{305}\x1b[39m").as_bytes()); + let state = a.graphics_state(0); + assert_eq!(state.placements.len(), 1); + match state.placements[0].position { + PlacementPosition::Placeholder(r) => assert_eq!((r.row, r.col), (0, 0)), + other => panic!("a palette foreground must name the image, got {other:?}"), + } +} + +/// Kitty's compact placeholder form leaves the diacritics off every cell but +/// the first: those cells inherit the row, the column plus one, and the image +/// id's high byte from the cell to their left. +#[test] +fn a_bare_continuation_cell_inherits_row_column_and_id_high_byte() { + let id: u32 = 0x0100_0009; + let mut a = actor(8, 20, 0); + a.write(format!("\x1b_Ga=t,f=100,q=2,i={id};{PNG_16X8_BASE64}\x1b\\").as_bytes()); + a.write(format!("\x1b_Ga=p,U=1,q=2,i={id},c=3,r=1\x1b\\").as_bytes()); + // Row 0, column 0, id high byte 1 — then two bare continuation cells. + let first = format!("{PLACEHOLDER}\u{305}\u{305}\u{30d}"); + a.write( + format!("\x1b[38;2;0;0;9m{first}{PLACEHOLDER}{PLACEHOLDER}\x1b[39m").as_bytes(), + ); + + let state = a.graphics_state(0); + assert_eq!(state.placements.len(), 1); + assert_eq!(state.images[0].id, id, "the high byte reached the image id"); + match state.placements[0].position { + PlacementPosition::Placeholder(r) => { + assert_eq!((r.row, r.col), (0, 0)); + assert_eq!( + (r.rows, r.cols), + (1, 3), + "all three cells belong to the same placement" + ); + } + other => panic!("expected a placeholder position, got {other:?}"), + } +} + +/// Raising the limit is the one way graphics get turned on, so it has to turn +/// them on completely: the cell metrics the terminal already had, and a PNG +/// decoder. The old shape rebuilt the options from the defaults, which +/// silently replaced a declared cell size with 8x16. +#[test] +fn raising_the_storage_limit_keeps_the_cell_size_and_decodes_png() { + let mut a = TerminalActor::new(8, 20, 0); + assert!(a.enable_graphics(GraphicsOptions { + cell: CellSize { + width: 10, + height: 21 + }, + ..GraphicsOptions::DEFAULT + })); + a.set_graphics_storage_limit(0); + assert!(!a.graphics_state(0).enabled); + + a.set_graphics_storage_limit(4 * 1024 * 1024); + let state = a.graphics_state(0); + assert!(state.enabled); + assert!(state.cell_declared); + assert_eq!( + state.cell, + CellSize { + width: 10, + height: 21 + }, + "the declared cell size survived" + ); + + // And PNG still decodes, which it would not with no decoder installed. + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert!(a.image_bytes(IMAGE_ID).is_some()); +} + +/// An actor that never enabled graphics and then has its limit raised must +/// end up in the same complete state, not one that stores images it cannot +/// decode. +#[test] +fn the_storage_limit_alone_turns_graphics_fully_on() { + let mut a = TerminalActor::new(8, 20, 0); + a.set_graphics_storage_limit(4 * 1024 * 1024); + assert!(a.graphics_state(0).enabled); + a.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + let bytes = a.image_bytes(IMAGE_ID).expect("a PNG transmission decodes"); + assert_eq!((bytes.desc.width, bytes.desc.height), (16, 8)); +} + +/// The normal screen's images have to land in the client's normal screen. +/// Kitty storage is per screen and the replay's alt-screen prefix is at byte +/// 0, so the normal half has to say where it belongs. +#[test] +fn a_replay_from_the_alt_screen_puts_the_normal_screens_images_on_the_normal_screen() { + let mut source = actor(6, 20, 100); + source.write(omp_image(IMAGE_ID, PLACEMENT_ID, 2, 2).as_bytes()); + assert_eq!(source.graphics_state(0).placements.len(), 1); + source.write(b"\x1b[?1049h"); + source.write(b"full-screen program"); + assert!(source.graphics_state(0).placements.is_empty()); + + let payload = source.serialize(SerializeOpts::ATTACH); + assert!( + payload.starts_with("\x1b[?1049h"), + "the prefix position is the contract" + ); + + let mut late = actor(6, 20, 100); + late.reset(); + late.write(payload.as_bytes()); + assert!( + late.graphics_state(0).placements.is_empty(), + "the alternate screen has no images, as in the source" + ); + assert_eq!( + late.plain(Range::Viewport), + source.plain(Range::Viewport), + "the alternate screen replays as it was" + ); + + // The program exits: the normal screen comes back, and so do its images. + late.write(b"\x1b[?1049l"); + let state = late.graphics_state(0); + assert_eq!( + state.placements.len(), + 1, + "the normal screen's image survived the replay" + ); + assert_eq!(state.placements[0].placement_id, PLACEMENT_ID); + assert!(late.image_bytes(IMAGE_ID).is_some()); +} diff --git a/crates/pty-terminal/tests/handle.rs b/crates/pty-terminal/tests/handle.rs index ddf2b57..d050224 100644 --- a/crates/pty-terminal/tests/handle.rs +++ b/crates/pty-terminal/tests/handle.rs @@ -279,6 +279,234 @@ fn attach_identity_reconnect_reaches_the_replacement() { rig.kill("a"); } +/// The case a Node daemon cannot serve: the child draws a kitty image, and a +/// client attaches only afterwards. The image was in `DATA` that this client +/// never saw, so everything it knows comes from the daemon's `SCREEN` — which +/// carries the image because the session's own terminal holds it +/// (docs/decisions/0012-kitty-graphics-replay.md). +/// +/// The bytes are OMP's (`packages/tui/src/terminal-capabilities.ts` +/// `encodeKittyTransmit`, `packages/tui/src/kitty-graphics.ts` +/// `encodeKittyVirtualPlacement` / `encodeKittyPlaceholderGrid`): a `f=100` +/// PNG transmission, a virtual placement, and a placeholder cell carrying the +/// image id in its foreground colour and the placement id in its underline +/// colour. +#[test] +fn a_late_attach_gets_the_image_the_child_drew_before_it_connected() { + let Some(rig) = Rig::new() else { + eprintln!("skipping: no pty binary"); + return; + }; + // A 1x1 red PNG (the kitty protocol's own example image), image id 4242, + // placement id 7, one placeholder cell at image row 0, column 0. + let png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; + let script = format!( + "printf 'drawn\\n'; \ + printf '\\033_Ga=t,f=100,q=2,i=4242;{png}\\033\\\\'; \ + printf '\\033_Ga=p,U=1,q=2,i=4242,p=7,c=1,r=1\\033\\\\'; \ + printf '\\033[38;2;0;16;146m\\033[58:2::0:0:7m\u{10eeee}\u{305}\u{305}\\033[39;59m'; \ + exec cat" + ); + rig.run("g", &script); + // Let the child finish drawing before anyone attaches: this client must + // learn the image from the replay, not from live DATA. + std::thread::sleep(Duration::from_millis(400)); + + let h = TerminalHandle::attach( + rig.session("g"), + AttachOptions { + graphics: Some(pty_terminal::GraphicsOptions::DEFAULT), + ..Default::default() + }, + ) + .expect("attach"); + assert!(h.wait_ready(Duration::from_secs(5)), "first SCREEN"); + assert!(h.plain(Range::Full).contains("drawn")); + + let deadline = Instant::now() + Duration::from_secs(5); + let state = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!( + Instant::now() < deadline, + "the replay carried no placement; screen:\n{}", + h.plain(Range::Full) + ); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + + let image = state.image(4242).expect("the image came with the replay"); + assert_eq!((image.width, image.height), (1, 1)); + assert_eq!( + h.image_bytes(4242).map(|b| b.data), + Some(vec![255, 0, 0, 255]), + "the pixels themselves, decoded from the PNG" + ); + let p = &state.placements[0]; + assert_eq!((p.image_id, p.placement_id), (4242, 7)); + assert!(p.is_virtual); + assert!( + matches!(p.position, pty_terminal::PlacementPosition::Placeholder(_)), + "located by its placeholder cell, at {:?}", + p.position + ); + + // A reconnect resets the terminal and replays a fresh SCREEN. The image + // has to come back with it: the reset must not take the storage away, or + // the replay's own transmission would be rejected. + let position = p.position; + h.reconnect().expect("reconnect"); + assert!(h.wait_ready(Duration::from_secs(5)), "SCREEN after reconnect"); + let deadline = Instant::now() + Duration::from_secs(5); + let after = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!(Instant::now() < deadline, "reconnect lost the image"); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + assert_eq!( + h.image_bytes(4242).map(|b| b.data), + Some(vec![255, 0, 0, 255]) + ); + assert_eq!(after.placements[0].placement_id, 7); + assert_eq!(after.placements[0].position, position, "same cell"); + + h.kill(); + rig.kill("g"); +} + +/// A daemon-side detach is a state a consumer has to be able to enter, so it +/// is an event, not something to poll `connected()` for. A reconnect is the +/// symmetric one: `Connected` means the new socket is up, and the attempt's +/// first SCREEN follows. +#[test] +fn a_lost_socket_and_a_reconnect_are_both_announced() { + let Some(rig) = Rig::new() else { + eprintln!("skipping: no pty binary"); + return; + }; + rig.run("d", "printf 'first\\n'; exec sleep 60"); + let h = TerminalHandle::attach(rig.session("d"), AttachOptions::default()).expect("attach"); + assert!(h.wait_ready(Duration::from_secs(5))); + let events = h.subscribe(); + + rig.kill("d"); + let deadline = Instant::now() + Duration::from_secs(5); + while h.connected() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + assert!(!h.connected()); + let mut saw_disconnected = false; + while let Ok(ev) = events.try_recv() { + if ev == HandleEvent::Disconnected { + saw_disconnected = true; + } + } + assert!(saw_disconnected, "the lost socket was announced"); + + rig.run("d", "printf 'second\\n'; exec sleep 60"); + h.reconnect().expect("reconnect"); + assert!(h.wait_ready(Duration::from_secs(5))); + let mut saw_connected = false; + while let Ok(ev) = events.try_recv() { + if ev == HandleEvent::Connected(h.attempt()) { + saw_connected = true; + } + } + assert!(saw_connected, "the new socket was announced"); + h.kill(); + rig.kill("d"); +} + +/// Cell metrics live on the client's host — its font — but the session's +/// terminal is what answers geometry for a placement that left `c=`/`r=` +/// implicit, including in the replay every other client gets. So the client +/// declares them on ATTACH and RESIZE, and the daemon adopts them. +#[test] +fn a_client_declares_its_cell_size_and_the_session_geometry_follows() { + let Some(rig) = Rig::new() else { + eprintln!("skipping: no pty binary"); + return; + }; + // A 16x16 RGBA image (f=32, 1024 pixel bytes) placed with no c=/r=, so + // its cell extent is purely derived from the cell size. + let px = format!("{}==", "A".repeat(1366)); + let script = format!( + "printf 'drawn\\n'; \ + printf '\\033_Ga=t,q=2,i=77,f=32,s=16,v=16;{px}\\033\\\\'; \ + printf '\\033_Ga=p,q=2,i=77,p=1\\033\\\\'; \ + exec cat" + ); + rig.run("c", &script); + std::thread::sleep(Duration::from_millis(400)); + + let graphics = pty_terminal::GraphicsOptions { + cell: pty_terminal::CellSize { + width: 16, + height: 16, + }, + ..pty_terminal::GraphicsOptions::DEFAULT + }; + let h = TerminalHandle::attach( + rig.session("c"), + AttachOptions { + graphics: Some(graphics), + ..Default::default() + }, + ) + .expect("attach"); + assert!(h.wait_ready(Duration::from_secs(5))); + + // 16x16 pixels at a declared 16x16 cell is 1x1 cells, on both sides: the + // client's own terminal and the daemon's, which was told on ATTACH. + let deadline = Instant::now() + Duration::from_secs(5); + let state = loop { + let state = h.graphics(0); + if !state.placements.is_empty() { + break state; + } + assert!( + Instant::now() < deadline, + "no placement; screen:\n{}", + h.plain(Range::Full) + ); + h.wait_rev(h.rev(), Duration::from_millis(100)); + }; + assert!(state.cell_declared, "the client declared its cell size"); + assert_eq!(state.placements[0].cell_size, (1, 1)); + assert_eq!(state.placements[0].requested_cells, (0, 0)); + + // The daemon's own answer, via a PEEK-based read-only attach that never + // declares anything: the replay it gets carries the image, and the + // session's terminal is what resolved the geometry. + let (code, out, err) = rig.pty(&["peek", "c"]); + assert_eq!(code, 0, "peek failed: {out}{err}"); + assert!( + out.contains("i=77"), + "the session's replay carries the image: {out:?}" + ); + + // A later declaration travels on RESIZE and moves the derived extent. + h.set_cell_size(8, 16); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let state = h.graphics(0); + if state.placements.first().map(|p| p.cell_size) == Some((2, 1)) { + assert_eq!(state.cell, pty_terminal::CellSize { width: 8, height: 16 }); + break; + } + assert!(Instant::now() < deadline, "the new cell size never took"); + h.wait_rev(h.rev(), Duration::from_millis(100)); + } + + h.kill(); + rig.kill("c"); +} + #[test] fn attach_to_missing_session_is_an_error() { let root = std::env::temp_dir(); diff --git a/crates/pty-terminal/tests/input.rs b/crates/pty-terminal/tests/input.rs new file mode 100644 index 0000000..b74aa61 --- /dev/null +++ b/crates/pty-terminal/tests/input.rs @@ -0,0 +1,242 @@ +//! The child input encoder: the bytes a key, a mouse event, a focus change, +//! or a paste turns into depend on modes the *child* set, so every case here +//! sets that mode the way a child would and then encodes. + +use std::time::Duration; + +use pty_terminal::input::{self, Key, KeyAction, KeyEvent, Mods, MouseAction, MouseButton, MouseEvent}; +use pty_terminal::{SpawnOptions, TerminalActor, TerminalHandle}; + +fn actor() -> TerminalActor { + TerminalActor::new(10, 20, 0) +} + +// ── keys ── + +#[test] +fn decckm_decides_whether_an_arrow_is_csi_or_ss3() { + let mut a = actor(); + assert!(!a.modes().app_cursor); + assert_eq!(a.encode_key(&KeyEvent::press(Key::ArrowUp)), b"\x1b[A"); + + a.write(b"\x1b[?1h"); + assert!(a.modes().app_cursor, "?1 is tracked"); + assert_eq!( + a.encode_key(&KeyEvent::press(Key::ArrowUp)), + b"\x1bOA", + "application cursor keys" + ); + + a.write(b"\x1b[?1l"); + assert!(!a.modes().app_cursor); + assert_eq!(a.encode_key(&KeyEvent::press(Key::ArrowUp)), b"\x1b[A"); +} + +#[test] +fn a_plain_character_is_itself() { + let a = actor(); + assert_eq!(a.encode_key(&KeyEvent::typed(Key::A, "a", Some('a'))), b"a"); + assert_eq!( + a.encode_key(&KeyEvent::press(Key::C).with_mods(Mods::CTRL)), + b"\x03", + "ctrl+c is a control byte until the child asks for more" + ); +} + +/// The kitty keyboard protocol's associated text: the child asked for it +/// (`CSI > 17 u` = disambiguate + report associated), so the shifted text has +/// to reach it alongside the base key. A `KeyEvent` that folded shift into one +/// character could not express this. +#[test] +fn kitty_associated_text_carries_the_shifted_character() { + let mut a = actor(); + a.write(b"\x1b[>17u"); + assert_eq!(a.kitty_flags(), 17); + assert_eq!(a.modes().kitty_stack, vec![17]); + + let shift_a = KeyEvent::typed(Key::A, "A", Some('a')).with_mods(Mods::SHIFT); + assert_eq!( + a.encode_key(&shift_a), + b"\x1b[97;2;65u", + "base key 97 ('a'), shift modifier, associated text 65 ('A')" + ); + assert_eq!( + a.encode_key(&KeyEvent::typed(Key::A, "a", Some('a'))), + b"a", + "an unmodified key still goes through as text" + ); +} + +/// The alternate-key form (`CSI > 5 u` = disambiguate + report alternates): +/// the shifted key travels in the key field itself, `base:shifted`. +#[test] +fn kitty_alternates_carry_the_shifted_key() { + let mut a = actor(); + a.write(b"\x1b[>5u"); + let shift_a = KeyEvent::typed(Key::A, "A", Some('a')).with_mods(Mods::SHIFT); + let out = String::from_utf8(a.encode_key(&shift_a)).expect("utf8"); + assert!( + out.contains("97:65"), + "expected base:shifted in {out:?} (flags {})", + a.kitty_flags() + ); +} + +#[test] +fn a_release_reaches_the_child_only_when_it_asked_for_events() { + let mut a = actor(); + let release = KeyEvent { + action: KeyAction::Release, + ..KeyEvent::typed(Key::A, "a", Some('a')) + }; + assert!( + a.encode_key(&release).is_empty(), + "no release reporting by default" + ); + + // `CSI > 3 u` = disambiguate + report events. + a.write(b"\x1b[>3u"); + assert_eq!( + a.encode_key(&release), + b"\x1b[97;1:3u", + "key 97, no modifiers, event type 3 = release" + ); + + // A character key with neither text nor an unshifted codepoint has no + // identity the protocol can report, so it encodes to nothing. A + // consumer that sees an empty encoding for a character key is missing + // `KeyEvent::unshifted`. + let anonymous = KeyEvent { + action: KeyAction::Release, + ..KeyEvent::press(Key::A) + }; + assert!(a.encode_key(&anonymous).is_empty()); +} + +// ── mouse ── + +fn press() -> MouseEvent { + MouseEvent::press(MouseButton::Left, 3, 4) +} + +#[test] +fn no_tracking_reports_nothing_so_the_surface_keeps_the_event() { + let a = actor(); + assert_eq!(a.encode_mouse(&press()), None); + assert_eq!(a.encode_mouse(&MouseEvent::wheel(true, 3, 4)), None); + assert!(!a.modes().mouse_reporting()); +} + +/// X10 (`?9`) reports button presses and nothing else. A consumer that folded +/// it into "mouse tracking is on" would forward the wheel to a child that +/// never hears it — and lose its own scrolling. +#[test] +fn x10_reports_presses_but_never_the_wheel() { + let mut a = actor(); + a.write(b"\x1b[?9h"); + assert!(a.modes().mouse_9); + assert!(!a.modes().mouse_tracking(), "?9 is not wheel-reporting"); + assert!(a.modes().mouse_reporting()); + + assert_eq!(a.encode_mouse(&press()), Some(b"\x1b[M \x24\x25".to_vec())); + assert_eq!( + a.encode_mouse(&MouseEvent::wheel(true, 3, 4)), + None, + "the surface keeps the wheel" + ); + assert_eq!( + a.encode_mouse(&MouseEvent { + action: MouseAction::Release, + ..press() + }), + None, + "X10 has no release report" + ); +} + +#[test] +fn normal_tracking_reports_the_wheel() { + let mut a = actor(); + a.write(b"\x1b[?1000h"); + assert!(a.modes().mouse_tracking()); + assert!(a.encode_mouse(&press()).is_some()); + assert_eq!( + a.encode_mouse(&MouseEvent::wheel(true, 3, 4)), + Some(b"\x1b[M`\x24\x25".to_vec()), + "button 64 = wheel up" + ); +} + +#[test] +fn sgr_mode_reports_the_cell_the_event_names() { + let mut a = actor(); + a.write(b"\x1b[?1000h\x1b[?1006h"); + assert_eq!( + a.encode_mouse(&press()), + Some(b"\x1b[<0;4;5M".to_vec()), + "col 3 / row 4, 1-based in the report" + ); +} + +// ── focus and paste ── + +#[test] +fn focus_is_reported_only_under_1004() { + let mut a = actor(); + assert_eq!(a.encode_focus(true), None); + a.write(b"\x1b[?1004h"); + assert_eq!(a.encode_focus(true), Some(b"\x1b[I".to_vec())); + assert_eq!(a.encode_focus(false), Some(b"\x1b[O".to_vec())); + a.write(b"\x1b[?1004l"); + assert_eq!(a.encode_focus(false), None); +} + +#[test] +fn paste_is_bracketed_only_under_2004() { + let mut a = actor(); + assert_eq!( + a.encode_paste("a\nb"), + b"a\rb", + "without bracketing a newline becomes a carriage return" + ); + a.write(b"\x1b[?2004h"); + assert_eq!(a.encode_paste("a\nb"), b"\x1b[200~a\nb\x1b[201~"); +} + +#[test] +fn a_multi_line_paste_is_flagged_unsafe() { + assert!(input::paste_is_safe("just text")); + assert!(!input::paste_is_safe("rm -rf /\n")); + assert!( + !input::paste_is_safe("a\x1b[201~b"), + "a forged bracketed-paste end escapes the brackets" + ); +} + +// ── through the handle ── + +/// The handle path: the events are `Send`, the encoding happens on the actor +/// thread against the live terminal, and `send_*` is ordered with `write`. +#[test] +fn send_key_reaches_the_child_and_encode_key_agrees() { + let h = TerminalHandle::spawn("cat", &[], SpawnOptions::default()).expect("spawn"); + assert!(h.wait_ready(Duration::from_secs(2))); + + assert_eq!(h.encode_key(&KeyEvent::press(Key::ArrowUp)), b"\x1b[A"); + assert_eq!(h.encode_mouse(&press()), None, "no tracking, no report"); + + // `cat` echoes: what the child received comes back on the screen. + h.send_key(&KeyEvent::typed(Key::A, "a", Some('a'))); + h.send_key(&KeyEvent::press(Key::Enter)); + let grid = h + .wait_for(Duration::from_secs(5), |g| g.text().starts_with('a')) + .expect("the child got the key"); + assert!(grid.text().starts_with('a')); + + h.send_paste("pasted"); + let grid = h + .wait_for(Duration::from_secs(5), |g| g.text().contains("pasted")) + .expect("the child got the paste"); + assert!(grid.text().contains("pasted")); + h.kill(); +} diff --git a/crates/pty-terminal/tests/replay.rs b/crates/pty-terminal/tests/replay.rs index 09da72a..95da551 100644 --- a/crates/pty-terminal/tests/replay.rs +++ b/crates/pty-terminal/tests/replay.rs @@ -66,7 +66,7 @@ fn peek_never_prefixes_1049h() { for opts in [SerializeOpts::PEEK, SerializeOpts::PEEK_FULL] { let screen = a.serialize(opts); let normal = a.normal_replay().unwrap_or(""); - let body = pty_terminal::serialize::vt(a.terminal(), opts.scrollback); + let body = pty_terminal::serialize::vt(a.terminal(), opts.scrollback, a.cell_size()); // The normal screen comes first and the alternate one after it, but // no mode prefix is added on top. assert_eq!(screen, format!("{normal}{body}"), "a prefix was added"); diff --git a/crates/pty-terminal/tests/scrollback.rs b/crates/pty-terminal/tests/scrollback.rs new file mode 100644 index 0000000..049d226 --- /dev/null +++ b/crates/pty-terminal/tests/scrollback.rs @@ -0,0 +1,147 @@ +//! The scrollback promise: a terminal asked for N lines of history retains N +//! lines, and says so. +//! +//! libghostty's `max_scrollback` is a byte budget for the history page list, +//! not a line count — its own doc comment says "lines" — and the page list +//! evicts whole pages. Passing 10 000 for "10 000 lines" buys one page: 745 +//! rows at 80 columns, 456 at 200, 3 310 at 20. Every case here is about the +//! conversion that makes the declared line count true, and about the reported +//! numbers being the truth rather than the request. + +use pty_terminal::actor::{DEFAULT_SCROLLBACK, MAX_SCROLLBACK_BYTES}; +use pty_terminal::{Range, TerminalActor}; + +/// Feed `n` numbered lines, each short enough that it cannot wrap, so a row +/// written is a row of history and nothing here depends on reflow. +fn fill(a: &mut TerminalActor, n: usize) { + for i in 0..n { + a.write(format!("L{i}\r\n").as_bytes()); + } +} + +/// The oldest of 10 000 lines is still reachable, at the geometry the +/// terminal was built with. This is the substrate's promised replay window; +/// before the byte conversion it retained 745 of them. +#[test] +fn ten_thousand_lines_of_history_are_all_retained() { + let mut a = TerminalActor::new(24, 80, DEFAULT_SCROLLBACK); + fill(&mut a, 10_008); + + // 10 008 written lines plus the row the cursor sits on, all within the + // 10 024-row capacity of a 24-row terminal with 10 000 lines of history. + assert!( + a.buffer_length() >= 10_008, + "expected every line retained, got {} rows", + a.buffer_length() + ); + let text = a.plain(Range::Full); + let first = text.lines().next().unwrap_or(""); + assert_eq!(first, "L0", "the oldest line is still the oldest line"); + assert!(text.contains("\nL9999\n"), "and the newest are there too"); +} + +/// The same promise at widths where the byte cost of a row differs by 5x. A +/// budget derived from the line count has to scale with the width, or a wide +/// terminal keeps a fraction of the history a narrow one does. +#[test] +fn the_promise_holds_at_every_width() { + for cols in [20u16, 80, 200, 400] { + let mut a = TerminalActor::new(24, cols, 10_000); + fill(&mut a, 10_008); + assert!( + a.buffer_length() >= 10_008, + "{cols} columns: expected 10 008 rows, got {}", + a.buffer_length() + ); + assert_eq!( + a.plain(Range::Full).lines().next().unwrap_or(""), + "L0", + "{cols} columns: the oldest line was evicted" + ); + } +} + +/// A small scrollback is a line promise too, and the promise is a floor: the +/// history is a list of pages and libghostty never holds less than one, so a +/// terminal asked for 100 lines keeps at least 100 and in practice more. +/// Bounded, though — a terminal asked for 100 lines does not keep 5 000. +#[test] +fn a_small_scrollback_keeps_at_least_what_it_promised() { + let mut a = TerminalActor::new(24, 80, 100); + fill(&mut a, 5_000); + let text = a.plain(Range::Full); + for i in 4_900..4_999 { + assert!( + text.contains(&format!("L{i}\n")), + "the promised window must be there: L{i} is missing" + ); + } + assert!(text.contains("L4999"), "including the newest line"); + assert!(!text.contains("L0\n"), "and the far past is evicted"); + assert!( + a.buffer_length() < 5_000, + "a 100-line request must not keep 5 000 rows, got {}", + a.buffer_length() + ); +} + +/// `scrollback_used` and `scrollback_capacity` are the numbers a consumer +/// budgets against, so they have to describe the terminal rather than the +/// request. +#[test] +fn used_and_capacity_are_honest() { + let mut a = TerminalActor::new(24, 80, DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback(), DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback_capacity(), 24 + DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback_used(), 24, "an empty terminal is its viewport"); + + fill(&mut a, 10_008); + assert_eq!(a.scrollback_used(), a.buffer_length()); + // Capacity is a floor, not a ceiling (see its doc comment): what it + // promises has to actually be there. + assert!( + a.scrollback_used() >= 10_008, + "capacity is not honest if the rows are not there: {}", + a.scrollback_used() + ); +} + +/// A request beyond the memory bound is reported as what it is. libghostty +/// takes the budget at construction and exposes no setter, so this cannot be +/// fixed by asking for more later — it can only be told truthfully. +#[test] +fn a_request_beyond_the_memory_bound_reports_what_fits() { + let a = TerminalActor::new(24, 80, 10_000_000); + assert_eq!(a.scrollback_request(), 10_000_000); + assert_eq!(a.scrollback_bytes(), MAX_SCROLLBACK_BYTES); + assert!( + a.scrollback() < 10_000_000, + "the request cannot be met and must not be reported as met" + ); + assert!( + a.scrollback() > DEFAULT_SCROLLBACK, + "but the bound still buys more than the default: {}", + a.scrollback() + ); + assert_eq!(a.scrollback_capacity(), 24 + a.scrollback()); +} + +/// Widening the terminal makes each row cost more out of a budget fixed at +/// construction, so the retainable line count drops. The reported number +/// follows it instead of repeating the original promise. +#[test] +fn a_widened_terminal_reports_the_history_it_can_still_hold() { + let mut a = TerminalActor::new(24, 80, DEFAULT_SCROLLBACK); + assert_eq!(a.scrollback(), DEFAULT_SCROLLBACK); + let budget = a.scrollback_bytes(); + + a.resize(400, 24); + assert_eq!(a.scrollback_bytes(), budget, "the budget does not move"); + assert!( + a.scrollback() < DEFAULT_SCROLLBACK, + "a five-times wider row cannot hold the same line count: {}", + a.scrollback() + ); + assert_eq!(a.scrollback_capacity(), 24 + a.scrollback()); + assert_eq!(a.scrollback_request(), DEFAULT_SCROLLBACK); +} diff --git a/crates/pty/src/daemon/clients.rs b/crates/pty/src/daemon/clients.rs index 5c5829f..6b4beed 100644 --- a/crates/pty/src/daemon/clients.rs +++ b/crates/pty/src/daemon/clients.rs @@ -15,7 +15,7 @@ use std::sync::mpsc::Sender; use std::time::{Duration, Instant}; use pty_core::protocol::{ - Packet, MessageType, decode_peek, decode_size, encode_exit, encode_geometry, encode_screen, + Packet, MessageType, decode_cell, decode_peek, decode_size, encode_exit, encode_geometry, encode_screen, encode_status_response, }; use pty_core::registry::{self, MutateOptions}; @@ -128,6 +128,7 @@ impl Daemon { return; } let (rows, cols) = decode_size(payload); + self.adopt_cell_size(payload); // Read before negotiation: a smaller client shrinks the session to // its own size, which would then look like it had matched. let size_matched = rows == self.actor.rows() && cols == self.actor.cols(); @@ -220,9 +221,33 @@ impl Daemon { c.cols = cols; self.attach_counter += 1; c.attach_seq = self.attach_counter; + self.adopt_cell_size(payload); self.negotiate_size(); } + /// Take the cell pixel metrics a client declared on ATTACH or RESIZE. + /// + /// Only a client knows how big a cell is — it comes from a font on the + /// client's host, which this process may never see — and the session's + /// terminal needs them to answer the cell extent of a kitty placement + /// that did not name `c=`/`r=` itself. A payload without them (every + /// older client, the Node one included) changes nothing, and the terminal + /// keeps its deterministic fallback. + /// + /// The most recent declaration wins. Clients that draw cells of different + /// sizes cannot all be right about an implicit placement, and unlike rows + /// and cols there is nothing to negotiate: the metrics change no bytes and + /// no client's screen, only what this session reports as derived + /// geometry. + fn adopt_cell_size(&mut self, payload: &[u8]) { + if let Some((width, height)) = decode_cell(payload) { + self.actor.set_cell_size(pty_terminal::CellSize { + width: width as u32, + height: height as u32, + }); + } + } + /// node: src/server.ts:1040-1043 /// A client asked to leave: close its socket and take it off the books /// at once. diff --git a/crates/pty/src/daemon/lifecycle.rs b/crates/pty/src/daemon/lifecycle.rs index bb219a7..e3f904c 100644 --- a/crates/pty/src/daemon/lifecycle.rs +++ b/crates/pty/src/daemon/lifecycle.rs @@ -104,7 +104,8 @@ pub(crate) struct Daemon { /// How long the activity write waits after the first chunk of a burst. /// -/// node: src/server.ts `scheduleActivityPersist` (1 s), docs/vrs R14. +/// node: src/server.ts `scheduleActivityPersist` (1 s), the Node pty +/// repository's `docs/vrs` R14 (not this repository's `docs/vrs`). const ACTIVITY_PERSIST_DEBOUNCE: Duration = Duration::from_secs(1); /// 32 hex characters, Node's `randomBytes(16).toString("hex")`. @@ -305,7 +306,7 @@ pub fn run(cfg: DaemonConfig) -> Result { name, generation, cfg, - actor: TerminalActor::new(rows, cols, pty_terminal::actor::DEFAULT_SCROLLBACK), + actor: terminal_actor(rows, cols), master: pair.master, writer, child_pid, @@ -331,6 +332,26 @@ pub fn run(cfg: DaemonConfig) -> Result { Ok(daemon.serve()) } +/// The session's terminal, with kitty graphics on. +/// +/// A session is the durable owner of the child's screen, and since libghostty +/// keeps image state per screen, that includes the child's images: without it +/// the `SCREEN` a late client replays would carry placeholder cells naming +/// images nobody has (docs/decisions/0012-kitty-graphics-replay.md). The +/// storage limit is a cap, not an allocation — a session whose child never +/// transmits an image holds nothing and serializes exactly as before. +fn terminal_actor(rows: u16, cols: u16) -> TerminalActor { + let mut actor = TerminalActor::new(rows, cols, pty_terminal::actor::DEFAULT_SCROLLBACK); + if !actor.enable_graphics(pty_terminal::GraphicsOptions::DEFAULT) { + // The session still runs: text is unaffected and the actor rolled the + // storage limit back, so the only loss is images. Say so rather than + // leaving a client to wonder why its replay carries placeholder cells + // and no pictures. + daemon_warn!("pty: kitty graphics unavailable for this session"); + } + actor +} + fn spawn_pty_reader(mut reader: Box, tx: Sender) { std::thread::spawn(move || { let mut buf = [0u8; 16384]; diff --git a/docs/decisions/0012-kitty-graphics-replay.md b/docs/decisions/0012-kitty-graphics-replay.md new file mode 100644 index 0000000..3cb88e5 --- /dev/null +++ b/docs/decisions/0012-kitty-graphics-replay.md @@ -0,0 +1,210 @@ +# 0012 — kitty graphics are terminal state, and a replay carries them + +**Status:** accepted + +**Node behavior.** The Node daemon has no image state. `xterm-headless` has no +kitty graphics handler, so a `ESC _G ... ESC \` transmission is parsed and +discarded; the placeholder cells of a virtual placement survive only because +they are ordinary text with a foreground colour. A client that was connected +when the child wrote the image saw the raw bytes in `DATA` and its own terminal +kept them; a client that attaches afterwards gets a `SCREEN` +(`src/server.ts:962`, `@xterm/addon-serialize`) with the placeholder cells and +no image and no placement. Nothing in the Node daemon can answer "which images +does this session hold, and where are they". + +**Rust behavior.** Graphics are terminal state, off by default and bounded when +on: + +- `TerminalActor::enable_graphics` (`SpawnOptions::graphics` / + `AttachOptions::graphics`) sets a byte limit on libghostty's image storage + and installs a PNG decoder for `f=100`. Without it the storage limit stays + zero, which is libghostty's own "protocol disabled": a child cannot make a + terminal hold images its owner never asked for. The file, temporary-file, + and shared-memory transmission media stay disabled, so a child cannot name a + path the owner did not authorize; inline transmission only. +- One number bounds the state and the wire: `graphics::MAX_STORAGE_BYTES` + (32 MiB), which `enable_graphics` and `set_graphics_storage_limit` clamp to + and which a replay carries in full. A smaller replay cap would have made a + supported state — an image the terminal accepted and reports — one that a + late client can never be given, which is the exact failure this record is + about. A caller asking for more gets the bound and can see it in + `graphics_options()`. +- Cell pixel metrics come from whoever draws the cells and travel on the wire. + A placement that named neither `c=` nor `r=` gets its cell extent from the + image's pixel size divided by the cell size, so that size has to be the + client's real one: it comes from a font on the client's host, which a + session daemon may never see. `AttachOptions::graphics`'s cell size is + appended to ATTACH, `TerminalHandle::set_cell_size` sends it on RESIZE + (`encode_attach_with_cell` / `encode_resize_with_cell`: four optional bytes + after the existing rows/cols, so every older reader — the Node daemon + included — takes the size it always took and ignores the rest), and the + daemon adopts it (`clients.rs`, `adopt_cell_size`). Undeclared is explicit, + not silent: geometry uses `CellSize::FALLBACK` (8x16) and + `GraphicsState::cell_declared` is false. +- `TerminalActor::graphics_state(scroll_offset)` / + `TerminalHandle::graphics(scroll_offset)` answer with owned values + (`GraphicsState`, `ImageDesc`, `Placement`, `SourceRect`, + `PlacementPosition`) for the same window `snapshot(scroll_offset)` reads, so + a grid and a graphics state taken with the same offset line up cell for cell. + `image_bytes(id)` copies the pixels once, on request, keyed by + `ImageDesc::generation`. +- `serialize::vt` appends a graphics block after the cursor move: one + `a=t` transmission per image (chunked at 4096 base64 bytes, `s=`/`v=` for raw + formats, `o=z` for a compressed one), then `a=p,U=1` per virtual placement + and `DECSC` + `CUP` + `a=p` + `DECRC` per cursor-positioned one. The source + rectangle is always emitted in full (`x=`,`y=`,`w=`,`h=`): `w=`/`h=` default + to "the whole image", so omitting them replays a cropped placement as the + wrong pixels at the right size. A virtual placement is emitted wherever its + cells are, history included, because its command carries no position at + all; a cursor-positioned one needs a cell in the active area, and one whose + top has scrolled above it is anchored at row 0 with its crop advanced by + the rows that are gone. It is empty + for a terminal with no graphics, so a session that never sent an image + serializes exactly as it did before + (`crates/pty-terminal/tests/graphics.rs::a_replay_without_graphics_is_unchanged`). +- The daemon's own terminal has graphics on + (`crates/pty/src/daemon/lifecycle.rs`, `terminal_actor`). The session is the + durable owner of the child's screen, so it is the durable owner of the + child's images; a daemon without them would serve a `SCREEN` whose + placeholder cells name images no client can have. The limit is a cap, not an + allocation: a session whose child never transmits holds nothing. + +**Why.** Two other shapes were available and are worse. Passing the child's +graphics bytes through to an outer terminal cannot work for an embedder that +draws a sub-rectangle: the child's coordinates are its own, and the embedder +clips, pans, and draws chrome around it. Keeping the state in each client +cannot work either: a client that attaches later never sees the `DATA` that +carried the image, so the state has to be reconstructible from the replay the +daemon already sends. Re-emitting the storage in the protocol the child used is +the smallest thing that makes a late client and a live client hold the same +images. + +Two deviations from libghostty's own API are deliberate: + +- Positions are resolved against the window that was asked for, not the live + viewport. A cursor-positioned placement's own rectangle + (`PlacementIteration::rect` + `point_from_grid_ref(PointSpace::Screen)`) + answers for any window and reports a negative row for a placement whose top + has scrolled above it; `viewport_pos` answers only for the live viewport and + reports nothing at all for a placement above it, which would make a + scrolled-back reader lose exactly the images it wants. +- libghostty reports no viewport position for a virtual placement — correctly, + because a virtual placement has none: it is wherever its placeholder cells + are. Those cells each name their own image row and column, so + `graphics::scan_placeholders` decodes them from the grid (one pass over the + window, never the scrollback) and reports both the visible cell box and the + image cell indices it covers. That is what makes a partially scrolled image + answerable: a two-row image scrolled by one row reports one visible row, + `cell_row = 1`, and `origin_row = -1`. +- `TerminalActor::reset` re-enables graphics after `RIS`. libghostty's reset + restores its defaults, which include no storage and no cell metrics; since + `reset` is what precedes a `SCREEN` replay, a terminal that lost its storage + there would reject the very images the replay carries. + +**Client effect.** A consumer that opts in can ask, at any time and from any +client, for the image bytes, the placement identity (`image_id` + +`placement_id`), the resolved source crop, the rendered pixel and cell size, +and the placement's position in a chosen window; `HandleEvent::Graphics(gen)` +says when the storage content changed, and `graphics_generation()` keys a +texture cache. Scrolling and resizing move placements without changing the +generation, so a dirty frame still re-reads positions. The alternate screen has +its own storage, as in the protocol: a full-screen program's images are not the +primary screen's, and neither set is lost when the child switches. + +Residual differences a consumer can observe: + +1. `GraphicsState::images` lists the images that have at least one placement. + libghostty exposes lookup by id, not enumeration, so an image transmitted + and never placed is not listed (it is also not drawable). +2. Grayscale images (`Gray`, `GrayAlpha`) have no kitty `f=` value and are not + re-emitted into a replay. Every PNG this terminal accepts is expanded to + 8-bit RGBA before libghostty stores it (a grayscale PNG decodes as + `Grayscale`/`GrayscaleAlpha`, which `PngDecoder` widens itself — rejecting + it instead would drop every monochrome plot a child sends), so the + grayscale variants are unreachable in practice and kept as a typed dead end + rather than a silent conversion. +3. Cell metrics are the newest declaration, not a negotiation. Two clients + whose fonts differ cannot both be right about a placement that left its + size implicit, and unlike rows and cols there is nothing to reconcile: the + metrics change no bytes and no client's screen, only the derived geometry + this session reports. A client that draws pixels should declare its own and + read `GraphicsState::cell_declared` rather than trust the fallback. +4. A cursor-positioned placement that has scrolled *entirely* above the + active area is not re-emitted: there is no cell left to put the cursor + on. One that is partly on screen is re-emitted clipped, so what a late + client gets is what the source terminal is showing. Virtual placements + have no such limit at all, which is one more reason they are the preferred + form. +5. A replay of the alternate screen brackets the normal half in + `ESC[?1049l` / `ESC[?1049h` + `ESC[H`. Node's payload writes its normal + half after `ESC[?1049h`, so a Node client's normal buffer takes it while + already switched; that is invisible for text but would put the normal + screen's images in the client's alternate storage, since kitty storage is + per screen. +6. `SIXEL` and the iTerm2 protocol are not read at all. Neither offers a + placement contract an embedder can reproject into a sub-rectangle. + +**Test.** `crates/pty-terminal/tests/graphics.rs` — twenty-six cases driven with +the exact bytes OMP writes (`packages/tui/src/terminal-capabilities.ts` +`encodeKittyTransmit`, `packages/tui/src/kitty-graphics.ts` +`encodeKittyVirtualPlacement` / `encodeKittyPlaceholderGrid`): opt-in, live +write, source crop, scroll, scrollback window, resize, alternate screen, +child-sent delete of a placement and of an image, `clear_graphics`, zeroed +limit, the replay cases +(`a_late_client_reconstructs_the_image_from_the_replay_alone`, +`a_cursor_positioned_placement_replays_at_its_cell`), the handle path +(`a_spawned_child_that_draws_an_image_is_queryable_through_the_handle`, which +also pins `HandleEvent::Graphics`), the bound +(`an_image_far_larger_than_three_mib_still_replays` — a 4 MiB image +transmitted in 4096-byte chunks, stored and then recovered byte for byte from +the replay alone — and +`the_storage_limit_is_clamped_to_what_a_replay_can_carry`), and the cell +metrics (`an_implicit_placement_takes_its_extent_from_the_declared_cell`, +`a_declared_cell_does_not_move_an_explicit_placement`). + +Replay fidelity has a case per failure it can have, each of which fails on the +shape that preceded it: `a_cropped_placement_keeps_its_crop_through_a_replay` +(the crop survives, not the whole image measured from its origin), +`a_virtual_placement_scrolled_into_history_still_replays`, +`a_partially_scrolled_direct_placement_replays_clipped`, +`a_grayscale_png_is_stored_as_rgba`, +`a_palette_foreground_names_a_placeholder_image`, +`a_bare_continuation_cell_inherits_row_column_and_id_high_byte`, +`raising_the_storage_limit_keeps_the_cell_size_and_decodes_png`, +`the_storage_limit_alone_turns_graphics_fully_on`, and +`a_replay_from_the_alt_screen_puts_the_normal_screens_images_on_the_normal_screen` +(the normal screen's image is there after the full-screen program exits, and +the alternate screen replays character for character). Unit tests for the +base64 encoder and the diacritic table live in `graphics.rs`; the wire form is +pinned in `crates/pty-core/tests/protocol.rs` +(`attach_can_declare_a_cell_size_without_changing_the_size_it_carries`, +`resize_can_declare_a_cell_size`, `a_plain_size_payload_declares_no_cell`). + +End to end, against a real session daemon: +`crates/pty-terminal/tests/handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` +— a child transmits a PNG, places it virtually, and writes a placeholder cell; +a `TerminalHandle` attaches only afterwards, so it never sees that `DATA`, and +still reads the decoded pixels (`[255, 0, 0, 255]`), the placement identity +(image 4242, placement 7), and its cell from the `SCREEN` replay alone. The +same test then reconnects and asserts the image and its cell come back, which +is what the graphics-preserving `reset` is for. Its sibling +`a_client_declares_its_cell_size_and_the_session_geometry_follows` attaches +with a declared 16x16 cell to a session whose child placed a 16x16 image with +no `c=`/`r=`, asserts the extent is 1x1 cell, then declares 8x16 and asserts +it becomes 2x1. + +No gated `_node` / `_rust` conformance pair exists, and none can: the Node side +has no graphics state to compare against. The difference is not "Rust renders +this differently" but "Rust has state Node does not", so the record stands on +the Rust tests plus the Node source above. + +**Migration / negotiation.** None. A client's graphics are off unless it asks +(`SpawnOptions::graphics` / `AttachOptions::graphics`); the daemon's are on, +and a session whose child never transmits an image serves the same `SCREEN` +bytes it did before. A client that does not opt in is unaffected by a session +that holds images: the extra `ESC _G ... ESC \` in its replay is an APC string, +which a terminal that does not implement the protocol ignores. A Node daemon +feeding a Rust client still works — the client then has only what the Node +`SCREEN` carries (placeholder cells, no images), which is exactly what a Node +client would have. Full replay fidelity needs the daemon side to be the Rust +one. diff --git a/docs/decisions/0013-scrollback-is-a-line-promise.md b/docs/decisions/0013-scrollback-is-a-line-promise.md new file mode 100644 index 0000000..662099a --- /dev/null +++ b/docs/decisions/0013-scrollback-is-a-line-promise.md @@ -0,0 +1,110 @@ +# 0013 — scrollback is a line promise, and libghostty counts bytes + +**Status:** accepted + +**Node behavior.** Node's daemon passes `scrollback: 10000` to +`xterm-headless` (`src/server.ts:333-338`), where it is a line count: the +buffer holds 10 000 lines of history and evicts the 10 001st line. `pty stats` +reports `scrollbackUsed` (`buffer.active.length`, `src/server.ts:1128`) against +`scrollbackCapacity` (`rows + scrollback`), and that capacity is a ceiling the +buffer never exceeds. + +**Rust behavior.** `TerminalActor::new(rows, cols, scrollback)` still takes a +line count, and now converts it to the byte budget libghostty actually wants. + +libghostty's `Options::max_scrollback` is documented as "maximum number of +lines to keep in scrollback history" and is not: it is a byte budget for the +history page list, and the list evicts whole pages. Measured against +libghostty-vt 0.2.1, a terminal given `max_scrollback: 10_000` and fed 10 008 +short lines keeps: + +| columns | rows retained | oldest line surviving | +| --- | --- | --- | +| 20 | 3 310 | `L6698` | +| 80 | 745 | `L9263` | +| 200 | 456 | `L9552` | +| 400 | 149 | `L9859` | + +The retained count scales inversely with the width, which a line count cannot +do, and doubling the number to 20 000 changes nothing at 80 columns — both are +smaller than one page, and one page is the floor. Passing a line count straight +through therefore delivered 7% of the promised history at 80 columns, and the +number `scrollback_capacity()` reported (`24 + 10_000`) described nothing that +existed. + +The conversion is `SCROLLBACK_ROW_OVERHEAD + SCROLLBACK_BYTES_PER_COL * cols` +per line (256 + 16), against a measured cost of ~838 bytes per row at 80 +columns and ~1 804 at 200 — roughly 1.8x headroom, because the page list rounds +up to whole pages and because a row of styled or multi-codepoint cells costs +more than a plain one. The total is capped at `MAX_SCROLLBACK_BYTES` (64 MiB, +which is 10 000 lines of a 400-column terminal). + +Three reads describe the result instead of the request: + +- `scrollback()` — lines actually retainable at the current width. +- `scrollback_request()` — what the owner asked for, met or not. +- `scrollback_bytes()` — the budget libghostty holds the history in. + +**Why.** The alternative was to leave the number as libghostty takes it and +weaken the promise to "up to N lines", which is what the first Fractal test did +when it asserted `used <= capacity` — an assertion that passes when 6 398 of +10 008 lines have been thrown away. A replay window is a product promise: a +consumer that shows history decides what to keep on the basis of that number, +and the honest options were to meet it or to publish a smaller one. Meeting it +costs memory that is bounded, documented, and only touched when the history +actually fills; publishing a smaller one would have meant every consumer +carrying its own conversion from lines to whatever libghostty's number means +this release. + +libghostty exposes the budget only in `Options`, with no setter, so it is fixed +at construction. That is the source of the one remaining gap below. + +**Client effect.** A session asked for 10 000 lines retains 10 000 lines at the +width it was created with, and `stats` reports numbers that are true. Memory +per session with the default 10 000 lines: 14.6 MiB of budget at 80 columns +(33 MiB at 200, 63.5 MiB at 400), against ~1 MiB before — the budget is address +space the page list fills only as history accumulates, so an idle session pays +nothing. + +Residual differences a consumer can observe: + +1. `scrollback_capacity()` is a guaranteed minimum, where Node's is a ceiling. + libghostty never holds less than one page, so a terminal asked for a small + scrollback keeps more than it promised: a 100-line request at 80 columns + retains about 1 000 rows. The promise is met and then some; code that + treated the number as an upper bound on `scrollback_used` has to stop. +2. Widening a terminal reduces the line count it can retain, because the byte + budget is fixed at construction and a wider row costs more. `scrollback()` + and `scrollback_capacity()` follow the width down; `scrollback_request()` + keeps saying what was asked for. A terminal created at 80 columns and + widened to 400 holds about a fifth of the lines. Fixing this needs either a + libghostty setter for the budget (upstream) or budgeting for the widest + plausible width at construction (10 000 lines at 1 000 columns is 154 MiB + per session, which is not worth it). +3. A request whose budget exceeds `MAX_SCROLLBACK_BYTES` is clamped, and + `scrollback()` reports the clamped line count rather than the request. + +**Test.** `crates/pty-terminal/tests/scrollback.rs` — six cases, all of which +fail on the pass-through: +`ten_thousand_lines_of_history_are_all_retained` (the oldest of 10 008 lines is +still `L0`, at 24x80 with the default scrollback), +`the_promise_holds_at_every_width` (the same at 20, 80, 200 and 400 columns, +where the per-row cost differs by 5x), +`a_small_scrollback_keeps_at_least_what_it_promised` (the promised window is +present and the far past is gone), +`used_and_capacity_are_honest`, +`a_request_beyond_the_memory_bound_reports_what_fits`, and +`a_widened_terminal_reports_the_history_it_can_still_hold`. + +Every line written by these tests is short enough that it cannot wrap, so no +result here depends on reflow: a row written is a row of history. + +No gated `_node` / `_rust` conformance pair exists for the byte conversion +itself — it is an implementation detail of reaching Node's behaviour, not a +deviation from it. The observable deviations are the three above. + +**Migration / negotiation.** None. `TerminalActor::new` and +`SpawnOptions`/`AttachOptions` still take a line count and now honour it; a +consumer reading `scrollback_capacity()` as a ceiling should read it as a floor +(residual 1), and one that resizes should re-read `scrollback()` rather than +assume the original number (residual 2). diff --git a/docs/parity.md b/docs/parity.md index cd60328..ab8c64b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -489,7 +489,8 @@ looked. | Mixed-fleet rig (section 11) | M | | Packaging and completions (section 14) | M | -Sources for this map: the Node source and its tests at `500eab2`, `docs/vrs`, +Sources for this map: the Node source and its tests at `500eab2`, and — all in +the `compoundingtech/pty` (Node) repository — its `docs/vrs`, `docs/disk-layout.md`, `docs/client.md`, `docs/testing.md`; this repository at `e4d6cda`; the `st2`, `pty-relay`, `deskset`, `ding`, `smalltalk`, and `evals` call sites; issues #1, #3, #4 here and the open issues and PRs on the Node diff --git a/docs/vrs/01-images/requirements.md b/docs/vrs/01-images/requirements.md new file mode 100644 index 0000000..af56ca8 --- /dev/null +++ b/docs/vrs/01-images/requirements.md @@ -0,0 +1,44 @@ +# Terminal images — Requirements + +## Context + +This node defines the durable terminal-image contract for pty-rust. The implementation uses the Kitty graphics protocol and libghostty. The rationale for the protocol and replay shape remains in [decision 0012](../../decisions/0012-kitty-graphics-replay.md). + +## Assumptions + +- **PTY.IMG-A01 Typed terminal ownership:** The terminal actor is the single owner of parsed image state. Clients read typed image descriptions, bytes, placements, and generations rather than untrusted escape sequences. +- **PTY.IMG-A02 Cell-relative composition:** An embedding client composes terminal images in character-cell coordinates and needs the same effective cell metrics that the terminal used. + +## Constraints + +- **PTY.IMG-C01 Library enumeration:** libghostty does not enumerate an image that was transmitted but never placed; it exposes image lookup by identifier. +- **PTY.IMG-C02 Pixel formats:** Kitty has no `f=` value for grayscale pixels, so a grayscale raw-pixel transmission cannot be represented as a typed replay image without conversion. +- **PTY.IMG-C03 Scrolled direct placements:** A cursor-positioned placement that is entirely above the active area cannot be re-emitted because restoring its position requires a cursor cell inside the active area. +- **PTY.IMG-C04 Protocol scope:** SIXEL and iTerm2 output do not provide the typed placement contract required for client-side reprojection and are outside this node. +- **PTY.IMG-C05 Cross-runtime conformance:** The Node implementation has no equivalent typed image state, so graphics behavior has no Node/Rust conformance pair. + +## Acceptable Tradeoffs + +- **PTY.IMG-T01 Complete bounded replay:** A session may retain up to 32 MiB of image state, and a replay may carry all retained bytes. The limit is a cap, not an eager allocation. +- **PTY.IMG-T02 Latest cell declaration wins:** Cell metrics use the newest client declaration rather than multi-client negotiation. + +## Requirements + +### Must bound accepted image state + +- **PTY.IMG-R01 Opt-in bounded storage:** Image storage is disabled until an owner enables it. The effective storage limit must not exceed 32 MiB and must be observable through the typed terminal API. +- **PTY.IMG-R08 Inline-only transmission:** The terminal must accept image bytes only from inline transmissions. File, temporary-file, and shared-memory media must remain disabled so a child cannot name an owner-unapproved path. +- **PTY.IMG-R09 Bounded PNG normalization:** PNG (`f=100`) input must be bounded before decoding and normalized to 8-bit RGBA before it enters retained image storage. + +### Must survive replay + +- **PTY.IMG-R02 Complete late-client replay:** Every retained image reported for a placement must be carried in full by ATTACH and PEEK replay. The replay must not impose a smaller byte cap than the reported retained state. +- **PTY.IMG-R04 Crop fidelity:** Replay must preserve the resolved, image-bounded source rectangle and offsets for each placement. +- **PTY.IMG-R06 Image-free compatibility:** A session that has accepted no image must retain the pre-image replay byte shape; its image replay block is empty. +- **PTY.IMG-R07 Screen and reset lifecycle:** Normal and alternate screens must retain separate image state. Terminal reset must preserve the configured graphics capability needed to replay retained state into its owning screen. + +### Must support cell-relative composition + +- **PTY.IMG-R03 Window-relative placements:** A graphics read and a grid read taken at the same scroll offset must describe the same terminal window and align cell for cell. Virtual placements follow bounded placeholder cells; direct placements use their screen-space rectangle. Placements outside the requested window must not be reported as visible. +- **PTY.IMG-R05 Declared cell metrics:** ATTACH and RESIZE may append backward-compatible cell width and height fields. An absent declaration must stay explicit and use a deterministic 8×16 fallback. Explicit placement coordinates must not move when cell metrics change. +- **PTY.IMG-R10 Observable content generation:** Image-content changes must advance an observable generation used to fence cached pixels. Scroll and resize may reproject placements without advancing the content generation. diff --git a/docs/vrs/01-images/spec.md b/docs/vrs/01-images/spec.md new file mode 100644 index 0000000..c6f73c0 --- /dev/null +++ b/docs/vrs/01-images/spec.md @@ -0,0 +1,207 @@ +# Terminal images — Specification + +## Status + +Implemented. The session daemon's terminal holds images, `serialize::vt` +replays them, and the embedding handle reads them. This node is lazy — it +stands on [./requirements.md](./requirements.md) alone and requires no parent +`docs/vrs` artifacts. The rationale for the shape below is +[decision 0012](../../decisions/0012-kitty-graphics-replay.md); this +specification states the mechanism and does not restate the argument. + +## Scope + +This specification defines how a pty session holds, bounds, reports, and +replays the terminal images a child draws with the Kitty graphics protocol: +storage admission and its byte bound, placement and crop geometry resolved +against a requested window, the cell pixel metrics the geometry needs, the +replay wire form, and the storage lifecycle across reset and screen switches. + +It does not define rendering — a consumer decides how, where, and whether to +draw what it reads. It does not define the SIXEL or iTerm2 protocols, which +are not read. It does not define Node parity for images: the Node daemon +holds no image state at all, so there is nothing to compare against +(see the Node parity map in [docs/parity.md](../../parity.md) for the +surfaces that do compare). + +Requirements `PTY.IMG-R01` through `PTY.IMG-R10` are in +[./requirements.md](./requirements.md) and are cited inline below. + +## Module map + +| Concern | Source | +| --- | --- | +| Admission, bound, cell metrics, generation | `crates/pty-terminal/src/actor.rs` — `TerminalActor::enable_graphics`, `set_graphics_storage_limit`, `graphics_options`, `set_cell_size`, `graphics_generation`, `reset` | +| Storage bound and options | `crates/pty-terminal/src/graphics.rs` — `MAX_STORAGE_BYTES` (32 MiB), `GraphicsOptions` | +| Window read | `crates/pty-terminal/src/graphics.rs` — `read`, `screen_origin`, `scan_placeholders`, `placeholder_cell`, `PLACEHOLDER` (U+10EEEE), `ROWCOLUMN_DIACRITICS` | +| Owned reported values | `crates/pty-terminal/src/graphics.rs` — `GraphicsState`, `ImageDesc`, `Placement`, `PlacementPosition`, `PlaceholderRect`, `SourceRect`, `image_bytes` | +| Replay emission | `crates/pty-terminal/src/graphics.rs` — `replay`, `transmit`, `place_virtual`, `place_direct`, `clip_top`, `placement_params`; called from `crates/pty-terminal/src/serialize.rs` — `vt` | +| PNG normalization | `crates/pty-terminal/src/graphics.rs` — `PngDecoder`, `expand`, `install_png_decoder`, `MAX_DECODED_PNG_BYTES` (64 MiB) | +| Cell metric wire | `crates/pty-core/src/protocol.rs` — `encode_attach_with_cell`, `encode_resize_with_cell`, `decode_cell` | +| Client and daemon adoption | `crates/pty/src/daemon/clients.rs` — `adopt_cell_size`; `crates/pty/src/daemon/lifecycle.rs` — `terminal_actor` | +| Embedding surface | `crates/pty-terminal/src/handle.rs` — `TerminalHandle::graphics`, `image_bytes`, `set_cell_size`, `graphics_generation`, `HandleEvent::Graphics` | + +## Admission and bound + +`TerminalActor::enable_graphics(GraphicsOptions)` sets libghostty's image +storage limit and installs the `f=100` PNG decoder. Until an owner asks — via +`SpawnOptions::graphics`, `AttachOptions::graphics`, or a non-zero +`set_graphics_storage_limit` — the limit stays zero, which is libghostty's own +"protocol disabled" (`PTY.IMG-R01`). Any requested limit is clamped to +`graphics::MAX_STORAGE_BYTES` and the effective value is readable through +`graphics_options()`, so a caller that asked for more can see what it got. +Zeroing the limit turns the protocol back off. + +File, temporary-file, and shared-memory transmission media are left disabled, +so a child can only send bytes and never name a path (`PTY.IMG-R08`). PNG +(`f=100`) payloads are expanded to 8-bit RGBA before storage sees them, and +rejected above `MAX_DECODED_PNG_BYTES` (`PTY.IMG-R09`). + +The same number bounds the state and the wire: there is no second, smaller +replay cap, so a retained image that `graphics_state` reports for a placement +is carried in full to a late client, whatever its size up to the bound +(`PTY.IMG-R02`). Reachability is scoped to placed images and to formats the +protocol can name — an image transmitted and never placed is not enumerated +(`PTY.IMG-C01`), and the grayscale pixel formats have no kitty `f=` value +(`PTY.IMG-C02`). The limit is a cap, not an allocation — a session whose +child never transmits holds nothing. + +## Window read + +`TerminalActor::graphics_state(scroll_offset)` (and +`TerminalHandle::graphics(scroll_offset)`) answers for the same window +`snapshot(scroll_offset)` reads, so a grid and a graphics state taken with one +offset line up cell for cell (`PTY.IMG-R03`). A cursor-positioned placement is +located by its own screen-space rectangle, which answers for any window and +reports a negative origin row when its top has scrolled above the active area. +A virtual placement has no position of its own, so `scan_placeholders` decodes +the `U+10EEEE` placeholder cells of the window — one pass, never the +scrollback — and reports both the visible cell box and the image cell indices +covered; a bare continuation cell inherits row, column, and image-id high byte +from its predecessor. + +Source rectangles are resolved on read and on replay: the protocol's `w=`/`h=` +default of "the whole dimension" is expanded and then clamped to the image, so +a crop is a concrete rectangle everywhere it is reported (`PTY.IMG-R04`). + +`image_bytes(id)` copies the pixels once, on request, keyed by +`ImageDesc::generation`. + +## Replay wire form + +`serialize::vt(term, scrollback, cell)` appends a graphics block after the +cursor move. The block is: + +```text +first transmission frame ESC _G a=t,q=2,i=,f=[,s=,v=][,o=z][,m=1]; ESC \ +continuation frame ESC _G q=2,m=<1|0>; ESC \ +virtual placement ESC _G a=p,U=1,q=2 ESC \ +cursor-positioned ESC 7 ESC [;H ESC _G a=p,q=2 ESC \ ESC 8 + + ,i=[,p=][,c=][,r=],x=,y=,w=,h=[,X=][,Y=][,z=] +``` + +`q=2` suppresses the terminal's replies on every command, so a replay is +silent. One `a=t` transmission per image, its base64 payload chunked at 4096 +bytes: the first frame carries the full parameter list and gains `,m=1` only +when another chunk follows, and each continuation frame carries just `q=2` +and `m=`, ending with `m=0`. A payload that fits one chunk therefore has no +`m=` at all. `s=`/`v=` carry the pixel dimensions of a raw format and `o=z` +marks a zlib-deflated payload. + +Placement commands carry no payload, so they have no `;` and no terminating +data: the parameter list runs straight into `ESC \`. `p=`, `c=`, `r=`, `X=`, +`Y=`, and `z=` are emitted only when nonzero, since zero is the protocol's +own default for each. The source rectangle is the exception and is always +emitted in full (`x=`, `y=`, `w=`, `h=`), because `w=`/`h=` default to "the +whole image", so omitting them would replay a cropped placement as the wrong +pixels at the right size (`PTY.IMG-R04`). + +A virtual placement is emitted wherever its cells are, history included, since +its command carries no position. A cursor-positioned one needs a cell in the +active area, so it is bracketed by `ESC 7` / `ESC 8` around a `CUP` to that +cell: one whose top has scrolled partly above it is anchored at row 0 with its +crop advanced by the rows that are gone (`clip_top`, which also shrinks `r=`), +and one that has scrolled entirely above it is not re-emitted +(`PTY.IMG-C03`). For a terminal holding no images the block is empty, so a +session that never sent one serializes exactly the bytes it did before +(`PTY.IMG-R06`). + +## Cell metric wire form + +Cell pixel metrics come from the font on the client's host, which a session +daemon may never see, so they are declared rather than guessed +(`PTY.IMG-R05`). The size payload of ATTACH and RESIZE grows an optional +4-byte suffix: + +```text +byte 0..1 rows u16 big-endian +byte 2..3 cols u16 big-endian +byte 4..5 cell_width u16 big-endian (optional) +byte 6..7 cell_height u16 big-endian (optional) +``` + +`encode_attach_with_cell` and `encode_resize_with_cell` write the 8-byte form; +`encode_attach` and `encode_resize` write the plain 4-byte one. Every reader +of a size payload takes rows and cols from the first four bytes and the frame +carries its own length, so a daemon that predates the suffix — the Node one +included — reads the size it always read and ignores the rest. `decode_cell` +returns `None` for a payload shorter than 8 bytes or a degenerate zero. + +The daemon adopts a declaration on either message (`adopt_cell_size` → +`TerminalActor::set_cell_size`); the newest declaration wins. Undeclared is +explicit, not silent: geometry falls back to `CellSize::FALLBACK` (8x16) and +`GraphicsState::cell_declared` is false. A declared cell changes only derived +geometry — a placement that named neither `c=` nor `r=` takes its cell extent +from the image's pixel size divided by the cell size — and never moves a +placement that declared its own extent. + +## Lifecycle + +The session daemon's own terminal has images on +(`lifecycle.rs`, `terminal_actor`, `GraphicsOptions::DEFAULT`): the session is +the durable owner of the child's screen, so it is the durable owner of the +child's images; a daemon without them would serve a replay whose placeholder +cells name images no client can have. When enabling fails the daemon warns and +serves text as before. + +`TerminalActor::reset` re-enables images after `RIS` (`PTY.IMG-R07`). +libghostty's reset restores its own defaults, which include no storage and no +cell metrics, and `reset` is what precedes a replay — a terminal that lost its +storage there would reject the very images the replay is about to carry. + +Storage is per screen, as in the protocol: the alternate screen holds its own +images, so a full-screen program's are not the primary screen's and neither +set is lost when the child switches. A replay taken from the alternate screen +brackets the normal half so that the normal screen's images land in the normal +screen's storage (`PTY.IMG-R07`). + +Content change is signalled once, by a counter: `HandleEvent::Graphics(gen)` +and `graphics_generation()` key a texture cache (`PTY.IMG-R10`). Scrolling and +resizing move placements without bumping the generation, so a dirty frame +re-reads positions but does not re-upload pixels; a new transmission and a +child-sent delete both bump it. + +## Validation + +Test names are function names in the files given. `graphics.rs` and +`handle.rs` are under `crates/pty-terminal/tests/`, `protocol.rs` under +`crates/pty-core/tests/`. + +| Requirement | Owning source | Executable evidence | +| --- | --- | --- | +| `PTY.IMG-R01` | `actor.rs`, `graphics.rs` | `graphics.rs::graphics_are_off_until_the_owner_asks`, `zeroing_the_limit_turns_the_protocol_off`, `the_storage_limit_alone_turns_graphics_fully_on`, `the_storage_limit_is_clamped_to_what_a_replay_can_carry`, `raising_the_storage_limit_keeps_the_cell_size_and_decodes_png` | +| `PTY.IMG-R02` | `graphics.rs` (`replay`, `transmit`), `serialize.rs` | `graphics.rs::a_late_client_reconstructs_the_image_from_the_replay_alone`, `an_image_far_larger_than_three_mib_still_replays`, `a_cursor_positioned_placement_replays_at_its_cell`, `handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` | +| `PTY.IMG-R03` | `graphics.rs` (`read`, `screen_origin`, `scan_placeholders`) | `graphics.rs::an_omp_write_gives_image_bytes_placement_identity_crop_and_position`, `scrolling_moves_the_placement_and_scrollback_still_finds_it`, `a_resize_keeps_the_image_and_reprojects_it`, `a_virtual_placement_scrolled_into_history_still_replays`, `a_partially_scrolled_direct_placement_replays_clipped`, `a_palette_foreground_names_a_placeholder_image`, `a_bare_continuation_cell_inherits_row_column_and_id_high_byte` | +| `PTY.IMG-R04` | `graphics.rs` (`placement_params`, `clip_top`) | `graphics.rs::a_source_rect_and_offsets_come_back_resolved`, `a_cropped_placement_keeps_its_crop_through_a_replay` | +| `PTY.IMG-R05` | `protocol.rs` (`decode_cell`), `clients.rs`, `graphics.rs` (`CellSize`) | `protocol.rs::attach_can_declare_a_cell_size_without_changing_the_size_it_carries`, `resize_can_declare_a_cell_size`, `a_plain_size_payload_declares_no_cell`, `graphics.rs::an_implicit_placement_takes_its_extent_from_the_declared_cell`, `a_declared_cell_does_not_move_an_explicit_placement`, `handle.rs::a_client_declares_its_cell_size_and_the_session_geometry_follows` | +| `PTY.IMG-R06` | `graphics.rs` (`replay`), `serialize.rs` (`vt`) | `graphics.rs::a_replay_without_graphics_is_unchanged` | +| `PTY.IMG-R07` | `actor.rs` (`reset`), `lifecycle.rs` | `graphics.rs::the_alternate_screen_has_its_own_storage`, `a_replay_from_the_alt_screen_puts_the_normal_screens_images_on_the_normal_screen`, `handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` (reconnect half) | +| `PTY.IMG-R08` | `actor.rs` (`enable_graphics`), `graphics.rs` (`GraphicsOptions`) | None. Enforced by leaving the file, temporary-file, and shared-memory media disabled; backed by [decision 0012](../../decisions/0012-kitty-graphics-replay.md) only. A regression here would not fail the suite. | +| `PTY.IMG-R09` | `graphics.rs` (`PngDecoder`, `expand`, `MAX_DECODED_PNG_BYTES`) | `graphics.rs::a_grayscale_png_is_stored_as_rgba`, `raising_the_storage_limit_keeps_the_cell_size_and_decodes_png`, `handle.rs::a_late_attach_gets_the_image_the_child_drew_before_it_connected` | +| `PTY.IMG-R10` | `graphics.rs` (`generation`), `handle.rs` (`HandleEvent::Graphics`) | `graphics.rs::a_spawned_child_that_draws_an_image_is_queryable_through_the_handle`, `a_delete_from_the_child_drops_the_placement_and_the_bytes`, `scrolling_moves_the_placement_and_scrollback_still_finds_it`, `a_resize_keeps_the_image_and_reprojects_it` | + +Run the whole map with `cargo test -p pty-terminal -p pty-core`; the image +cases alone with `cargo test -p pty-terminal --test graphics`. No gated +`_node` / `_rust` conformance pair exists or can: the Node side has no image +state to compare against (`PTY.IMG-C05`).