Merge pull request #13 from compoundingtech/schickling-assistant/2026-09-04-kitty-graphics-state

feat(terminal): kitty graphics state, replay, child input encoding, and an honest scrollback

Author: schickling

Cargo.lock

@@ -19,6 +19,7 @@ serde = { version = "1.0.229", features = ["derive"] }
 serde_json = { version = "1.0.151", features = ["preserve_order"] }
 indexmap = { version = "2.14.0", features = ["serde"] }
 notify = "8.2.0"
+png = "0.18.1"
 toml = { version = "1.1.3", features = ["preserve_order"] }
 
 # Workspace crates.

Cargo.toml

@@ -73,6 +73,12 @@ Where the port stands against the Node `pty`, surface by surface, is in
 [docs/parity.md](docs/parity.md); the work packages that close the gap are in
 [docs/parity-plan.md](docs/parity-plan.md).
 
+What must stay true of terminal images — the Kitty graphics protocol state a
+session holds and replays — is in
+[docs/vrs/01-images/requirements.md](docs/vrs/01-images/requirements.md); how
+the session meets it is in
+[docs/vrs/01-images/spec.md](docs/vrs/01-images/spec.md).
+
 ## Install
 
 With Nix (flakes), from a checkout or straight from GitHub:

README.md

@@ -1,7 +1,8 @@
 //! `lastOutputAtMs`: the daemon records when the child last printed.
 //!
-//! The Node tool grew this field in its PR #168, merged on 2026-08-29
-//! (`docs/vrs/requirements.md` R14, `docs/disk-layout.md`). The contract is:
+//! The Node tool grew this field in its PR #168, merged on 2026-08-29 (the
+//! Node pty repository's `docs/vrs/requirements.md` R14 and
+//! `docs/disk-layout.md`, not this repository's `docs/`). The contract is:
 //!
 //! - absent until the session produces output, and absent on a record an
 //!   older daemon wrote — never zero, and never a claim of idleness;

crates/pty-conformance/tests/output_activity.rs

@@ -114,11 +114,35 @@ fn size_payload(rows: u16, cols: u16) -> [u8; 4] {
     [r[0], r[1], c[0], c[1]]
 }
 
+/// `rows u16BE, cols u16BE, cell_width u16BE, cell_height u16BE`: the size
+/// payload with the client's cell pixel metrics appended.
+///
+/// The four extra bytes are an *optional suffix*, which is what makes this
+/// safe to send to any peer: every reader of a size payload takes rows and
+/// cols from the first four bytes and the frame carries its own length, so a
+/// daemon that predates this (the Node one included) reads the size it
+/// always read and ignores the rest.
+fn size_cell_payload(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> [u8; 8] {
+    let s = size_payload(rows, cols);
+    let w = cell_width.to_be_bytes();
+    let h = cell_height.to_be_bytes();
+    [s[0], s[1], s[2], s[3], w[0], w[1], h[0], h[1]]
+}
+
 /// Encode an ATTACH with a terminal size (4-byte payload).
 pub fn encode_attach(rows: u16, cols: u16) -> Vec<u8> {
     encode_packet(MessageType::Attach, &size_payload(rows, cols))
 }
 
+/// Encode an ATTACH that also declares the client's cell pixel size
+/// (8-byte payload; see [`decode_cell`]).
+pub fn encode_attach_with_cell(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> Vec<u8> {
+    encode_packet(
+        MessageType::Attach,
+        &size_cell_payload(rows, cols, cell_width, cell_height),
+    )
+}
+
 /// Encode a DETACH.
 pub fn encode_detach() -> Vec<u8> {
     encode_packet(MessageType::Detach, &[])
@@ -129,6 +153,14 @@ pub fn encode_resize(rows: u16, cols: u16) -> Vec<u8> {
     encode_packet(MessageType::Resize, &size_payload(rows, cols))
 }
 
+/// Encode a RESIZE that also declares the client's cell pixel size.
+pub fn encode_resize_with_cell(rows: u16, cols: u16, cell_width: u16, cell_height: u16) -> Vec<u8> {
+    encode_packet(
+        MessageType::Resize,
+        &size_cell_payload(rows, cols, cell_width, cell_height),
+    )
+}
+
 /// Encode a GEOMETRY (effective shared rows/cols, server → client).
 pub fn encode_geometry(rows: u16, cols: u16) -> Vec<u8> {
     encode_packet(MessageType::Geometry, &size_payload(rows, cols))
@@ -177,6 +209,21 @@ pub fn decode_size(payload: &[u8]) -> (u16, u16) {
     (rows, cols)
 }
 
+/// The cell pixel size a client appended to an ATTACH or RESIZE payload, or
+/// `None` when it sent the plain 4-byte size or a degenerate zero.
+///
+/// Cell metrics are the client's to know — they come from its font, on its
+/// host — so a session daemon can only be told. `None` means nobody has, and
+/// the reader keeps its own deterministic fallback.
+pub fn decode_cell(payload: &[u8]) -> Option<(u16, u16)> {
+    if payload.len() < 8 {
+        return None;
+    }
+    let width = u16::from_be_bytes([payload[4], payload[5]]);
+    let height = u16::from_be_bytes([payload[6], payload[7]]);
+    (width > 0 && height > 0).then_some((width, height))
+}
+
 /// Decode a GEOMETRY payload (rows, cols); same layout and fallback as
 /// [`decode_size`].
 pub fn decode_geometry(payload: &[u8]) -> (u16, u16) {

crates/pty-core/src/protocol.rs

@@ -89,7 +89,8 @@ pub struct SessionMetadata {
     /// nothing to take; it is persisted at most once a second while output
     /// flows.
     ///
-    /// node: src/sessions.ts (`lastOutputAtMs`), docs/vrs/requirements.md R14
+    /// node: src/sessions.ts (`lastOutputAtMs`), the Node pty repository's
+    /// `docs/vrs/requirements.md` R14 (not this repository's `docs/vrs`)
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub last_output_at_ms: Option<i64>,
     /// Every field this version does not model, round-tripped verbatim.

crates/pty-core/src/registry/metadata.rs

@@ -1,9 +1,10 @@
 //! Port of the pty project's `tests/protocol.test.ts`.
 
 use pty_core::protocol::{
-    MAX_PACKET_LENGTH, MessageType, PacketReader, decode_exit, decode_geometry, decode_size,
-    encode_attach, encode_data, encode_detach, encode_exit, encode_geometry, encode_packet,
-    encode_resize, encode_screen, encode_status, encode_status_response,
+    MAX_PACKET_LENGTH, MessageType, PacketReader, decode_cell, decode_exit, decode_geometry,
+    decode_size, encode_attach, encode_attach_with_cell, encode_data, encode_detach, encode_exit,
+    encode_geometry, encode_packet, encode_resize, encode_resize_with_cell, encode_screen,
+    encode_status, encode_status_response,
 };
 use pty_core::stats::{ClientStats, ConnectionStats, Constrains, StatsResult};
 
@@ -36,6 +37,50 @@ fn attach_byte_identical_to_hand_built_packet() {
     );
 }
 
+/// The cell pixel size is an optional suffix on a size payload: a reader that
+/// does not know about it takes the same rows and cols it always did, which
+/// is what makes this safe to send to any daemon.
+#[test]
+fn attach_can_declare_a_cell_size_without_changing_the_size_it_carries() {
+    let with_cell = encode_attach_with_cell(24, 80, 9, 18);
+    assert_eq!(
+        with_cell,
+        encode_packet(MessageType::Attach, &[0, 24, 0, 80, 0, 9, 0, 18])
+    );
+
+    let mut reader = PacketReader::new();
+    let packets = reader.feed(&with_cell).unwrap();
+    assert_eq!(packets[0].type_, MessageType::Attach);
+    assert_eq!(
+        decode_size(&packets[0].payload),
+        (24, 80),
+        "the size is where it always was"
+    );
+    assert_eq!(decode_cell(&packets[0].payload), Some((9, 18)));
+}
+
+#[test]
+fn resize_can_declare_a_cell_size() {
+    let mut reader = PacketReader::new();
+    let packets = reader.feed(&encode_resize_with_cell(30, 100, 7, 15)).unwrap();
+    assert_eq!(packets[0].type_, MessageType::Resize);
+    assert_eq!(decode_size(&packets[0].payload), (30, 100));
+    assert_eq!(decode_cell(&packets[0].payload), Some((7, 15)));
+}
+
+/// No declaration is the normal case, and it must not be mistaken for one.
+#[test]
+fn a_plain_size_payload_declares_no_cell() {
+    assert_eq!(decode_cell(&encode_attach(24, 80)[5..]), None);
+    assert_eq!(decode_cell(&encode_resize(24, 80)[5..]), None);
+    assert_eq!(decode_cell(&[]), None);
+    assert_eq!(
+        decode_cell(&[0, 24, 0, 80, 0, 0, 0, 0]),
+        None,
+        "a zero cell is no cell"
+    );
+}
+
 /// node: tests/protocol.test.ts:288-297
 #[test]
 fn round_trips_geometry() {

crates/pty-core/tests/protocol.rs

@@ -12,6 +12,7 @@ libghostty-vt.workspace = true
 portable-pty.workspace = true
 pty-core.workspace = true
 libc.workspace = true
+png.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true