fix(wcwidth): treat surrogates as non-printable

utf8_decode() accepts CESU-8 surrogate encodings, and the Unicode 17
tables default unlisted codepoints to width 1, so a surrogate reaching
draw_text was emitted to the terminal as invalid UTF-8 (the old
termbox2 table returned -1 for U+D800..U+DFFF and the emit path
sanitized it to U+FFFD). Guard the surrogate range in the generated
wcwidth() so iswprint() rejects it and the emit path substitutes
U+FFFD again.

Also derive the generated header's Unicode version from the same
constant as the data URL; it was left saying 16.0 after the 17.0
upgrade.

Author: natemoo-re

src/wcwidth.c

@@ -1,5 +1,5 @@
 /* wcwidth.c - Unicode character width lookup
- * Unicode 16.0 - generated by tasks/gen-wcwidth.ts
+ * Unicode 17.0 - generated by tasks/gen-wcwidth.ts
  * Data hash: d73f21ced2426cb4
  *
  * Only zero-width (combining marks + default ignorable) and double-width
@@ -19,7 +19,8 @@
  * Noncharacters (U+FDD0-U+FDEF and U+nFFFE/U+nFFFF) return -1: they are
  * permanently unassigned and never printable. They cannot be stored in the
  * packed table (the width bit only encodes 0 vs 2), so wcwidth() handles them
- * with a dedicated guard.
+ * with a dedicated guard. Surrogates (U+D800-U+DFFF) return -1 for the same
+ * reason: they are not scalar values and only reach us via malformed UTF-8.
  *
  * Combining (width 0) and wide (width 2) ranges are merged into a single
  * sorted table so wcwidth() needs only one binary search for any codepoint.
@@ -182,6 +183,10 @@ int wcwidth(uint32_t codepoint) {
     return 1;
   if (codepoint < 0x20 || (codepoint > 0x7e && codepoint < 0xa0))
     return codepoint == 0 ? 0 : -1;
+  /* Surrogates are not scalar values; they only appear via malformed
+   * UTF-8 and are never printable. */
+  if ((codepoint & 0xfffff800) == 0xd800)
+    return -1;
   /* Noncharacters are permanently unassigned and never printable:
    * U+FDD0..U+FDEF and the last two codepoints of every plane
    * (U+nFFFE/U+nFFFF). */

tasks/gen-wcwidth.ts

@@ -8,12 +8,14 @@
 // BMP coarse filter: a 64-byte bitmap (1 bit per 128-codepoint block) gates
 // the binary search. Clean blocks return width 1 without touching the table.
 //
-// Lookup order in wcwidth(): ASCII/Latin-1 fast paths, then a noncharacter
-// guard, then codepoint_in_special() which checks (1) a TUI box-drawing fast
-// lane, (2) the large contiguous blocks (CJK/Hangul/SIP/tags), (3) the BMP
-// coarse filter, (4) the packed small-range binary search.
+// Lookup order in wcwidth(): ASCII/Latin-1 fast paths, then surrogate and
+// noncharacter guards, then codepoint_in_special() which checks (1) a TUI
+// box-drawing fast lane, (2) the large contiguous blocks
+// (CJK/Hangul/SIP/tags), (3) the BMP coarse filter, (4) the packed
+// small-range binary search.
 
-let UNICODE_BASE = "https://www.unicode.org/Public/17.0.0/ucd";
+let UNICODE_VERSION = "17.0";
+let UNICODE_BASE = `https://www.unicode.org/Public/${UNICODE_VERSION}.0/ucd`;
 
 interface Interval {
   start: number;
@@ -389,7 +391,7 @@ let dataHash = (await sha256Hex(
 
 let output = `\
 /* wcwidth.c - Unicode character width lookup
- * Unicode 16.0 - generated by tasks/gen-wcwidth.ts
+ * Unicode ${UNICODE_VERSION} - generated by tasks/gen-wcwidth.ts
  * Data hash: ${dataHash}
  *
  * Only zero-width (combining marks + default ignorable) and double-width
@@ -409,7 +411,8 @@ let output = `\
  * Noncharacters (U+FDD0-U+FDEF and U+nFFFE/U+nFFFF) return -1: they are
  * permanently unassigned and never printable. They cannot be stored in the
  * packed table (the width bit only encodes 0 vs 2), so wcwidth() handles them
- * with a dedicated guard.
+ * with a dedicated guard. Surrogates (U+D800-U+DFFF) return -1 for the same
+ * reason: they are not scalar values and only reach us via malformed UTF-8.
  *
  * Combining (width 0) and wide (width 2) ranges are merged into a single
  * sorted table so wcwidth() needs only one binary search for any codepoint.
@@ -510,6 +513,10 @@ int wcwidth(uint32_t codepoint) {
     return 1;
   if (codepoint < 0x20 || (codepoint > 0x7e && codepoint < 0xa0))
     return codepoint == 0 ? 0 : -1;
+  /* Surrogates are not scalar values; they only appear via malformed
+   * UTF-8 and are never printable. */
+  if ((codepoint & 0xfffff800) == 0xd800)
+    return -1;
   /* Noncharacters are permanently unassigned and never printable:
    * U+FDD0..U+FDEF and the last two codepoints of every plane
    * (U+nFFFE/U+nFFFF). */

test/width.test.ts

@@ -1,10 +1,21 @@
 import { describe, expect, it } from "./suite.ts";
 import { createTerm } from "../term.ts";
-import { close, grow, open, text } from "../ops.ts";
+import { close, grow, open, pack, text } from "../ops.ts";
+import { createTermNative } from "../term-native.ts";
 import { print } from "./print.ts";
 
 const decode = (bytes: Uint8Array) => new TextDecoder().decode(bytes);
 
+function indexOfSeq(haystack: Uint8Array, needle: number[]): number {
+  outer: for (let i = 0; i + needle.length <= haystack.length; i++) {
+    for (let j = 0; j < needle.length; j++) {
+      if (haystack[i + j] !== needle[j]) continue outer;
+    }
+    return i;
+  }
+  return -1;
+}
+
 describe("width", () => {
   it("measures non-printable codepoints as one replacement cell", async () => {
     let term = await createTerm({ width: 20, height: 4 });
@@ -44,4 +55,43 @@ describe("width", () => {
 
     expect(out).toContain("a�b");
   });
+
+  it("replaces surrogate codepoints from malformed UTF-8 with U+FFFD", async () => {
+    let native = await createTermNative(10, 3);
+
+    // TextEncoder never produces surrogate bytes, so pack a placeholder
+    // (U+1234 = E1 88 B4) and patch it to CESU-8 U+D800 (ED A0 80) to
+    // reach the decoder the way a malformed C-side buffer would.
+    let ops = [
+      open("root", {
+        layout: { width: grow(), height: grow(), direction: "ttb" },
+      }),
+      text("aሴb"),
+      close(),
+    ];
+    let len = pack(
+      ops,
+      native.memory.buffer,
+      native.opsBuf,
+      native.memory.buffer.byteLength,
+    );
+    // pack() returns a length in 32-bit words
+    let buf = new Uint8Array(native.memory.buffer, native.opsBuf, len * 4);
+    let at = indexOfSeq(buf, [0xe1, 0x88, 0xb4]);
+    expect(at).toBeGreaterThan(-1);
+    buf[at] = 0xed;
+    buf[at + 1] = 0xa0;
+    buf[at + 2] = 0x80;
+
+    native.reduce(native.statePtr, native.opsBuf, len, 0, 1, 0);
+    let out = new Uint8Array(
+      native.memory.buffer,
+      native.output(native.statePtr),
+      native.length(native.statePtr),
+    );
+
+    // the invalid sequence must never reach the terminal
+    expect(indexOfSeq(out, [0xed, 0xa0, 0x80])).toBe(-1);
+    expect(print(decode(out), 10, 3)).toContain("a�b");
+  });
 });