Test suites (#429)

Port from XCTest to Swift Test, new tests, lots of fixes.

First, this is a port from XCTest to Swift Test, but once I started using this in parallel
mode various crashes would show up, which turned out to be that CharData was not
thread-safe as it had shared static state.   So I made this code thread safe, so now you 
can run various Headless terminals (for tests or whatever you want in multiple threads).

This did change the API for CharData and how it is consumed, rather than having
CharData own these static tables, the tables are owned by the Terminal, which
means that you no longer create CharData() yourself, you have to create them
and update them via the Terminal API.

Second, we now have various tests imported from Ghostty, just to
give me more peace of mind, they uncovered a couple of parsing 
limitations on extended color parsing, and it now uses colon separator
counts to distinguish colorspace and non-colorspace ones

Then ported some Xterm.js tests, in particular the reflow ones, after
someone had mentioned they struggled with Reflow and some Claude
agent, but never got any details - but this test showed that we had an
issue, with wide‑char reflow behavior, adjusted to match the xterm.js tests, 
fixed the trimmed length math, wide‑char placeholders, and cleanup during 
unwrap.

The above regressed the nice work on wcwidth that took place in main, so
also fixed that regression, since we are now using the new CharData API, we have
to use getCharacter(for:) for the VS base check so combined graphemes
so they do not get treated as invalid scalars in Terminal.

No regressions on ucs-detect, which makes me very happy.

Author: migueldeicaza

Sources/SwiftTerm/Apple/AppleTerminalView.swift

@@ -549,7 +549,7 @@ extension TerminalView {
                 currentAttributes[.selectionBackgroundColor] = selectedTextBackgroundColor
             }
 
-            let character = ch.code == 0 ? " " : ch.getCharacter()
+            let character = ch.code == 0 ? " " : terminal.getCharacter(for: ch)
             if let placeholder = KittyPlaceholderDecoder.decode(character: character,
                                                                 attribute: attr,
                                                                 row: row,

Sources/SwiftTerm/Buffer.swift

@@ -258,7 +258,7 @@ public final class Buffer {
     public func getNullCell (attribute: Attribute? = nil) -> CharData
     {
         let fgbg = attribute == nil ? Attribute.empty : attribute!.justColor ()
-        return CharData(attribute: fgbg, char: " ", size: 1)
+        return CharData(attribute: fgbg, scalar: UnicodeScalar(32)!, size: 1)
     }
 
     public func getBlankLine (attribute: Attribute, isWrapped: Bool = false) -> BufferLine
@@ -496,10 +496,10 @@ public final class Buffer {
         }
     }
 
-    func translateBufferLineToString (lineIndex: Int, trimRight: Bool, startCol: Int = 0, endCol: Int = -1, skipNullCellsFollowingWide: Bool = false) -> String
+    func translateBufferLineToString (lineIndex: Int, trimRight: Bool, startCol: Int = 0, endCol: Int = -1, skipNullCellsFollowingWide: Bool = false, characterProvider: ((CharData) -> Character)? = nil) -> String
     {
         let line = _lines [lineIndex]
-        return line.translateToString(trimRight: trimRight, startCol: startCol, endCol: endCol, skipNullCellsFollowingWide: skipNullCellsFollowingWide)
+        return line.translateToString(trimRight: trimRight, startCol: startCol, endCol: endCol, skipNullCellsFollowingWide: skipNullCellsFollowingWide, characterProvider: characterProvider)
     }
 
     func setupTabStops (index: Int = -1, tabStopWidth: Int)
@@ -673,7 +673,7 @@ public final class Buffer {
                         wrappedLines [destLineIndex].copyFrom (wrappedLines [destLineIndex - 1], srcCol: newCols - 1, dstCol: destCol, len: 1)
                         destCol += 1
                         // Null out the end of the last row
-                        wrappedLines [destLineIndex - 1].replaceCells (start: newCols - 1, end: 1, fillData: nullChar)
+                        wrappedLines [destLineIndex - 1].replaceCells (start: newCols - 1, end: newCols, fillData: nullChar)
                     }
                 }
             }
@@ -1088,13 +1088,13 @@ public final class Buffer {
             if wraparound {
                 _x = marginMode ? marginLeft : 0
 
-                if _y >= scrollBottom {
+                if _y >= _scrollBottom {
                     scroll (true)
                 } else {
                     // The line already exists (eg. the initial viewport), mark it as a
                     // wrapped line
                     _y += 1
-                    lines [y].isWrapped = true
+                    _lines [y].isWrapped = true
                 }
                 // row changed, get it again
             } else {
@@ -1110,6 +1110,7 @@ public final class Buffer {
         let bufferRow = _lines[_y+_yBase]
         var empty = CharData.Null
         empty.attribute = curAttr
+        let wideEmpty = CharData(attribute: curAttr, scalar: UnicodeScalar(0)!, size: 0)
         // insert mode: move characters to right
         if insertMode {
             // right shift cells according to the width
@@ -1137,7 +1138,7 @@ public final class Buffer {
         if chWidth > 0 {
             chWidth -= 1
             while chWidth != 0 && _x < _cols {
-                bufferRow [_x] = empty
+                bufferRow [_x] = wideEmpty
                 _x += 1
                 chWidth -= 1
             }

Sources/SwiftTerm/BufferLine.swift

@@ -227,7 +227,7 @@ public final class BufferLine: CustomDebugStringConvertible {
     {
         for i in (0..<dataSize).reversed() {
             if data [i].code != 0 {
-                return i + 1
+                return i + Int(data[i].width)
             }
         }
         return 0
@@ -250,7 +250,7 @@ public final class BufferLine: CustomDebugStringConvertible {
     /// - Parameter startCol: the starting column to copy the data from, defaults toe zero if not provided
     /// - Parameter endCol: the end column (not included) to consume.  If the value -1, this copies all the way to the end
     /// - Returns: a string containing the contents of the BufferLine from [startCol..<endCol]
-    public func translateToString (trimRight: Bool = false, startCol: Int = 0, endCol: Int = -1, skipNullCellsFollowingWide: Bool = false) -> String
+    public func translateToString (trimRight: Bool = false, startCol: Int = 0, endCol: Int = -1, skipNullCellsFollowingWide: Bool = false, characterProvider: ((CharData) -> Character)? = nil) -> String
     {
         var ec = endCol == -1 ? dataSize : endCol
         if trimRight {
@@ -260,7 +260,8 @@ public final class BufferLine: CustomDebugStringConvertible {
         if !skipNullCellsFollowingWide {
             var result = ""
             for i in startCol..<limit {
-                result.append (data [i].getCharacter ())
+                let character = characterProvider?(data [i]) ?? data [i].getCharacter ()
+                result.append (character)
             }
             return result
         }
@@ -272,7 +273,8 @@ public final class BufferLine: CustomDebugStringConvertible {
                 continue
             }
             let cell = data [idx]
-            result.append (cell.getCharacter ())
+            let character = characterProvider?(cell) ?? cell.getCharacter ()
+            result.append (character)
             if cell.width == 2 {
                 let nextIndex = idx + 1
                 if nextIndex < limit && data [nextIndex].code == 0 {

Sources/SwiftTerm/CharData.swift

@@ -219,7 +219,7 @@ public struct TinyAtom {
  * This uses an Int32 to store the value, if the value can not be encoded as a single Unicode.Scalar,
  * then an index is stored that is looked up in parallel, so that full grapheme clusters can be tracked.
  *
- * Use the `getCharacter` function to get the stored Character, and use the `attribute` property
+ * Use the `getCharacter` function to get simple runes, and use `Terminal.getCharacter(for:)` for extended graphemes. Use the `attribute` property
  * to retrieve the color and other character attributes.   The `width` property contains the number of
  * columns used by the `Character` stored in this `CharData` on the screen.
  *
@@ -238,15 +238,7 @@ public struct CharData: CustomDebugStringConvertible {
     }
 
     static let maxRune = 1 << 22
-    
-    // Contains the character to index mapping
-    static var charToIndexMap: [Character:Int32] = [:]
-    
-    // Contains the index to character mapping, could be a plain array
-    static var indexToCharMap: [Int32: Character] = [:]
-    static var lastCharIndex: Int32 = (1 << 22)+1
-    
-    
+
     static let defaultAttr = Attribute(fg: .defaultColor, bg: .defaultColor, style: .none)
     static let invertedAttr = Attribute(fg: .defaultInvertedColor, bg: .defaultInvertedColor, style: .none)
 
@@ -264,48 +256,28 @@ public struct CharData: CustomDebugStringConvertible {
     /// The color and character attributes for the cell
     public var attribute: Attribute
 
-    /// Initializes a new instance of the CharData structure with the provided attribute, character and the dimension
+    /// Initializes a new instance of the CharData structure with the provided attribute and code.
+    /// Use `Terminal.makeCharData` for Character-based construction.
     /// - Parameter attribute: an attribute containing the color and style attributes for the cell
-    /// - Parameter char: the character that will be stored in this cell
+    /// - Parameter code: the character code that will be stored in this cell
     /// - Parameter size: the number of columns used by the `Character` stored in this `CharData` on the screen.
-    init (attribute: Attribute, char: Character, size: Int8 = 1)
+    init (attribute: Attribute, code: Int32, size: Int8 = 1)
     {
         self.attribute = attribute
-        if let acode = char.asciiValue {
-            code = Int32(acode)
-        } else if char.utf16.count == 1 {
-            code = Int32 (char.utf16.first!)
-        } else {
-            if let existingIdx = CharData.charToIndexMap [char] {
-                code = existingIdx
-            } else {
-                CharData.charToIndexMap [char] = CharData.lastCharIndex
-                CharData.indexToCharMap [CharData.lastCharIndex] = char
-                code = CharData.lastCharIndex
-                CharData.lastCharIndex = CharData.lastCharIndex + 1
-            }
-        }
-        width = Int8 (size)
+        self.code = code
+        width = size
         payload = TinyAtom.empty
         unused = 0
     }
 
     init (attribute: Attribute, scalar: UnicodeScalar, size: Int8 = 1) {
-        self.attribute = attribute
-        code = Int32(scalar.value)
-        width = Int8 (size)
-        payload = TinyAtom.empty
-        unused = 0
+        self.init(attribute: attribute, code: Int32(scalar.value), size: size)
     }
 
     // Empty cell sets the code to zero
     init (attribute: Attribute)
     {
-        self.attribute = attribute
-        code = 0
-        width = 1
-        payload = TinyAtom.empty
-        unused = 0
+        self.init(attribute: attribute, code: 0, size: 1)
     }
 
     public var isSimpleRune: Bool {
@@ -334,34 +306,19 @@ public struct CharData: CustomDebugStringConvertible {
     /// The `Null` character can be used when filling up parts of the screeb
     public static var Null : CharData = CharData (attribute: defaultAttr)
 
-    /// Updates the contents of this CharData with a new character.
-    /// - Parameter char: the new character that will be stored
+    /// Updates the contents of this CharData with a new code.
+    /// - Parameter code: the new character code that will be stored
     /// - Paramerter size: the number of fixed sized columns this character will take on the screen
-    mutating public func setValue (char: Character, size: Int32)
+    public mutating func setValue (code: Int32, size: Int32)
     {
-        if char.utf16.count == 1 {
-            self.code = Int32 (char.utf16.first!)
-        } else {
-            if let existingIdx = CharData.charToIndexMap [char] {
-                code = existingIdx
-            } else {
-                CharData.charToIndexMap [char] = CharData.lastCharIndex
-                CharData.indexToCharMap [CharData.lastCharIndex] = char
-                code = CharData.lastCharIndex
-                CharData.lastCharIndex = CharData.lastCharIndex + 1
-            }
-        }
+        self.code = code
         width = Int8 (size)
     }
 
     /// Use this method to retrieve the Character stored in the CharData
+    /// For extended grapheme clusters use `Terminal.getCharacter(for:)`.
     public func getCharacter () -> Character
     {
-        if code > CharData.maxRune {
-            // This is an invariant - no code can be stored without the equivalent being tracked, but for the sake
-            // of not having a "!" return a space.
-            return CharData.indexToCharMap [code] ?? " "
-        }
         if let c = Unicode.Scalar (UInt32 (code)) {
             return Character(c)
         } else {

Sources/SwiftTerm/Mac/MacCaretView.swift

@@ -37,8 +37,9 @@ class CaretView: NSView, CALayerDelegate {
     }
 
     func setText (ch: CharData) {
+        let character = terminal?.terminal.getCharacter(for: ch) ?? " "
         let res = NSAttributedString (
-            string: String (ch.getCharacter()),
+            string: String (character),
             attributes: terminal?.getAttributedValue(ch.attribute, usingFg: caretColor, andBg: caretTextColor ?? terminal?.nativeForegroundColor ?? NSColor.black))
         ctline = CTLineCreateWithAttributedString(res)
 

Sources/SwiftTerm/Mac/MacDebugView.swift

@@ -81,7 +81,7 @@ public class TerminalDebugView: NSView {
                         attr = ch.attribute
                     }
                 }
-                str.append(ch.code == 0 ? " " : ch.getCharacter())
+                str.append(ch.code == 0 ? " " : terminal.getCharacter(for: ch))
             }
         } else {
             str = "<empty>"

Sources/SwiftTerm/SelectionService.swift

@@ -314,6 +314,12 @@ class SelectionService: CustomDebugStringConvertible {
         setActiveAndNotify()
     }
 
+    private func character (at position: Position, in buffer: Buffer) -> Character
+    {
+        let cell = buffer.getChar (atBufferRelative: position)
+        return terminal.getCharacter (for: cell)
+    }
+
     /**
      * Performs a simple "word" selection based on a function that determines inclussion into the group
      */
@@ -323,7 +329,7 @@ class SelectionService: CustomDebugStringConvertible {
         var colScan = position.col
         var left = colScan
         while colScan >= 0 {
-            let ch = buffer.getChar(atBufferRelative: Position (col: colScan, row: position.row)).getCharacter()
+            let ch = character (at: Position (col: colScan, row: position.row), in: buffer)
             if !includeFunc (ch) {
                 break
             }
@@ -336,7 +342,7 @@ class SelectionService: CustomDebugStringConvertible {
         var right = colScan
         let limit = terminal.cols
         while colScan < limit {
-            let ch = buffer.getChar(atBufferRelative: Position (col: colScan, row: position.row)).getCharacter()
+            let ch = character (at: Position (col: colScan, row: position.row), in: buffer)
             if !includeFunc (ch) {
                 break
             }
@@ -365,7 +371,7 @@ class SelectionService: CustomDebugStringConvertible {
         for line in position.row..<maxRow {
             for col in startCol..<terminal.cols {
                 let p =  Position(col: col, row: line)
-                let ch = buffer.getChar (atBufferRelative: p).getCharacter ()
+                let ch = character (at: p, in: buffer)
 
                 if ch == "(" {
                     wait.append (")")
@@ -403,7 +409,7 @@ class SelectionService: CustomDebugStringConvertible {
         for line in (0...position.row).reversed() {
             for col in (0...startCol).reversed() {
                 let p =  Position(col: col, row: line)
-                let ch = buffer.getChar (atBufferRelative: p).getCharacter ()
+                let ch = character (at: p, in: buffer)
 
                 if ch == ")" {
                     wait.append ("(")
@@ -434,7 +440,7 @@ class SelectionService: CustomDebugStringConvertible {
      * Extends a position to the nearest word boundary based on the character at that position
      */
     func extendToWordBoundary(position: Position, in buffer: Buffer, direction: Int) -> Position {
-        let ch = buffer.getChar(atBufferRelative: position).getCharacter()
+        let ch = character (at: position, in: buffer)
         var includeFunc: (Character) -> Bool
 
         switch ch {
@@ -453,7 +459,7 @@ class SelectionService: CustomDebugStringConvertible {
             // Extend backward
             var col = position.col
             while col >= 0 {
-                let testCh = buffer.getChar(atBufferRelative: Position(col: col, row: position.row)).getCharacter()
+                let testCh = character (at: Position(col: col, row: position.row), in: buffer)
                 if !includeFunc(testCh) {
                     break
                 }
@@ -464,7 +470,7 @@ class SelectionService: CustomDebugStringConvertible {
             // Extend forward
             var col = position.col
             while col < terminal.cols {
-                let testCh = buffer.getChar(atBufferRelative: Position(col: col, row: position.row)).getCharacter()
+                let testCh = character (at: Position(col: col, row: position.row), in: buffer)
                 if !includeFunc(testCh) {
                     break
                 }
@@ -486,7 +492,7 @@ class SelectionService: CustomDebugStringConvertible {
 //            row: max (min (uncheckedPosition.row, buffer.rows-1+buffer.yDisp), buffer.yDisp))
         let position = Position (col: (min (terminal.cols, max (uncheckedPosition.col, 0))),
                                  row: (max (uncheckedPosition.row, 0)))
-        switch buffer.getChar(atBufferRelative: position).getCharacter() {
+        switch character (at: position, in: buffer) {
         case Character(UnicodeScalar(0)):
             simpleScanSelection (from: position, in: buffer) { ch in ch == nullChar }
         case " ":