Move away from unsafe code to decode a UTF8 byte stream, and use Swift's APIs

for it.   And also implement the usage of the variation selector to choose the
width for a handful of scenarios.

See #412 for discussion

Author: migueldeicaza

Sources/SwiftTerm/Terminal.swift

@@ -1174,23 +1174,29 @@ open class Terminal {
                 for _ in 1..<n {
                     x.append (readingBuffer.getNext())
                 }
-                x.append(0)
-                x.withUnsafeBytes { ptr in
-                    let unsafeBound = ptr.bindMemory(to: UInt8.self)
-                    let unsafePointer = unsafeBound.baseAddress!
-                    
-                    let s = String (cString: unsafePointer)
-                    ch = s.first ?? Character (" ")
-
-                    // Now the challenge is that we have a character, not a rune, and we want to compute
-                    // the width of it.
-                    if ch.unicodeScalars.count == 1 {
-                        chWidth = UnicodeUtil.columnWidth(rune: ch.unicodeScalars.first!)
-                    } else {
-                        chWidth = 0
-                        for scalar in ch.unicodeScalars {
-                            chWidth = max (chWidth, UnicodeUtil.columnWidth(rune: scalar))
-                        }
+
+                var iterator = x.makeIterator()
+                var decoder = UTF8()
+                switch decoder.decode(&iterator) {
+                case .scalarValue(let scalar):
+                    ch = Character(scalar)
+                default:
+                    // Invalid UTF-8 sequence, fall back to interpreting the first byte
+                    let rune = UnicodeScalar(code)
+                    chWidth = UnicodeUtil.columnWidth(rune: rune)
+                    let charData = CharData (attribute: curAttr, scalar: rune, size: Int8 (chWidth))
+                    buffer.insertCharacter(charData)
+                    continue
+                }
+
+                // Now the challenge is that we have a character, not a rune, and we want to compute
+                // the width of it.
+                if ch.unicodeScalars.count == 1 {
+                    chWidth = UnicodeUtil.columnWidth(rune: ch.unicodeScalars.first!)
+                } else {
+                    chWidth = 0
+                    for scalar in ch.unicodeScalars {
+                        chWidth = max (chWidth, UnicodeUtil.columnWidth(rune: scalar))
                     }
                 }
             } else {
@@ -1239,7 +1245,15 @@ open class Terminal {
                         // If the resulting string is 1 grapheme cluster, then it combined properly
                         if newStr.count == 1 {
                             if let newCh = newStr.first {
-                                cd.setValue(char: newCh, size: Int32 (cd.width))
+                                switch firstScalar.value {
+                                    // This is the "This should use color modifier" on the previous item
+                                    // and we are going to take this to mean two columns
+                                    // See https://github.com/migueldeicaza/SwiftTerm/pull/412
+                                case 0xFE0F:
+                                    cd.setValue(char: newCh, size: 2)
+                                default:
+                                    cd.setValue(char: newCh, size: Int32 (cd.width))
+                                }
                                 existingLine [lastx] = cd
                                 updateRange (last.y)
                                 continue

Tests/SwiftTermTests/UnicodeTests.swift

@@ -31,7 +31,28 @@ final class SwiftTermUnicode: XCTestCase {
         XCTAssertEqual(t.getCharacter (col:0, row: 4), "b⃑")
 
     }
-    
+
+    func testVariationSelector ()
+    {
+        let h = HeadlessTerminal (queue: SwiftTermTests.queue) { exitCode in }
+        let t = h.terminal!
+
+        // This will send ⛩️ (0x26e9) is actually in a special class: it can either be one-column (⛩) or two-columns (⛩️)
+        // depending on the unicode "variation selector" that follows: 0x26e9 0xfe0e = ⛩, 0x26e9 0xfe0f = ⛩️.
+        // Globally, any unicode character followed by 0xfe0e will be single column, any unicode character
+        // followed by 0xfe0f will be double-column:
+        // https://en.wikipedia.org/wiki/Variation_Selectors_(Unicode_block)
+        t.feed (text: "\u{026e9}\u{0fe0f}\n\r\u{026e9}\u{0fe0e}")
+
+        // The first line should have 2 columns
+        let char0_0 = t.getCharData(col: 0, row: 0)
+        XCTAssertEqual(char0_0?.width, 2)
+
+        // The first line should have 1 columns
+        let char1_0 = t.getCharData(col: 0, row: 1)
+        XCTAssertEqual(char1_0?.width, 1)
+    }
+
     func testEmoji ()
     {
         let h = HeadlessTerminal (queue: SwiftTermTests.queue) { exitCode in }