Adapt to latest libunicode, making use of new grapheme_line_segmenter API

cmake/presets/common.json

@@ -27,7 +27,16 @@
             "hidden": true,
             "cacheVariables": {
                 "CMAKE_BUILD_TYPE": "Debug",
-                "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}"
+                "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
+                "LIBUNICODE_TABLEGEN_FASTBUILD": "ON"
+            }
+        },
+        {
+            "name": "gcc-like-debug",
+            "hidden": true,
+            "inherits": ["debug"],
+            "cacheVariables": {
+                "CMAKE_CXX_FLAGS_DEBUG": "-O0 -ggdb3 -fno-inline-small-functions -ginline-points -fno-omit-frame-pointer"
             }
         }
     ]

cmake/presets/os-linux.json

@@ -45,7 +45,7 @@
         {
             "name": "linux-debug",
             "displayName": "Linux 64-bit",
-            "inherits": ["linux-common", "debug"]
+            "inherits": ["linux-common", "gcc-like-debug"]
         },
         {
             "name": "linux-clang-release",
@@ -55,7 +55,7 @@
         {
             "name": "linux-clang-debug",
             "displayName": "Linux 64-bit (Clang)",
-            "inherits": ["linux-clang", "debug"]
+            "inherits": ["linux-clang", "gcc-like-debug"]
         },
         {
             "name": "linux-gcc-release",
@@ -65,7 +65,7 @@
         {
             "name": "linux-gcc-debug",
             "displayName": "Linux 64-bit (GCC)",
-            "inherits": ["linux-gcc", "debug"]
+            "inherits": ["linux-gcc", "gcc-like-debug"]
         }
     ],
     "buildPresets": [

cmake/presets/os-macos.json

@@ -20,7 +20,7 @@
         {
             "name": "macos-debug",
             "displayName": "MacOS - Debug",
-            "inherits": ["macos-common", "debug"]
+            "inherits": ["macos-common", "gcc-like-debug"]
         },
         {
             "name": "macos-release",

cmake/presets/os-windows.json

@@ -5,9 +5,9 @@
         {
             "name": "windows-common",
             "inherits": "contour-common",
+            "generator": "Visual Studio 17 2022",
             "displayName": "Windows - common settings",
             "hidden": true,
-            "toolset": "host=x64",
             "binaryDir": "${sourceDir}/out/build/${presetName}",
             "condition": {
                 "type": "equals",

scripts/install-deps.ps1

@@ -14,9 +14,9 @@ class ThirdParty {
 $ThirdParties =
 @(
     [ThirdParty]@{
-        Folder  = "libunicode-23d7b30166a914b10526bb8fe7a469a9610c07dc";
-        Archive = "libunicode-23d7b30166a914b10526bb8fe7a469a9610c07dc.zip";
-        URI     = "https://github.com/contour-terminal/libunicode/archive/23d7b30166a914b10526bb8fe7a469a9610c07dc.zip";
+        Folder  = "libunicode-dabfea48f7fd2a8bf6ae19e37581de5c127c607f";
+        Archive = "libunicode-dabfea48f7fd2a8bf6ae19e37581de5c127c607f.zip";
+        URI     = "https://github.com/contour-terminal/libunicode/archive/dabfea48f7fd2a8bf6ae19e37581de5c127c607f.zip";
         Macro   = "libunicode"
     };
     [ThirdParty]@{

scripts/install-deps.sh

@@ -121,7 +121,7 @@ fetch_and_unpack_boxed()
 fetch_and_unpack_libunicode()
 {
     if test x$LIBUNICODE_SRC_DIR = x; then
-        local libunicode_git_sha="23d7b30166a914b10526bb8fe7a469a9610c07dc"
+        local libunicode_git_sha="dabfea48f7fd2a8bf6ae19e37581de5c127c607f"
         fetch_and_unpack \
             libunicode-$libunicode_git_sha \
             libunicode-$libunicode_git_sha.tar.gz \

src/contour/ContourApp.cpp

@@ -140,18 +140,6 @@ ContourApp::ContourApp(): app("contour", "Contour Terminal Emulator", CONTOUR_VE
     signal(SIGABRT, segvHandler);
 #endif
 
-#if defined(_WIN32)
-    // Enable VT output processing on Conhost.
-    HANDLE stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);
-    DWORD savedModes {}; // NOTE: Is it required to restore that upon process exit?
-    if (GetConsoleMode(stdoutHandle, &savedModes) != FALSE)
-    {
-        DWORD modes = savedModes;
-        modes |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-        SetConsoleMode(stdoutHandle, modes);
-    }
-#endif
-
     link("contour.capture", bind(&ContourApp::captureAction, this));
     link("contour.list-debug-tags", bind(&ContourApp::listDebugTagsAction, this));
     link("contour.set.profile", bind(&ContourApp::profileAction, this));

src/crispy/App.cpp

@@ -21,6 +21,10 @@
     #include <unistd.h>
 #endif
 
+#if defined(_WIN32)
+    #include <Windows.h>
+#endif
+
 using std::bind;
 using std::cout;
 using std::exception;
@@ -110,11 +114,7 @@ app::app(std::string appName, std::string appTitle, std::string appVersion, std:
     _appLicense { std::move(appLicense) },
     _localStateDir { xdgStateHome() / _appName }
 {
-    if (char const* logFilterString = getenv("LOG"))
-    {
-        logstore::configure(logFilterString);
-        customizeLogStoreOutput();
-    }
+    basicSetup();
 
     _instance = this;
 
@@ -128,6 +128,39 @@ app::~app()
     _instance = nullptr;
 }
 
+void app::basicSetup() noexcept
+{
+    enableVTProcessing();
+    enableUtf8Output();
+    if (char const* logFilterString = getenv("LOG"))
+    {
+        logstore::configure(logFilterString);
+        customizeLogStoreOutput();
+    }
+}
+
+void app::enableVTProcessing() noexcept
+{
+#if defined(_WIN32)
+    // Enable VT output processing on Conhost.
+    HANDLE stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+    DWORD savedModes {}; // NOTE: Is it required to restore that upon process exit?
+    if (GetConsoleMode(stdoutHandle, &savedModes) != FALSE)
+    {
+        SetConsoleMode(stdoutHandle,
+                       savedModes | ENABLE_VIRTUAL_TERMINAL_PROCESSING | ENABLE_PROCESSED_OUTPUT
+                           | ENABLE_WRAP_AT_EOL_OUTPUT);
+    }
+#endif
+}
+
+void app::enableUtf8Output() noexcept
+{
+#if defined(_WIN32)
+    SetConsoleOutputCP(CP_UTF8);
+#endif
+}
+
 void app::link(std::string command, std::function<int()> handler)
 {
     _handlers[std::move(command)] = std::move(handler);

src/crispy/App.h

@@ -32,6 +32,9 @@ class app
     [[nodiscard]] std::string const& appVersion() const noexcept { return _appVersion; }
     [[nodiscard]] std::filesystem::path const& localStateDir() const noexcept { return _localStateDir; }
 
+    static void basicSetup() noexcept;
+    static void enableVTProcessing() noexcept;
+    static void enableUtf8Output() noexcept;
     static void customizeLogStoreOutput();
 
   protected:

src/text_shaper/CMakeLists.txt

@@ -11,6 +11,8 @@ if("${CMAKE_SYSTEM}" MATCHES "Windows")
     list(APPEND text_shaper_SRC directwrite_analysis_wrapper.h)
     list(APPEND text_shaper_SRC directwrite_locator.cpp directwrite_locator.h)
     list(APPEND text_shaper_SRC directwrite_shaper.cpp directwrite_shaper.h)
+else()
+    list(APPEND text_shaper_SRC fontconfig_locator.cpp fontconfig_locator.h)
 endif()
 if(APPLE)
     list(APPEND text_shaper_SRC coretext_locator.h coretext_locator.mm)

src/vtbackend/CellUtil.h

@@ -103,12 +103,12 @@ template <CellConcept Cell>
     if (!AllowWidthChange)
         return 0;
 
-    auto const newWidth = [codepoint]() {
+    auto const newWidth = [codepoint]() -> int {
         switch (codepoint)
         {
             case 0xFE0E: return 1;
             case 0xFE0F: return 2;
-            default: return unicode::width(codepoint);
+            default: return static_cast<int>(unicode::width(codepoint));
         }
     }();
 

src/vtbackend/Line.cpp

@@ -1,8 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 #include <vtbackend/GraphicsAttributes.h>
 #include <vtbackend/Line.h>
+#include <vtbackend/logging.h>
 #include <vtbackend/primitives.h>
 
+#include <libunicode/grapheme_line_segmenter.h>
 #include <libunicode/grapheme_segmenter.h>
 #include <libunicode/utf8.h>
 #include <libunicode/width.h>
@@ -154,76 +156,108 @@ std::string Line<Cell>::toUtf8Trimmed(bool stripLeadingSpaces, bool stripTrailin
 }
 
 template <CellConcept Cell>
-InflatedLineBuffer<Cell> inflate(TrivialLineBuffer const& input)
+struct TrivialLineInflater
 {
-    static constexpr char32_t ReplacementCharacter { 0xFFFD };
+    TrivialLineBuffer const& input;
+    InflatedLineBuffer<Cell> columns;
 
-    auto columns = InflatedLineBuffer<Cell> {};
-    columns.reserve(unbox<size_t>(input.displayWidth));
+    explicit TrivialLineInflater(TrivialLineBuffer const& input): input { input }
+    {
+        columns.reserve(unbox<size_t>(input.displayWidth));
+    }
 
-    auto lastChar = char32_t { 0 };
-    auto utf8DecoderState = unicode::utf8_decoder_state {};
-    auto gapPending = 0;
+    InflatedLineBuffer<Cell> inflate() &&
+    {
+        vtParserLog()("Inflating TrivialLineBuffer: '{}'", input.text.data() ? crispy::escape(input.text.data()) : "");
+        auto lineSegmenter = unicode::grapheme_line_segmenter { *this, input.text.view() };
+        [[maybe_unused]] auto result = lineSegmenter.process(std::numeric_limits<unsigned>::max());
+        assert(result.stop_condition == unicode::StopCondition::EndOfInput);
+        [[maybe_unused]] auto const flushed = lineSegmenter.flush(std::numeric_limits<unsigned>::max());
+        assert(flushed.stop_condition == unicode::StopCondition::EndOfInput);
+        vtParserLog()("Inflated {}/{} columns", columns.size(), input.displayWidth);
+
+        // Fill remaining columns
+        for (unsigned i = columns.size(); i < unbox<size_t>(input.displayWidth); ++i)
+        {
+            columns.emplace_back(input.fillAttributes);
+        }
+        assert(columns.size() == unbox<size_t>(input.displayWidth));
+
+        return std::move(columns);
+    }
 
-    for (char const ch: input.text.view())
+    void on_invalid(std::string_view /*invalid*/) noexcept
     {
-        unicode::ConvertResult const r = unicode::from_utf8(utf8DecoderState, static_cast<uint8_t>(ch));
-        if (holds_alternative<unicode::Incomplete>(r))
-            continue;
+        fmt::print("inflate invalid\n");
+        static constexpr char32_t ReplacementCharacter { 0xFFFD };
 
-        auto const nextChar =
-            holds_alternative<unicode::Success>(r) ? get<unicode::Success>(r).value : ReplacementCharacter;
+        columns.emplace_back();
+        columns.back().setHyperlink(input.hyperlink);
+        columns.back().write(input.textAttributes, ReplacementCharacter, 1);
+    }
 
-        if (unicode::grapheme_segmenter::breakable(lastChar, nextChar))
+    void on_ascii(std::string_view text) noexcept
+    {
+        fmt::print("inflate ASCII: '{}'\n", text);
+        for (auto const ch: text)
         {
-            while (gapPending > 0)
-            {
-                columns.emplace_back(input.textAttributes.with(CellFlag::WideCharContinuation),
-                                     input.hyperlink);
-                --gapPending;
-            }
-            auto const charWidth = unicode::width(nextChar);
-            columns.emplace_back(Cell {});
+            columns.emplace_back();
             columns.back().setHyperlink(input.hyperlink);
-            columns.back().write(input.textAttributes, nextChar, static_cast<uint8_t>(charWidth));
-            gapPending = charWidth - 1;
+            columns.back().write(input.textAttributes, ch, 1);
         }
-        else
+    }
+
+    void on_grapheme_cluster(std::string_view text, unsigned width) noexcept
+    {
+        fmt::print("inflate GC: '{}', width: {}\n", text, width);
+        columns.emplace_back(input.textAttributes, input.hyperlink);
+        Cell& cell = columns.back();
+        cell.setHyperlink(input.hyperlink);
+
+        auto utf8DecoderState = unicode::utf8_decoder_state {};
+        for (auto const ch: text)
         {
-            Cell& prevCell = columns.back();
-            auto const extendedWidth = prevCell.appendCharacter(nextChar);
-            if (extendedWidth > 0)
+            unicode::ConvertResult const r = unicode::from_utf8(utf8DecoderState, static_cast<uint8_t>(ch));
+            if (auto const* cp = std::get_if<unicode::Success>(&r))
             {
-                auto const cellsAvailable = *input.displayWidth - static_cast<int>(columns.size()) + 1;
-                auto const n = min(extendedWidth, cellsAvailable);
-                for (int i = 1; i < n; ++i)
-                {
-                    columns.emplace_back(Cell { input.textAttributes });
-                    columns.back().setHyperlink(input.hyperlink);
-                }
+                std::cout << fmt::format(" - codepoint: U+{:X}\n", (unsigned) cp->value);
+                if (cell.codepointCount() == 0)
+                    cell.setCharacter(cp->value);
+                else
+                    (void) cell.appendCharacter(cp->value);
             }
         }
-        lastChar = nextChar;
-    }
 
-    while (gapPending > 0)
-    {
-        columns.emplace_back(Cell { input.textAttributes, input.hyperlink });
-        --gapPending;
-    }
+        fmt::print(" -> result (UTF-8): \"{}\"\n", cell.toUtf8());
 
-    assert(columns.size() == unbox<size_t>(input.usedColumns));
-    assert(unbox(input.displayWidth) > 0);
-
-    while (columns.size() < unbox<size_t>(input.displayWidth))
-        columns.emplace_back(Cell { input.fillAttributes });
+        // Fill remaining columns for wide characters
+        for (unsigned i = 1; i < width; ++i)
+        {
+            std::cout << fmt::format(" - continuation\n");
+            columns.emplace_back(input.textAttributes.with(CellFlag::WideCharContinuation), input.hyperlink);
+            cell.setWidth(width);
+        }
+    }
+};
 
-    return columns;
+template <CellConcept Cell>
+InflatedLineBuffer<Cell> inflate(TrivialLineBuffer const& input)
+{
+    return TrivialLineInflater<Cell>(input).inflate();
 }
+
 } // end namespace vtbackend
 
+// {{{ Explicit instantiation of Line<Cell> for supported cell types.
 #include <vtbackend/cell/CompactCell.h>
-template class vtbackend::Line<vtbackend::CompactCell>;
-
 #include <vtbackend/cell/SimpleCell.h>
-template class vtbackend::Line<vtbackend::SimpleCell>;
+
+namespace vtbackend
+{
+
+template class Line<CompactCell>;
+template class Line<SimpleCell>;
+template InflatedLineBuffer<SimpleCell> inflate(TrivialLineBuffer const& input);
+
+} // namespace vtbackend
+// }}}

src/vtbackend/Screen.h

@@ -669,7 +669,14 @@ template <CellConcept Cell>
 inline bool Screen<Cell>::isContiguousToCurrentLine(std::string_view continuationChars) const noexcept
 {
     auto const& line = currentLine();
+#if !defined(_WIN32)
     return line.isTrivialBuffer() && line.trivialBuffer().text.view().end() == continuationChars.begin();
+#else
+    char const* const end = line.trivialBuffer().text.data() + line.trivialBuffer().text.size();
+    char const* const next = continuationChars.data();
+    return line.isTrivialBuffer() && end == next;
+    // TODO: && line.trivialBuffer().text.view().end() == continuationChars.begin();
+#endif
 }
 
 } // namespace vtbackend