unicode: overhaul the way unicode codepoints are combined

Not all unicode codepoints that should be combined into previous
codepoints to create new graphemes are considered zero width.
Specifically, regional indicators by themselves are valid graphemes, but
when combined, they create various regional graphemes.

Additionally, there are a few emoji modifiers (for skin tone) that are
colors by themselves, but modify the previous grapheme if it is
combinable and _become_ zero width if so.

ZWJ (the zero-width joiner) is also not itself considered a combining
character, but could possibly cause two non-zero-width codepoints to all
be combined, and may increase the grapheme's width from one to two. VS16
can also modify a grapheme to force it to be rendered as an emoji, which
can increase its width from one to two.

All of this considering, the way mosh combines unicode codepoints into
cells needed some work. Now, it'll always check if it's possible to
combine with the previous cell and do it if it makes sense. This largely
follows tmux's general way of looking for combining opportunities.

Still a TODO is handling Hangul Jamo Korean combining which has its own
rules.

Author: jdrouhard

src/terminal/terminal.cc

@@ -33,7 +33,6 @@
 #include <cassert>
 #include <cstdlib>
 #include <cstring>
-#include <typeinfo>
 
 #include <unistd.h>
 
@@ -56,6 +55,69 @@ void Emulator::execute( const Parser::Execute* act )
   dispatch.dispatch( CONTROL, act, &fb );
 }
 
+bool Emulator::try_combine( wchar_t ch, int chwidth )
+{
+  bool zero_width = chwidth == 0;
+  bool force_wide = ch == 0xFE0F; // VS16
+
+  Cell* combining_cell = fb.get_combining_cell();
+  if ( !combining_cell ) {
+    return zero_width;
+  }
+
+  if ( !zero_width ) {
+    // TODO: check Hangul Jamo state here
+    if ( is_unicode_modifier( ch ) ) {
+      if ( combining_cell->size() < 2 ) {
+        return false;
+      }
+      force_wide = true;
+    } else if ( combining_cell->size() < 3
+                || combining_cell->get_contents().rfind( "\342\200\215" ) == std::string::npos ) {
+      return false;
+    }
+  }
+
+  if ( combining_cell->empty() ) {
+    /* cell starts with combining character */
+    /* ... but isn't necessarily the target for a new
+       base character [e.g. start of line], if the
+       combining character has been cleared with
+       a sequence like ED ("J") or EL ("K") */
+    assert( !combining_cell->get_wide() );
+    combining_cell->set_fallback( true );
+    fb.ds.move_col( 1, true, true );
+  }
+
+  if ( !combining_cell->full() ) {
+    combining_cell->append( ch );
+
+    if ( force_wide && !combining_cell->get_wide() ) {
+      // have to move this emoji to the next line
+      if ( fb.ds.auto_wrap_mode && fb.ds.next_print_will_wrap ) {
+        fb.get_mutable_row( -1 )->set_wrap( false );
+        fb.ds.move_col( 0 );
+        fb.move_rows_autoscroll( 1 );
+        *fb.get_mutable_cell() = *combining_cell;
+        fb.reset_cell( combining_cell );
+        fb.ds.move_col( 1, true, true );
+        combining_cell = fb.get_combining_cell();
+      }
+      combining_cell->set_wide( true );
+      if ( fb.ds.insert_mode ) {
+        fb.insert_cell( fb.ds.get_cursor_row(), fb.ds.get_cursor_col() );
+      } else if ( fb.ds.get_cursor_col() < fb.ds.get_width() ) {
+        fb.reset_cell( fb.get_mutable_cell() );
+      }
+      fb.get_mutable_cell()->set_wide_padding( true );
+      fb.ds.move_col( 1, true, true );
+    }
+    return true;
+  }
+
+  return false;
+}
+
 void Emulator::print( const Parser::Print* act )
 {
   assert( act->char_present );
@@ -68,6 +130,11 @@ void Emulator::print( const Parser::Print* act )
    */
   const int chwidth = ch == L'\0' ? -1 : ( Cell::isprint_iso8859_1( ch ) ? 1 : mosh_wcwidth( ch ) );
 
+  // attempt to combine with previous cell if necessary
+  if ( !Cell::isprint_iso8859_1( ch ) && try_combine( ch, chwidth ) ) {
+    return;
+  }
+
   Cell* this_cell = fb.get_mutable_cell();
 
   switch ( chwidth ) {
@@ -109,54 +176,17 @@ void Emulator::print( const Parser::Print* act )
       fb.apply_renditions_to_cell( this_cell );
 
       if ( chwidth == 2 && fb.ds.get_cursor_col() + 1 < fb.ds.get_width() ) { /* erase overlapped cell */
-        fb.reset_cell( fb.get_mutable_cell( fb.ds.get_cursor_row(), fb.ds.get_cursor_col() + 1 ) );
+        Cell* overlapped = fb.get_mutable_cell( fb.ds.get_cursor_row(), fb.ds.get_cursor_col() + 1 );
+        fb.reset_cell( overlapped );
+        overlapped->set_wide_padding( true );
       }
 
       fb.ds.move_col( chwidth, true, true );
 
       break;
     case 0: /* combining character */
-    {
-      Cell* combining_cell = fb.get_combining_cell(); /* can be null if we were resized */
-      if ( combining_cell == NULL ) {                 /* character is now offscreen */
-        break;
-      }
-
-      if ( combining_cell->empty() ) {
-        /* cell starts with combining character */
-        /* ... but isn't necessarily the target for a new
-           base character [e.g. start of line], if the
-           combining character has been cleared with
-           a sequence like ED ("J") or EL ("K") */
-        assert( !combining_cell->get_wide() );
-        combining_cell->set_fallback( true );
-        fb.ds.move_col( 1, true, true );
-      }
-      if ( !combining_cell->full() ) {
-        combining_cell->append( ch );
-        // VS16 causes the previous codepoint to be rendered as its emoji representation
-        // which could cause it to change from 1 to 2 characters wide
-        if ( ch == 0xFE0F && !combining_cell->get_wide() ) {
-          // have to move this emoji to the next line
-          if ( fb.ds.auto_wrap_mode && fb.ds.next_print_will_wrap ) {
-            fb.get_mutable_row( -1 )->set_wrap( false );
-            fb.ds.move_col( 0 );
-            fb.move_rows_autoscroll( 1 );
-            *fb.get_mutable_cell() = *combining_cell;
-            fb.reset_cell( combining_cell );
-            fb.ds.move_col( 1, true, true );
-            combining_cell = fb.get_combining_cell();
-          }
-          combining_cell->set_wide( true );
-          if ( fb.ds.insert_mode ) {
-            fb.insert_cell( fb.ds.get_cursor_row(), fb.ds.get_cursor_col() );
-          } else if ( fb.ds.get_cursor_col() < fb.ds.get_width() ) {
-            fb.reset_cell( fb.get_mutable_cell() );
-          }
-          fb.ds.move_col( 1, true, true );
-        }
-      }
-    } break;
+      // handled above
+      break;
     case -1: /* unprintable character */
       break;
     default:

src/terminal/terminal.h

@@ -74,6 +74,8 @@ class Emulator
   void OSC_end( const Parser::OSC_End* act );
   void resize( size_t s_width, size_t s_height );
 
+  bool try_combine(wchar_t ch, int chwidth);
+
 public:
   Emulator( size_t s_width, size_t s_height );
 

src/terminal/terminalframebuffer.cc

@@ -39,7 +39,8 @@
 using namespace Terminal;
 
 Cell::Cell( color_type background_color )
-  : contents(), renditions( background_color ), wide( false ), fallback( false ), wrap( false )
+  : contents(), renditions( background_color ), wide( false ), fallback( false ), wrap( false ),
+    wide_padding( false )
 {}
 
 void Cell::reset( color_type background_color )
@@ -49,6 +50,7 @@ void Cell::reset( color_type background_color )
   wide = false;
   fallback = false;
   wrap = false;
+  wide_padding = false;
 }
 
 void DrawState::reinitialize_tabs( unsigned int start )
@@ -60,9 +62,8 @@ void DrawState::reinitialize_tabs( unsigned int start )
 }
 
 DrawState::DrawState( int s_width, int s_height )
-  : width( s_width ), height( s_height ), cursor_col( 0 ), cursor_row( 0 ), combining_char_col( 0 ),
-    combining_char_row( 0 ), default_tabs( true ), tabs( s_width ), scrolling_region_top_row( 0 ),
-    scrolling_region_bottom_row( height - 1 ), renditions( 0 ), save(),
+  : width( s_width ), height( s_height ), cursor_col( 0 ), cursor_row( 0 ), default_tabs( true ), tabs( s_width ),
+    scrolling_region_top_row( 0 ), scrolling_region_bottom_row( height - 1 ), renditions( 0 ), save(),
     cursor_style( Terminal::CursorStyle::BLINKING_BLOCK ), next_print_will_wrap( false ), origin_mode( false ),
     auto_wrap_mode( true ), insert_mode( false ), cursor_visible( true ), reverse_video( false ),
     bracketed_paste( false ), mouse_reporting_mode( MOUSE_REPORTING_NONE ), mouse_focus_event( false ),
@@ -115,8 +116,6 @@ void Framebuffer::scroll( int N )
 
 void DrawState::new_grapheme( void )
 {
-  combining_char_col = cursor_col;
-  combining_char_row = cursor_row;
 }
 
 void DrawState::snap_cursor_to_border( void )
@@ -191,13 +190,23 @@ void Framebuffer::move_rows_autoscroll( int rows )
 
 Cell* Framebuffer::get_combining_cell( void )
 {
-  if ( ( ds.get_combining_char_col() < 0 ) || ( ds.get_combining_char_row() < 0 )
-       || ( ds.get_combining_char_col() >= ds.get_width() )
-       || ( ds.get_combining_char_row() >= ds.get_height() ) ) {
+  int cursor_x = ds.get_cursor_col();
+  if ( cursor_x < 1 ) {
     return NULL;
-  } /* can happen if a resize came in between */
+  }
+
+  int n = 1;
+  Cell* last = get_mutable_cell( -1, cursor_x - n );
+  if ( cursor_x != 1 && ( last->get_wide_padding() ) ) {
+    n = 2;
+    last = get_mutable_cell( -1, cursor_x - n );
+  }
+
+  if ( last && ( last->get_width() != n || last->get_wide_padding() ) ) {
+    return NULL;
+  }
 
-  return get_mutable_cell( ds.get_combining_char_row(), ds.get_combining_char_col() );
+  return last;
 }
 
 void DrawState::set_tab( void )
@@ -441,11 +450,6 @@ void DrawState::resize( int s_width, int s_height )
   snap_cursor_to_border();
 
   /* saved cursor will be snapped to border on restore */
-
-  /* invalidate combining char cell if necessary */
-  if ( ( combining_char_col >= width ) || ( combining_char_row >= height ) ) {
-    combining_char_col = combining_char_row = -1;
-  }
 }
 
 Renditions::Renditions( color_type s_background )
@@ -750,5 +754,10 @@ bool Cell::compare( const Cell& other ) const
     fprintf( stderr, "wrap: %d vs. %d\n", wrap, other.wrap );
   }
 
+  if ( wide_padding != other.wide_padding ) {
+    ret = true;
+    fprintf( stderr, "wide_padding: %d vs. %d\n", wide_padding, other.wide_padding );
+  }
+
   return ret;
 }

src/terminal/terminalframebuffer.h

@@ -124,6 +124,7 @@ class Cell
   unsigned int wide : 1;     /* 0 = narrow, 1 = wide */
   unsigned int fallback : 1; /* first character is combining character */
   unsigned int wrap : 1;
+  unsigned int wide_padding : 1;
 
 private:
   Cell();
@@ -136,18 +137,20 @@ class Cell
   bool operator==( const Cell& x ) const
   {
     return ( ( contents == x.contents ) && ( fallback == x.fallback ) && ( wide == x.wide )
-             && ( renditions == x.renditions ) && ( wrap == x.wrap ) );
+             && ( renditions == x.renditions ) && ( wrap == x.wrap ) && ( wide_padding == x.wide_padding ) );
   }
 
   bool operator!=( const Cell& x ) const { return !operator==( x ); }
 
   /* Accessors for contents field */
   std::string debug_contents( void ) const;
 
+  std::string const& get_contents() const { return contents; }
   bool empty( void ) const { return contents.empty(); }
   /* 32 seems like a reasonable limit on combining characters */
   bool full( void ) const { return contents.size() >= 32; }
   void clear( void ) { contents.clear(); }
+  std::size_t size() const { return contents.size(); }
 
   bool is_blank( void ) const
   {
@@ -220,6 +223,8 @@ class Cell
   void set_renditions( const Renditions& r ) { renditions = r; }
   bool get_wide( void ) const { return wide; }
   void set_wide( bool w ) { wide = w; }
+  bool get_wide_padding( void ) const { return wide_padding; }
+  void set_wide_padding( bool w ) { wide_padding = w; }
   unsigned int get_width( void ) const { return wide + 1; }
   bool get_fallback( void ) const { return fallback; }
   void set_fallback( bool f ) { fallback = f; }
@@ -278,7 +283,6 @@ class DrawState
   void snap_cursor_to_border( void );
 
   int cursor_col, cursor_row;
-  int combining_char_col, combining_char_row;
 
   bool default_tabs;
   std::vector<bool> tabs;
@@ -332,8 +336,6 @@ class DrawState
 
   int get_cursor_col( void ) const { return cursor_col; }
   int get_cursor_row( void ) const { return cursor_row; }
-  int get_combining_char_col( void ) const { return combining_char_col; }
-  int get_combining_char_row( void ) const { return combining_char_row; }
   int get_width( void ) const { return width; }
   int get_height( void ) const { return height; }