fix: clear all RGB bits when switching from RGB to P16/P256 colors

Use AttrRGBMask (0xFFFFFF) instead of AttrPColorMask (0xFF) when
clearing color bits before setting a palette color. The narrower mask
left stale bytes in the upper 16 bits of the 24-bit color field,
causing incorrect raw Fg/Bg values after RGB-to-palette transitions.

Port of upstream xterm.js commit 92339ac9.

Fixes #22

Co-authored-by: Ona <no-reply@ona.com>

Author: ona-automations

inputhandler_sgr.go

@@ -21,22 +21,22 @@ func (h *InputHandler) charAttributes(params *Params) bool {
 		switch {
 		case p >= 30 && p <= 37:
 			// fg color 8
-			attr.Fg &= ^(AttrCMMask | AttrPColorMask)
+			attr.Fg &= ^(AttrCMMask | AttrRGBMask)
 			attr.Fg |= AttrCMP16 | uint32(p-30)
 
 		case p >= 40 && p <= 47:
 			// bg color 8
-			attr.Bg &= ^(AttrCMMask | AttrPColorMask)
+			attr.Bg &= ^(AttrCMMask | AttrRGBMask)
 			attr.Bg |= AttrCMP16 | uint32(p-40)
 
 		case p >= 90 && p <= 97:
 			// fg color 16 (bright)
-			attr.Fg &= ^(AttrCMMask | AttrPColorMask)
+			attr.Fg &= ^(AttrCMMask | AttrRGBMask)
 			attr.Fg |= AttrCMP16 | uint32(p-90) | 8
 
 		case p >= 100 && p <= 107:
 			// bg color 16 (bright)
-			attr.Bg &= ^(AttrCMMask | AttrPColorMask)
+			attr.Bg &= ^(AttrCMMask | AttrRGBMask)
 			attr.Bg |= AttrCMP16 | uint32(p-100) | 8
 
 		case p == 0:
@@ -120,13 +120,13 @@ func (h *InputHandler) charAttributes(params *Params) bool {
 			// reset fg
 			attr.Fg &= ^(AttrCMMask | AttrRGBMask)
 			def := DefaultAttrData()
-			attr.Fg |= def.Fg & (AttrPColorMask | AttrRGBMask)
+			attr.Fg |= def.Fg & AttrRGBMask
 
 		case p == 49:
 			// reset bg
 			attr.Bg &= ^(AttrCMMask | AttrRGBMask)
 			def := DefaultAttrData()
-			attr.Bg |= def.Bg & (AttrPColorMask | AttrRGBMask)
+			attr.Bg |= def.Bg & AttrRGBMask
 
 		case p == 38 || p == 48 || p == 58:
 			// extended color (fg/bg/underline)
@@ -171,7 +171,7 @@ func (h *InputHandler) updateAttrColor(color uint32, mode int32, c1, c2, c3 int3
 		color &= ^AttrRGBMask
 		color |= FromColorRGB(ColorRGB{uint8(c1), uint8(c2), uint8(c3)})
 	case 5: // P256
-		color &= ^(AttrCMMask | AttrPColorMask)
+		color &= ^(AttrCMMask | AttrRGBMask)
 		color |= AttrCMP256 | (uint32(c1) & 0xFF)
 	}
 	return color

inputhandler_sgr_test.go

@@ -444,6 +444,94 @@ func TestSGR_SGR0_Preserves_URLId(t *testing.T) {
 	}
 }
 
+func TestSGR_RGBToP16_ClearsStaleRGBBits_Fg(t *testing.T) {
+	t.Parallel()
+	h := newTestHandler()
+	// Set RGB fg #FF8040
+	h.ParseString("\x1b[38;2;255;128;64m")
+	// Switch to P16 red (index 1)
+	h.ParseString("\x1b[31m")
+
+	// The full 24-bit color field must be clean — no stale RGB bytes
+	colorBits := h.curAttrData.Fg & AttrRGBMask
+	wantColor := uint32(1) // palette index 1 (red)
+	if colorBits != wantColor {
+		t.Errorf("stale RGB bits in Fg: got 0x%06x, want 0x%06x", colorBits, wantColor)
+	}
+	if cm := h.curAttrData.Fg & AttrCMMask; cm != AttrCMP16 {
+		t.Errorf("wrong color mode: got 0x%x, want 0x%x (P16)", cm, AttrCMP16)
+	}
+}
+
+func TestSGR_RGBToP16_ClearsStaleRGBBits_Bg(t *testing.T) {
+	t.Parallel()
+	h := newTestHandler()
+	// Set RGB bg #FF8040
+	h.ParseString("\x1b[48;2;255;128;64m")
+	// Switch to P16 green bg (index 2)
+	h.ParseString("\x1b[42m")
+
+	colorBits := h.curAttrData.Bg & AttrRGBMask
+	wantColor := uint32(2)
+	if colorBits != wantColor {
+		t.Errorf("stale RGB bits in Bg: got 0x%06x, want 0x%06x", colorBits, wantColor)
+	}
+	if cm := h.curAttrData.Bg & AttrCMMask; cm != AttrCMP16 {
+		t.Errorf("wrong color mode: got 0x%x, want 0x%x (P16)", cm, AttrCMP16)
+	}
+}
+
+func TestSGR_RGBToP16Bright_ClearsStaleRGBBits(t *testing.T) {
+	t.Parallel()
+	h := newTestHandler()
+	// Set RGB fg
+	h.ParseString("\x1b[38;2;200;100;50m")
+	// Switch to bright red fg (SGR 91 = index 1|8 = 9)
+	h.ParseString("\x1b[91m")
+
+	colorBits := h.curAttrData.Fg & AttrRGBMask
+	wantColor := uint32(1) | 8
+	if colorBits != wantColor {
+		t.Errorf("stale RGB bits in Fg after bright: got 0x%06x, want 0x%06x", colorBits, wantColor)
+	}
+}
+
+func TestSGR_RGBToP256_ClearsStaleRGBBits_Fg(t *testing.T) {
+	t.Parallel()
+	h := newTestHandler()
+	// Set RGB fg #AABBCC
+	h.ParseString("\x1b[38;2;170;187;204m")
+	// Switch to P256 color 196
+	h.ParseString("\x1b[38;5;196m")
+
+	colorBits := h.curAttrData.Fg & AttrRGBMask
+	wantColor := uint32(196)
+	if colorBits != wantColor {
+		t.Errorf("stale RGB bits in Fg: got 0x%06x, want 0x%06x", colorBits, wantColor)
+	}
+	if cm := h.curAttrData.Fg & AttrCMMask; cm != AttrCMP256 {
+		t.Errorf("wrong color mode: got 0x%x, want 0x%x (P256)", cm, AttrCMP256)
+	}
+}
+
+func TestSGR_RGBToP256_ClearsStaleRGBBits_Bg(t *testing.T) {
+	t.Parallel()
+	h := newTestHandler()
+	// Set RGB bg
+	h.ParseString("\x1b[48;2;170;187;204m")
+	// Switch to P256 bg color 42
+	h.ParseString("\x1b[48;5;42m")
+
+	colorBits := h.curAttrData.Bg & AttrRGBMask
+	wantColor := uint32(42)
+	if colorBits != wantColor {
+		t.Errorf("stale RGB bits in Bg: got 0x%06x, want 0x%06x", colorBits, wantColor)
+	}
+	if cm := h.curAttrData.Bg & AttrCMMask; cm != AttrCMP256 {
+		t.Errorf("wrong color mode: got 0x%x, want 0x%x (P256)", cm, AttrCMP256)
+	}
+}
+
 func TestSGR_MultipleResets(t *testing.T) {
 	t.Parallel()
 	h := newTestHandler()