Merge pull request #1626 from iodic/fix/utf8-pty-render

fix(pty): preserve UTF-8 locale for embedded terminals

Author: arnestrickmann

src/main/services/ptyManager.ts

@@ -2,12 +2,14 @@ import os from 'os';
 import fs from 'fs';
 import path from 'path';
 import crypto from 'crypto';
+import { StringDecoder } from 'string_decoder';
 import type { IPty } from 'node-pty';
 import { log } from '../lib/logger';
 import { PROVIDERS, type ProviderDefinition } from '@shared/providers/registry';
 import { parsePtyId } from '@shared/ptyId';
 import { providerStatusCache } from './providerStatusCache';
 import { errorTracking } from '../errorTracking';
+import { LOCALE_ENV_VARS, DEFAULT_UTF8_LOCALE, isUtf8Locale } from '../utils/locale';
 
 /**
  * Suppress EPIPE/EIO errors on a PTY's underlying socket.
@@ -103,6 +105,64 @@ type PtyRecord = {
 const ptys = new Map<string, PtyRecord>();
 const MIN_PTY_COLS = 2;
 const MIN_PTY_ROWS = 1;
+export function getLocaleEnv(sourceEnv: NodeJS.ProcessEnv = process.env): Record<string, string> {
+  if (process.platform === 'win32') {
+    const localeEnv: Record<string, string> = {};
+    for (const key of LOCALE_ENV_VARS) {
+      const value = sourceEnv[key];
+      if (value && isUtf8Locale(value)) {
+        localeEnv[key] = value;
+      }
+    }
+    return localeEnv;
+  }
+
+  // On non-Windows, preserve explicit UTF-8 locale choices and only fall back
+  // to a minimal UTF-8 locale when no effective UTF-8 locale is available.
+  const localeEnv: Record<string, string> = {};
+  const lang = sourceEnv.LANG;
+  const lcAll = sourceEnv.LC_ALL;
+  const lcCtype = sourceEnv.LC_CTYPE;
+
+  if (lcAll && isUtf8Locale(lcAll)) {
+    localeEnv.LC_ALL = lcAll;
+  }
+  if (lang && isUtf8Locale(lang)) {
+    localeEnv.LANG = lang;
+  }
+  if (lcCtype && isUtf8Locale(lcCtype)) {
+    localeEnv.LC_CTYPE = lcCtype;
+  }
+
+  if (localeEnv.LC_ALL || localeEnv.LANG || localeEnv.LC_CTYPE) {
+    return localeEnv;
+  }
+
+  localeEnv.LANG = DEFAULT_UTF8_LOCALE;
+  localeEnv.LC_CTYPE = DEFAULT_UTF8_LOCALE;
+  return localeEnv;
+}
+
+export function mergeEnvWithNormalizedLocale(
+  ...envs: Array<NodeJS.ProcessEnv | undefined>
+): Record<string, string> {
+  const mergedEnv: NodeJS.ProcessEnv = {};
+
+  for (const env of envs) {
+    if (!env) continue;
+    Object.assign(mergedEnv, env);
+  }
+
+  const localeEnv = getLocaleEnv(mergedEnv);
+  for (const key of LOCALE_ENV_VARS) {
+    delete mergedEnv[key];
+  }
+
+  return {
+    ...mergedEnv,
+    ...localeEnv,
+  } as Record<string, string>;
+}
 
 function applyAgentEventHookEnv(env: Record<string, string>, ptyId: string): void {
   const hookPort = agentEventService.getPort();
@@ -1036,7 +1096,7 @@ export function startSshPty(options: {
     HOME: process.env.HOME || os.homedir(),
     USER: process.env.USER || os.userInfo().username,
     PATH: process.env.PATH || process.env.Path || '',
-    ...(process.env.LANG && { LANG: process.env.LANG }),
+    ...getLocaleEnv(),
     ...(process.env.TMPDIR && { TMPDIR: process.env.TMPDIR }),
     ...getDisplayEnv(),
     ...(process.env.SSH_AUTH_SOCK && { SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK }),
@@ -1201,7 +1261,7 @@ export function startDirectPty(options: {
     USER: process.env.USER || os.userInfo().username,
     // Include PATH so CLI can find its dependencies
     PATH: process.env.PATH || process.env.Path || '',
-    ...(process.env.LANG && { LANG: process.env.LANG }),
+    ...getLocaleEnv(),
     ...(process.env.TMPDIR && { TMPDIR: process.env.TMPDIR }),
     ...getDisplayEnv(),
     ...(process.env.SSH_AUTH_SOCK && { SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK }),
@@ -1323,21 +1383,20 @@ export async function startPty(options: {
   // tools create clean user environments.
   //
   // See: https://github.com/generalaction/emdash/issues/485
-  const useEnv: Record<string, string> = {
+  const useEnv = mergeEnvWithNormalizedLocale({
     TERM: 'xterm-256color',
     COLORTERM: 'truecolor',
     TERM_PROGRAM: 'emdash',
     HOME: process.env.HOME || os.homedir(),
     USER: process.env.USER || os.userInfo().username,
     SHELL: process.env.SHELL || defaultShell,
     ...(process.platform === 'win32' ? getWindowsEssentialEnv() : {}),
-    ...(process.env.LANG && { LANG: process.env.LANG }),
     ...(process.env.TMPDIR && { TMPDIR: process.env.TMPDIR }),
     ...(process.env.DISPLAY && { DISPLAY: process.env.DISPLAY }),
     ...getDisplayEnv(),
     ...(process.env.SSH_AUTH_SOCK && { SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK }),
     ...(env || {}),
-  };
+  });
 
   applyAgentEventHookEnv(useEnv, id);
 
@@ -1651,6 +1710,82 @@ export interface LifecyclePtyHandle {
   kill: (signal?: string) => void;
 }
 
+export function createUtf8StreamForwarder(emitData: (data: string) => void): {
+  pushStdout: (buf: Buffer) => void;
+  pushStderr: (buf: Buffer) => void;
+  flush: () => void;
+} {
+  const stdoutDecoder = new StringDecoder('utf8');
+  const stderrDecoder = new StringDecoder('utf8');
+  let flushed = false;
+
+  const emitIfPresent = (data: string) => {
+    if (!data) return;
+    emitData(data);
+  };
+
+  return {
+    pushStdout: (buf: Buffer) => {
+      emitIfPresent(stdoutDecoder.write(buf));
+    },
+    pushStderr: (buf: Buffer) => {
+      emitIfPresent(stderrDecoder.write(buf));
+    },
+    flush: () => {
+      if (flushed) return;
+      flushed = true;
+      emitIfPresent(stdoutDecoder.end());
+      emitIfPresent(stderrDecoder.end());
+    },
+  };
+}
+
+type LifecycleSpawnFallbackChild = {
+  stdout?: { on: (event: 'data', listener: (buf: Buffer) => void) => void } | null;
+  stderr?: { on: (event: 'data', listener: (buf: Buffer) => void) => void } | null;
+  on: (event: 'error' | 'exit' | 'close', listener: (...args: any[]) => void) => void;
+};
+
+export function attachLifecycleSpawnFallbackHandlers(
+  child: LifecycleSpawnFallbackChild,
+  callbacks: {
+    onData: (data: string) => void;
+    onExit: (exitCode: number | null, signal: string | null) => void;
+    onError: (error: Error) => void;
+  }
+): void {
+  const { onData, onExit, onError } = callbacks;
+  let didExit = false;
+  let exitCode: number | null = null;
+  let exitSignal: string | null = null;
+  const forwarder = createUtf8StreamForwarder(onData);
+
+  child.stdout?.on('data', (buf: Buffer) => {
+    forwarder.pushStdout(buf);
+  });
+  child.stderr?.on('data', (buf: Buffer) => {
+    forwarder.pushStderr(buf);
+  });
+
+  child.on('error', (error: Error) => {
+    forwarder.flush();
+    onError(error);
+  });
+
+  child.on('exit', (code: number | null, signal: string | null) => {
+    didExit = true;
+    exitCode = code;
+    exitSignal = signal ?? null;
+  });
+
+  child.on('close', () => {
+    // Flush only after stdio closes so buffered UTF-8 bytes can complete.
+    forwarder.flush();
+    if (!didExit) return;
+    onExit(exitCode, exitSignal);
+  });
+}
+
 function startLifecycleSpawnFallback(options: {
   id: string;
   command: string;
@@ -1664,26 +1799,22 @@ function startLifecycleSpawnFallback(options: {
     cwd: cwd || os.homedir(),
     shell: true,
     detached: true,
-    env: { ...process.env, ...(env || {}) },
+    env: mergeEnvWithNormalizedLocale(process.env, env),
   });
 
   const dataCallbacks: Array<(data: string) => void> = [];
   const exitCallbacks: Array<(exitCode: number | null, signal: string | null) => void> = [];
   const errorCallbacks: Array<(error: Error) => void> = [];
-
-  const onData = (buf: Buffer) => {
-    const str = buf.toString();
-    for (const cb of dataCallbacks) cb(str);
-  };
-  child.stdout?.on('data', onData);
-  child.stderr?.on('data', onData);
-
-  child.on('error', (error: Error) => {
-    for (const cb of errorCallbacks) cb(error);
-  });
-
-  child.on('exit', (code, signal) => {
-    for (const cb of exitCallbacks) cb(code, signal ?? null);
+  attachLifecycleSpawnFallbackHandlers(child, {
+    onData: (data) => {
+      for (const cb of dataCallbacks) cb(data);
+    },
+    onExit: (code, signal) => {
+      for (const cb of exitCallbacks) cb(code, signal);
+    },
+    onError: (error) => {
+      for (const cb of errorCallbacks) cb(error);
+    },
   });
 
   return {
@@ -1731,21 +1862,20 @@ export function startLifecyclePty(options: {
   const { id, command, cwd, env } = options;
   const defaultShell = getDefaultShell();
 
-  const useEnv: Record<string, string> = {
+  const useEnv = mergeEnvWithNormalizedLocale({
     TERM: 'xterm-256color',
     COLORTERM: 'truecolor',
     TERM_PROGRAM: 'emdash',
     HOME: process.env.HOME || os.homedir(),
     USER: process.env.USER || os.userInfo().username,
     SHELL: process.env.SHELL || defaultShell,
     ...(process.platform === 'win32' ? getWindowsEssentialEnv() : {}),
-    ...(process.env.LANG && { LANG: process.env.LANG }),
     ...(process.env.TMPDIR && { TMPDIR: process.env.TMPDIR }),
     ...(process.env.DISPLAY && { DISPLAY: process.env.DISPLAY }),
     ...getDisplayEnv(),
     ...(process.env.SSH_AUTH_SOCK && { SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK }),
     ...(env || {}),
-  };
+  });
 
   const proc = pty.spawn(defaultShell, ['-ilc', command], {
     name: 'xterm-256color',