Fixed crash from TaskbarProgress when BuiltInComInteropSupport is disabled.

Fixed terminal sequence printed to console in legacy console.

Author: timcassell

src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs

@@ -9,40 +9,40 @@ internal class TaskbarProgress : IDisposable
             // Must be windows 7 or greater
             && Environment.OSVersion.Version >= new Version(6, 1);
 
-        private IntPtr consoleWindowHandle = IntPtr.Zero;
-        private IntPtr consoleHandle = IntPtr.Zero;
+        private TaskbarProgressCom? com;
+        private TaskbarProgressTerminal? terminal;
 
-        [DllImport("kernel32.dll")]
-        private static extern IntPtr GetConsoleWindow();
-        [DllImport("kernel32.dll", SetLastError = true)]
-        private static extern IntPtr GetStdHandle(int nStdHandle);
+        private bool IsEnabled => com != null || terminal != null;
 
-        private const int STD_OUTPUT_HANDLE = -11;
-
-        internal TaskbarProgress()
+        internal TaskbarProgress(TaskbarProgressState initialTaskbarState)
         {
             if (OsVersionIsSupported)
             {
-                consoleWindowHandle = GetConsoleWindow();
-                consoleHandle = GetStdHandle(STD_OUTPUT_HANDLE);
-                Console.CancelKeyPress += OnConsoleCancelEvent;
+                com = TaskbarProgressCom.MaybeCreateInstanceAndSetInitialState(initialTaskbarState);
+                terminal = TaskbarProgressTerminal.MaybeCreateInstanceAndSetInitialState(initialTaskbarState);
+                if (IsEnabled)
+                {
+                    Console.CancelKeyPress += OnConsoleCancelEvent;
+                }
             }
         }
 
         internal void SetState(TaskbarProgressState state)
         {
-            if (OsVersionIsSupported)
-            {
-                TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, state);
-            }
+            com?.SetState(state);
+            terminal?.SetState(state);
         }
 
         internal void SetProgress(float progressValue)
         {
-            if (OsVersionIsSupported)
+            bool isValidRange = progressValue >= 0 & progressValue <= 1;
+            if (!isValidRange)
             {
-                TaskbarProgressCom.SetValue(consoleWindowHandle, consoleHandle, progressValue);
+                throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive.");
             }
+            uint value = (uint) (progressValue * 100);
+            com?.SetValue(value);
+            terminal?.SetValue(value);
         }
 
         private void OnConsoleCancelEvent(object sender, ConsoleCancelEventArgs e)
@@ -52,11 +52,12 @@ private void OnConsoleCancelEvent(object sender, ConsoleCancelEventArgs e)
 
         public void Dispose()
         {
-            if (OsVersionIsSupported)
+            if (IsEnabled)
             {
-                TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, TaskbarProgressState.NoProgress);
-                consoleWindowHandle = IntPtr.Zero;
-                consoleHandle = IntPtr.Zero;
+                com?.SetState(TaskbarProgressState.NoProgress);
+                terminal?.SetState(TaskbarProgressState.NoProgress);
+                com = null;
+                terminal = null;
                 Console.CancelKeyPress -= OnConsoleCancelEvent;
             }
         }
@@ -72,33 +73,8 @@ internal enum TaskbarProgressState
         Warning = Paused
     }
 
-    internal static class TaskbarProgressCom
+    internal sealed class TaskbarProgressCom
     {
-        [DllImport("kernel32.dll", SetLastError = true)]
-        private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode);
-        [DllImport("kernel32.dll", SetLastError = true)]
-        private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode);
-
-        [Flags]
-        private enum ConsoleModes : uint
-        {
-            ENABLE_PROCESSED_INPUT = 0x0001,
-            ENABLE_LINE_INPUT = 0x0002,
-            ENABLE_ECHO_INPUT = 0x0004,
-            ENABLE_WINDOW_INPUT = 0x0008,
-            ENABLE_MOUSE_INPUT = 0x0010,
-            ENABLE_INSERT_MODE = 0x0020,
-            ENABLE_QUICK_EDIT_MODE = 0x0040,
-            ENABLE_EXTENDED_FLAGS = 0x0080,
-            ENABLE_AUTO_POSITION = 0x0100,
-
-            ENABLE_PROCESSED_OUTPUT = 0x0001,
-            ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002,
-            ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004,
-            DISABLE_NEWLINE_AUTO_RETURN = 0x0008,
-            ENABLE_LVB_GRID_WORLDWIDE = 0x0010
-        }
-
         [ComImport]
         [Guid("ea1afb91-9e28-4b86-90e9-9e9f8a5eefaf")]
         [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
@@ -134,64 +110,149 @@ private class TaskbarInstance
         {
         }
 
-        private static readonly ITaskbarList3 s_taskbarInstance = (ITaskbarList3) new TaskbarInstance();
+        [DllImport("kernel32.dll")]
+        private static extern IntPtr GetConsoleWindow();
 
-        internal static void SetState(IntPtr consoleWindowHandle, IntPtr consoleHandle, TaskbarProgressState taskbarState)
+        private readonly ITaskbarList3 taskbarInstance;
+        private readonly IntPtr consoleWindowHandle;
+
+        private TaskbarProgressCom(IntPtr handle)
         {
-            if (consoleWindowHandle != IntPtr.Zero)
-            {
-                s_taskbarInstance.SetProgressState(consoleWindowHandle, taskbarState);
-            }
+            taskbarInstance = (ITaskbarList3) new TaskbarInstance();
+            consoleWindowHandle = handle;
+        }
 
-            if (consoleHandle != IntPtr.Zero)
+        internal static TaskbarProgressCom? MaybeCreateInstanceAndSetInitialState(TaskbarProgressState initialTaskbarState)
+        {
+            try
             {
-                // Write progress state to console for Windows Terminal (https://github.com/microsoft/terminal/issues/6700).
-                GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode);
-                SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT);
-                switch (taskbarState)
+                IntPtr handle = GetConsoleWindow();
+                if (handle == IntPtr.Zero)
                 {
-                    case TaskbarProgressState.NoProgress:
-                        Console.Write("\x1b]9;4;0;0\x1b\\");
-                        break;
-                    case TaskbarProgressState.Indeterminate:
-                        Console.Write("\x1b]9;4;3;0\x1b\\");
-                        break;
-                    case TaskbarProgressState.Normal:
-                        // Do nothing, this is set automatically when SetValue is called (and WT has no documented way to set this).
-                        break;
-                    case TaskbarProgressState.Error:
-                        Console.Write("\x1b]9;4;2;0\x1b\\");
-                        break;
-                    case TaskbarProgressState.Warning:
-                        Console.Write("\x1b]9;4;4;0\x1b\\");
-                        break;
-                };
-                SetConsoleMode(consoleHandle, previousConsoleMode);
+                    return null;
+                }
+                var com = new TaskbarProgressCom(handle);
+                com.SetState(initialTaskbarState);
+                return com;
+            }
+            // COM may be disabled, in which case this will throw (#2253).
+            catch (NotSupportedException)
+            {
+                return null;
             }
         }
 
-        internal static void SetValue(IntPtr consoleWindowHandle, IntPtr consoleHandle, float progressValue)
+        internal void SetState(TaskbarProgressState taskbarState)
         {
-            bool isValidRange = progressValue >= 0 & progressValue <= 1;
-            if (!isValidRange)
+            taskbarInstance.SetProgressState(consoleWindowHandle, taskbarState);
+        }
+
+        /// <summary>
+        /// Sets the progress value out of 100.
+        /// </summary>
+        internal void SetValue(uint progressValue)
+        {
+            taskbarInstance.SetProgressValue(consoleWindowHandle, progressValue, 100);
+        }
+    }
+
+    internal sealed class TaskbarProgressTerminal
+    {
+        [Flags]
+        private enum ConsoleModes : uint
+        {
+            ENABLE_PROCESSED_INPUT = 0x0001,
+            ENABLE_LINE_INPUT = 0x0002,
+            ENABLE_ECHO_INPUT = 0x0004,
+            ENABLE_WINDOW_INPUT = 0x0008,
+            ENABLE_MOUSE_INPUT = 0x0010,
+            ENABLE_INSERT_MODE = 0x0020,
+            ENABLE_QUICK_EDIT_MODE = 0x0040,
+            ENABLE_EXTENDED_FLAGS = 0x0080,
+            ENABLE_AUTO_POSITION = 0x0100,
+
+            ENABLE_PROCESSED_OUTPUT = 0x0001,
+            ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002,
+            ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004,
+            DISABLE_NEWLINE_AUTO_RETURN = 0x0008,
+            ENABLE_LVB_GRID_WORLDWIDE = 0x0010
+        }
+
+        [DllImport("kernel32.dll", SetLastError = true)]
+        private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode);
+        [DllImport("kernel32.dll", SetLastError = true)]
+        private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode);
+        [DllImport("kernel32.dll", SetLastError = true)]
+        private static extern IntPtr GetStdHandle(int nStdHandle);
+        private const int STD_OUTPUT_HANDLE = -11;
+
+        private readonly IntPtr consoleHandle = IntPtr.Zero;
+
+        private TaskbarProgressTerminal(IntPtr handle)
+        {
+            consoleHandle = handle;
+        }
+
+        internal static TaskbarProgressTerminal? MaybeCreateInstanceAndSetInitialState(TaskbarProgressState initialTaskbarState)
+        {
+            IntPtr handle = GetStdHandle(STD_OUTPUT_HANDLE);
+            if (handle == IntPtr.Zero)
             {
-                throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive.");
+                return null;
             }
-            uint value = (uint) (progressValue * 100);
-
-            if (consoleWindowHandle != IntPtr.Zero)
+            if (!GetConsoleMode(handle, out ConsoleModes previousConsoleMode)
+                || !SetConsoleMode(handle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT))
             {
-                s_taskbarInstance.SetProgressValue(consoleWindowHandle, value, 100);
+                // If we failed to set virtual terminal processing mode, it is likely due to an older Windows version that does not support it,
+                // or legacy console. In either case the TaskbarProgressCom will take care of the progress. See https://stackoverflow.com/a/44574463/5703407.
+                // If we try to write without VT mode, the sequence will be printed for the user to see, which clutters the output.
+                return null;
             }
+            SetStateThenRevertConsoleMode(handle, initialTaskbarState, previousConsoleMode);
+            return new TaskbarProgressTerminal(handle);
+        }
 
-            if (consoleHandle != IntPtr.Zero)
+        internal void SetState(TaskbarProgressState taskbarState)
+        {
+            GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode);
+            SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT);
+            SetStateThenRevertConsoleMode(consoleHandle, taskbarState, previousConsoleMode);
+        }
+
+        private static void SetStateThenRevertConsoleMode(IntPtr handle, TaskbarProgressState taskbarState, ConsoleModes previousConsoleMode)
+        {
+            // Write progress state to console for Windows Terminal (https://github.com/microsoft/terminal/issues/6700).
+            switch (taskbarState)
             {
-                // Write progress sequence to console for Windows Terminal (https://github.com/microsoft/terminal/discussions/14268).
-                GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode);
-                SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT);
-                Console.Write($"\x1b]9;4;1;{value}\x1b\\");
-                SetConsoleMode(consoleHandle, previousConsoleMode);
-            }
+                case TaskbarProgressState.NoProgress:
+                    Console.Write("\x1b]9;4;0;0\x1b\\");
+                    break;
+                case TaskbarProgressState.Indeterminate:
+                    Console.Write("\x1b]9;4;3;0\x1b\\");
+                    break;
+                case TaskbarProgressState.Normal:
+                    // Do nothing, this is set automatically when SetValue is called (and WT has no documented way to set this).
+                    break;
+                case TaskbarProgressState.Error:
+                    Console.Write("\x1b]9;4;2;0\x1b\\");
+                    break;
+                case TaskbarProgressState.Warning:
+                    Console.Write("\x1b]9;4;4;0\x1b\\");
+                    break;
+            };
+            SetConsoleMode(handle, previousConsoleMode);
+        }
+
+        /// <summary>
+        /// Sets the progress value out of 100.
+        /// </summary>
+        internal void SetValue(uint progressValue)
+        {
+            // Write progress sequence to console for Windows Terminal (https://github.com/microsoft/terminal/discussions/14268).
+            GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode);
+            SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT);
+            Console.Write($"\x1b]9;4;1;{progressValue}\x1b\\");
+            SetConsoleMode(consoleHandle, previousConsoleMode);
         }
     }
 }

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -35,8 +35,7 @@ internal static class BenchmarkRunnerClean
 
         internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
         {
-            using var taskbarProgress = new TaskbarProgress();
-            taskbarProgress.SetState(TaskbarProgressState.Indeterminate);
+            using var taskbarProgress = new TaskbarProgress(TaskbarProgressState.Indeterminate);
 
             var resolver = DefaultResolver;
             var artifactsToCleanup = new List<string>();