.NET 8 OS compatibility test (#3422)

* Revert "Warn for soon to be deprecated OS versions (#3413)"

This reverts commit ae04147.

* Add .NET 8 OS compatibility test

* feedback

Author: ericsciple

src/Runner.Common/Constants.cs

@@ -280,6 +280,8 @@ public static class System
                 public static readonly string PhaseDisplayName = "system.phaseDisplayName";
                 public static readonly string JobRequestType = "system.jobRequestType";
                 public static readonly string OrchestrationId = "system.orchestrationId";
+                public static readonly string TestDotNet8Compatibility = "system.testDotNet8Compatibility";
+                public static readonly string DotNet8CompatibilityWarning = "system.dotNet8CompatibilityWarning";
             }
         }
 

src/Runner.Worker/JobExtension.cs

@@ -129,7 +129,7 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
 
                     // Check OS warning
                     var osWarningChecker = HostContext.GetService<IOSWarningChecker>();
-                    await osWarningChecker.CheckOSAsync(context, message.OSWarnings);
+                    await osWarningChecker.CheckOSAsync(context);
 
                     try
                     {

src/Runner.Worker/OSWarningChecker.cs

@@ -1,10 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
-using System.Text.RegularExpressions;
 using GitHub.DistributedTask.WebApi;
-using GitHub.DistributedTask.Pipelines;
 using GitHub.Runner.Common;
 using GitHub.Runner.Sdk;
 
@@ -13,75 +12,85 @@ namespace GitHub.Runner.Worker
     [ServiceLocator(Default = typeof(OSWarningChecker))]
     public interface IOSWarningChecker : IRunnerService
     {
-        Task CheckOSAsync(IExecutionContext context, IList<OSWarning> osWarnings);
+        Task CheckOSAsync(IExecutionContext context);
     }
 
-#if OS_WINDOWS || OS_OSX
     public sealed class OSWarningChecker : RunnerService, IOSWarningChecker
     {
-        public Task CheckOSAsync(IExecutionContext context, IList<OSWarning> osWarnings)
+        public async Task CheckOSAsync(IExecutionContext context)
         {
             ArgUtil.NotNull(context, nameof(context));
-            ArgUtil.NotNull(osWarnings, nameof(osWarnings));
-            return Task.CompletedTask;
-        }
-    }
-#else
-    public sealed class OSWarningChecker : RunnerService, IOSWarningChecker
-    {
-        private static readonly TimeSpan s_matchTimeout = TimeSpan.FromMilliseconds(100);
-        private static readonly RegexOptions s_regexOptions = RegexOptions.CultureInvariant | RegexOptions.IgnoreCase;
-
-        public async Task CheckOSAsync(IExecutionContext context, IList<OSWarning> osWarnings)
-        {
-            ArgUtil.NotNull(context, nameof(context));
-            ArgUtil.NotNull(osWarnings, nameof(osWarnings));
-            foreach (var osWarning in osWarnings)
+            if (!context.Global.Variables.System_TestDotNet8Compatibility)
             {
-                if (string.IsNullOrEmpty(osWarning.FilePath))
-                {
-                    Trace.Error("The file path is not specified in the OS warning check.");
-                    continue;
-                }
+                return;
+            }
 
-                if (string.IsNullOrEmpty(osWarning.RegularExpression))
+            context.Output("Testing runner upgrade compatibility");
+            List<string> output = new();
+            object outputLock = new();
+            try
+            {
+                using (var process = HostContext.CreateService<IProcessInvoker>())
                 {
-                    Trace.Error("The regular expression is not specified in the OS warning check.");
-                    continue;
-                }
+                    process.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
+                    {
+                        if (!string.IsNullOrEmpty(stdout.Data))
+                        {
+                            lock (outputLock)
+                            {
+                                output.Add(stdout.Data);
+                                Trace.Info(stdout.Data);
+                            }
+                        }
+                    };
 
-                if (string.IsNullOrEmpty(osWarning.Warning))
-                {
-                    Trace.Error("The warning message is not specified in the OS warning check.");
-                    continue;
-                }
+                    process.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
+                    {
+                        if (!string.IsNullOrEmpty(stderr.Data))
+                        {
+                            lock (outputLock)
+                            {
+                                output.Add(stderr.Data);
+                                Trace.Error(stderr.Data);
+                            }
+                        }
+                    };
 
-                try
-                {
-                    if (File.Exists(osWarning.FilePath))
+                    using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
                     {
-                        var lines = await File.ReadAllLinesAsync(osWarning.FilePath, context.CancellationToken);
-                        var regex = new Regex(osWarning.RegularExpression, s_regexOptions, s_matchTimeout);
-                        foreach (var line in lines)
+                        int exitCode = await process.ExecuteAsync(
+                            workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
+                            fileName: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "testDotNet8Compatibility", $"TestDotNet8Compatibility{IOUtil.ExeExtension}"),
+                            arguments: string.Empty,
+                            environment: null,
+                            cancellationToken: cancellationTokenSource.Token);
+
+                        var outputStr = string.Join("\n", output).Trim();
+                        if (exitCode != 0 || !string.Equals(outputStr, "Hello from .NET 8!", StringComparison.Ordinal))
                         {
-                            if (regex.IsMatch(line))
+                            var warningMessage = context.Global.Variables.System_DotNet8CompatibilityWarning;
+                            if (!string.IsNullOrEmpty(warningMessage))
                             {
-                                context.Warning(osWarning.Warning);
-                                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"OS warning: {osWarning.Warning}" });
-                                return;
+                                context.Warning(warningMessage);
                             }
+
+                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(output)}" });
                         }
                     }
                 }
-                catch (Exception ex)
-                {
-                    Trace.Error("An error occurred while checking OS warnings for file '{0}' and regex '{1}'.", osWarning.FilePath, osWarning.RegularExpression);
-                    Trace.Error(ex);
-                    context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"An error occurred while checking OS warnings for file '{osWarning.FilePath}' and regex '{osWarning.RegularExpression}': {ex.Message}" });
-                }
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("An error occurred while testing .NET 8 compatibility'");
+                Trace.Error(ex);
+                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(output)}'" });
             }
         }
+
+        private static string GetShortOutput(List<string> output)
+        {
+            var outputStr = string.Join("\n", output).Trim();
+            return outputStr.Length > 200 ? string.Concat(outputStr.Substring(0, 200), "[...]") : outputStr;
+        }
     }
-#endif
 }
-

src/Runner.Worker/Variables.cs

@@ -72,8 +72,12 @@ public Variables(IHostContext hostContext, IDictionary<string, VariableValue> co
 
         public bool? Step_Debug => GetBoolean(Constants.Variables.Actions.StepDebug);
 
+        public string System_DotNet8CompatibilityWarning => Get(Constants.Variables.System.DotNet8CompatibilityWarning);
+
         public string System_PhaseDisplayName => Get(Constants.Variables.System.PhaseDisplayName);
 
+        public bool System_TestDotNet8Compatibility => GetBoolean(Constants.Variables.System.TestDotNet8Compatibility) ?? false;
+
         public string Get(string name)
         {
             Variable variable;

src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs

@@ -44,7 +44,6 @@ public AgentJobRequestMessage(
             IList<TemplateToken> defaults,
             ActionsEnvironmentReference actionsEnvironment,
             TemplateToken snapshot,
-            IList<OSWarning> osWarnings,
             String messageType = JobRequestMessageTypes.PipelineAgentJobRequest)
         {
             this.MessageType = messageType;
@@ -74,11 +73,6 @@ public AgentJobRequestMessage(
                 m_defaults = new List<TemplateToken>(defaults);
             }
 
-            if (osWarnings?.Count > 0)
-            {
-                m_osWarnings = new List<OSWarning>(osWarnings);
-            }
-
             this.ContextData = new Dictionary<String, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
             if (contextData?.Count > 0)
             {
@@ -294,18 +288,6 @@ public IList<String> FileTable
             }
         }
 
-        public IList<OSWarning> OSWarnings
-        {
-            get
-            {
-                if (m_osWarnings == null)
-                {
-                    m_osWarnings = new List<OSWarning>();
-                }
-                return m_osWarnings;
-            }
-        }
-
         // todo: remove after feature-flag DistributedTask.EvaluateContainerOnRunner is enabled everywhere
         public void SetJobSidecarContainers(IDictionary<String, String> value)
         {
@@ -443,11 +425,6 @@ private void OnSerializing(StreamingContext context)
             {
                 JobContainer = new StringToken(null, null, null, m_jobContainerResourceAlias);
             }
-
-            if (m_osWarnings?.Count == 0)
-            {
-                m_osWarnings = null;
-            }
         }
 
         [DataMember(Name = "EnvironmentVariables", EmitDefaultValue = false)]
@@ -472,9 +449,6 @@ private void OnSerializing(StreamingContext context)
         [DataMember(Name = "JobSidecarContainers", EmitDefaultValue = false)]
         private IDictionary<String, String> m_jobSidecarContainers;
 
-        [DataMember(Name = "OSWarnings", EmitDefaultValue = false)]
-        private List<OSWarning> m_osWarnings;
-
         // todo: remove after feature-flag DistributedTask.EvaluateContainerOnRunner is enabled everywhere
         [IgnoreDataMember]
         private string m_jobContainerResourceAlias;

src/Sdk/DTPipelines/Pipelines/OSWarning.cs

@@ -41,7 +41,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage()
             TaskOrchestrationPlanReference plan = new();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null, null);
+            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
             result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
             return result;
         }
@@ -810,7 +810,6 @@ private static AgentJobRequestMessage GetAgentJobRequestMessage()
                 null,
                 new List<TemplateToken>(),
                 new ActionsEnvironmentReference("env"),
-                null,
                 null
             );
             return message;

src/Test/L0/Listener/JobDispatcherL0.cs

@@ -44,7 +44,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName)
             TaskOrchestrationPlanReference plan = new();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null, null);
+            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
         }
 
         private JobCancelMessage CreateJobCancelMessage()

src/Test/L0/Listener/RunnerL0.cs

@@ -232,7 +232,7 @@ public void EchoProcessCommandDebugOn()
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,

src/Test/L0/Worker/ActionCommandManagerL0.cs

@@ -193,7 +193,7 @@ private TestHostContext Setup([CallerMemberName] string name = "")
             TimelineReference timeline = new();
             Guid jobId = Guid.NewGuid();
             string jobName = "Summary Job";
-            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null, null);
+            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
             jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
             {
                 Alias = Pipelines.PipelineConstants.SelfAlias,