diff --git a/docs/powershell-startup-detection.md b/docs/powershell-startup-detection.md new file mode 100644 index 000000000..3937d0933 --- /dev/null +++ b/docs/powershell-startup-detection.md @@ -0,0 +1,48 @@ +# PowerShell Startup Detection + +## Background + +When `powershell.exe` is invoked to run a script it can occasionally start the OS process but silently stall before executing +any script content — typically due to Group Policy, antivirus, or other startup hooks that block or hang the PowerShell host. +Tentacle had no way to distinguish this from a legitimately long-running script, so affected deployments would hang indefinitely +with no useful error. + +The startup detection mechanism lets Tentacle detect and report when PowerShell starts but never executes the script body. + +## Opting in + +Detection is opt-in. To enable it, include the following marker comment somewhere near the top of your script body (before any code you want to run): + +```powershell +# TENTACLE-POWERSHELL-STARTUP-DETECTION +``` + +When Tentacle bootstraps the script it replaces this marker with generated detection code. Scripts that do not include the marker are completely unaffected. + +## What happens at runtime + +When the marker is present, Tentacle: + +1. Writes a `.octopus_powershell_should_run` file to the script workspace before launching PowerShell. +2. Replaces the marker with generated PowerShell that, at the point it executes: + - Exclusively creates a `.octopus_powershell_started` sentinel file. + - Verifies the `.octopus_powershell_should_run` file still exists. + - Exits with code `-47` if either check fails (meaning the monitor already concluded PowerShell never started). +3. Runs a monitoring task concurrently with script execution. If the sentinel file is not created within the startup timeout the monitor: + - Creates the sentinel itself (so any late-starting PowerShell process will exit immediately). + - Deletes the should-run file (so any late-starting process that somehow missed the sentinel will also exit). + - Returns exit code `-47` (`PowerShellNeverStartedExitCode`) with a "process did not start within…" message written to the task log. + +## Startup timeout + +The default timeout is **5 minutes**. It can be overridden by setting the environment variable: + +``` +TentaclePowerShellStartupTimeout= +``` + +For example, `00:02:00` for a 2-minute timeout. + +## Platform support + +Currently scoped to `powershell.exe`. \ No newline at end of file diff --git a/source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs b/source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs new file mode 100644 index 000000000..3a2b9471d --- /dev/null +++ b/source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs @@ -0,0 +1,39 @@ +using System; +using Octopus.Tentacle.Core.Diagnostics; +using Octopus.Tentacle.Diagnostics; +using Serilog; +using Serilog.Events; +using LogEvent = Octopus.Tentacle.Diagnostics.LogEvent; + +namespace Octopus.Tentacle.CommonTestUtils.Diagnostics +{ + public class SerilogSystemLog : SystemLog + { + readonly ILogger logger; + + public SerilogSystemLog(ILogger logger) + { + this.logger = logger; + } + + protected override void WriteEvent(LogEvent logEvent) + { + var level = ToSerilogLevel(logEvent.Category); + if (logEvent.Error != null) + logger.Write(level, logEvent.Error, logEvent.MessageText); + else + logger.Write(level, logEvent.MessageText); + } + + static LogEventLevel ToSerilogLevel(LogCategory category) => category switch + { + LogCategory.Trace => LogEventLevel.Verbose, + LogCategory.Verbose => LogEventLevel.Debug, + LogCategory.Info => LogEventLevel.Information, + LogCategory.Warning => LogEventLevel.Warning, + LogCategory.Error => LogEventLevel.Error, + LogCategory.Fatal => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } +} diff --git a/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs b/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs new file mode 100644 index 000000000..a9b754306 --- /dev/null +++ b/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs @@ -0,0 +1,7 @@ +namespace Octopus.Tentacle.Contracts +{ + public static class PowerShellStartupDetectionTemplateValues + { + public const string PowershellStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs b/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs index c58468d88..2e0ce1b15 100644 --- a/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs +++ b/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs @@ -11,6 +11,7 @@ public static class ScriptExitCodes public const int TimeoutExitCode = -44; public const int UnknownScriptExitCode = -45; public const int UnknownResultExitCode = -46; + public const int PowerShellNeverStartedExitCode = -47; //Kubernetes Agent public const int KubernetesScriptPodNotFound = -81; diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs new file mode 100644 index 000000000..51c8e7bce --- /dev/null +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -0,0 +1,121 @@ +using System; +using System.IO; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Core.Util; + +namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup +{ + /// + /// Provides opt-in detection of PowerShell processes that start but never execute the script body. + /// + /// + /// + /// When powershell.exe is invoked to run a script it can occasionally start the OS process + /// but silently stall before executing any script content. This was seen happening because + /// CrowdStrike prevented the script body from running. + /// + /// When this happens, we get no output from the script AND the script is un-killable. + /// + /// To work-around this, powershell scripts can place: # TENTACLE-POWERSHELL-STARTUP-DETECTION + /// at the start. This will result in Tentacle being able to detect if the script has started + /// or not. If it has not started, Tentacle will prevent the script from ever progressing and + /// report the script as failed to start. + /// + /// + /// How it works
+ /// Scripts opt in by including the marker comment # TENTACLE-POWERSHELL-STARTUP-DETECTION + /// at the start of the script body, before any work is done in the script. + /// When Tentacle bootstraps the script via + /// , the marker is replaced with generated PowerShell that: + /// + /// Attempts to exclusively create a .octopus_powershell_started sentinel file. + /// If the file already exists (because beat it to + /// the punch after the timeout), the script exits with code -47. + /// Checks that the .octopus_powershell_should_run file written by Tentacle before + /// launch is still present. If it has been deleted by cleanup workspace or by the monitor, + /// the script exits with code -47. + /// + /// When we run the script we also run which waits for the timeout window + /// (default 5 minutes, overridable via TentaclePowerShellStartupTimeout). It will then attempt + /// to exclusively create the .octopus_powershell_started. + /// - If it can make the file, the running script is cancelled, although it likely will not cancel. Tentacle + /// returns exit code -47, without waiting for the script to finish. + /// - If it can't make the file, then the script started and Tentacle simply waits for the script to complete. + /// + /// + /// Design notes
+ /// Detection is entirely opt-in; scripts without the marker are unaffected. + /// + /// + public static class PowerShellStartupDetection + { + public const string StartedFileName = ".octopus_powershell_started"; + public const string ShouldRunFileName = ".octopus_powershell_should_run"; + + public static TimeSpan PowerShellStartupTimeout + { + get + { + var raw = Environment.GetEnvironmentVariable(EnvironmentVariables.TentaclePowerShellStartupTimeout); + return TimeSpan.TryParse(raw, out var value) ? value : TimeSpan.FromMinutes(5); + } + } + + public static string GetStartedFilePath(string workingDirectory) + { + return Path.Combine(workingDirectory, StartedFileName); + } + + public static string GetShouldRunFilePath(string workingDirectory) + { + return Path.Combine(workingDirectory, ShouldRunFileName); + } + + public static string GenerateDetectionCode() + { + return $@" +# PowerShell startup detection code (auto-generated by Octopus Tentacle) +& {{ + $startedFile = '{StartedFileName}' + $shouldRunFile = '{ShouldRunFileName}' + + try {{ + # Try to create the started file exclusively + $fileStream = [System.IO.File]::Open($startedFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + $fileStream.Close() + $fileStream.Dispose() + }} catch {{ + # If we couldn't create the file, it means either: + # 1. Another PowerShell instance already created it (race condition) + # 2. Tentacle already created it (meaning we never started) + # In either case, we should exit + write-output ""PowerShell startup detection: Started file already exists, exiting"" + exit -47 + }} + + # Check if the should-run file exists + if (-not (Test-Path $shouldRunFile)) {{ + write-output ""PowerShell startup detection: Should-run file does not exist, exiting"" + exit -47 + }} +}} +"; + } + + public static bool ScriptContainsPowershellStartupDetectionComment(string scriptBody) + { + return scriptBody.Contains(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionComment); + } + + public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody) + { + if (!ScriptContainsPowershellStartupDetectionComment(scriptBody)) + { + return (scriptBody, false); + } + + var detectionCode = GenerateDetectionCode(); + return (scriptBody.Replace(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionComment, detectionCode), true); + } + } +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs new file mode 100644 index 000000000..1c40c86dd --- /dev/null +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs @@ -0,0 +1,97 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Halibut.Util; +using Octopus.Tentacle.Core.Diagnostics; + +namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup +{ + class PowerShellStartupMonitor + { + readonly string workSpaceWorkingDirectory; + readonly TimeSpan powerShellStartupTimeout; + readonly ILog log; + readonly string taskId; + + public PowerShellStartupMonitor(string workSpaceWorkingDirectory, TimeSpan powerShellStartupTimeout, ILog log, string taskId) + { + this.workSpaceWorkingDirectory = workSpaceWorkingDirectory; + this.powerShellStartupTimeout = powerShellStartupTimeout; + this.log = log; + this.taskId = taskId; + } + + public Task WaitForStartup(CancellationToken cancellationToken) + { + var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workSpaceWorkingDirectory); + + if (!File.Exists(shouldRunFilePath)) + { + return Task.FromResult(PowerShellStartupStatus.NotMonitored); + } + + return Task.Run(async () => + { + try + { + await DelayWithoutException.Delay(powerShellStartupTimeout, cancellationToken); + + if (cancellationToken.IsCancellationRequested) + { + return PowerShellStartupStatus.NotMonitored; + } + + try + { + var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workSpaceWorkingDirectory); + // If we can make the file, then the script has not started. + using var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None); + log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupTimeout.TotalMinutes} minutes for task {taskId}"); + DeleteShouldRunFileToEnsureThePowerShellCanNeverStart(); + return PowerShellStartupStatus.NeverStarted; + } + catch (IOException) + { + log.Info($"PowerShell startup detection: PowerShell started late (after {powerShellStartupTimeout.TotalMinutes} minutes) for task {taskId}"); + return PowerShellStartupStatus.Started; + } + } + catch (OperationCanceledException) + { + return PowerShellStartupStatus.NotMonitored; + } + catch (Exception ex) + { + log.Warn(ex, $"Error in PowerShell startup monitoring for task {taskId}"); + return PowerShellStartupStatus.NotMonitored; + } + }); + } + + /// + /// Deletes the should-run file so that if PowerShell does eventually start, the startup guard + /// will detect its absence and exit immediately. + /// + /// Without this, a race exists: if the workspace is cleaned up while PowerShell is blocked, + /// the started sentinel file disappears. The guard would then be able to create it successfully + /// and incorrectly conclude that it is safe to proceed. Deleting the should-run file closes + /// that gap — the guard always checks for it and exits with code -47 if it is missing. + /// + public void DeleteShouldRunFileToEnsureThePowerShellCanNeverStart() + { + try + { + var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workSpaceWorkingDirectory); + if (File.Exists(shouldRunFilePath)) + { + File.Delete(shouldRunFilePath); + } + } + catch (Exception ex) + { + log.Warn(ex, $"Failed to delete should-run file for task {taskId}"); + } + } + } +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupStatus.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupStatus.cs new file mode 100644 index 000000000..0f48c31ff --- /dev/null +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupStatus.cs @@ -0,0 +1,9 @@ +namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup +{ + public enum PowerShellStartupStatus + { + NotMonitored, + Started, + NeverStarted + } +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 33090d372..3ed08eb59 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -1,11 +1,13 @@ -using System; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Halibut.Util; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Core.Diagnostics; using Octopus.Tentacle.Core.Services.Scripts.Locking; using Octopus.Tentacle.Core.Services.Scripts.Logging; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Core.Services.Scripts.StateStore; using Octopus.Tentacle.Scripts; @@ -19,10 +21,11 @@ public class RunningScript: IRunningScript readonly IScriptStateStore? stateStore; readonly IShell shell; readonly string taskId; - readonly CancellationToken token; + readonly CancellationToken runningScriptToken; readonly IReadOnlyDictionary environmentVariables; readonly ILog log; readonly ScriptIsolationMutex scriptIsolationMutex; + readonly TimeSpan powerShellStartupTimeout; public RunningScript(IShell shell, IScriptWorkspace workspace, @@ -30,20 +33,23 @@ public RunningScript(IShell shell, IScriptLog scriptLog, string taskId, ScriptIsolationMutex scriptIsolationMutex, - CancellationToken token, + CancellationToken runningScriptToken, IReadOnlyDictionary environmentVariables, - ILog log) + TimeSpan powerShellStartupTimeout, + ILog log + ) { this.shell = shell; this.workspace = workspace; this.stateStore = stateStore; this.taskId = taskId; - this.token = token; + this.runningScriptToken = runningScriptToken; this.environmentVariables = environmentVariables; this.log = log; this.scriptIsolationMutex = scriptIsolationMutex; this.ScriptLog = scriptLog; this.State = ProcessState.Pending; + this.powerShellStartupTimeout = powerShellStartupTimeout; } public RunningScript(IShell shell, @@ -51,9 +57,10 @@ public RunningScript(IShell shell, IScriptLog scriptLog, string taskId, ScriptIsolationMutex scriptIsolationMutex, - CancellationToken token, + CancellationToken runningScriptToken, IReadOnlyDictionary environmentVariables, - ILog log) : this(shell, workspace, null, scriptLog, taskId, scriptIsolationMutex, token, environmentVariables, log) + TimeSpan powerShellStartupTimeout, + ILog log) : this(shell, workspace, null, scriptLog, taskId, scriptIsolationMutex, runningScriptToken, environmentVariables, powerShellStartupTimeout, log) { } @@ -63,7 +70,7 @@ public RunningScript(IShell shell, public IScriptLog ScriptLog { get; } public Task Cleanup(CancellationToken cancellationToken) => Task.CompletedTask; - public void Execute() + public async Task Execute() { var exitCode = -1; @@ -80,14 +87,16 @@ public void Execute() workspace.ScriptMutexName ?? nameof(RunningScript), message => writer.WriteOutput(ProcessOutputSource.StdOut, message), taskId, - token, + runningScriptToken, log)) { State = ProcessState.Running; RecordScriptHasStarted(writer); - exitCode = RunScript(shellPath, writer); + exitCode = workspace.ShouldMonitorPowerShellStartup() + ? await RunPowershellScriptWithMonitoring(shellPath, writer) + : RunScript(shellPath, writer, runningScriptToken); } } catch (OperationCanceledException) @@ -121,6 +130,42 @@ public void Execute() } } + async Task RunPowershellScriptWithMonitoring(string shellPath, IScriptLogWriter writer) + { + // We want to be able to make some effort to cancel the running script, if the monitor task detects it as hung. + // Hence, we make a linked cancellation token with the runningScriptToken + await using var scriptTaskCts = new CancelOnDisposeCancellationToken(runningScriptToken); + + // The monitoring task is NOT linked to the runningScriptToken, since it should keep monitoring even if an attempt to + // cancel the script is made. Remember, these hung powershell scripts WILL NOT CANCEL, so we must continue to monitor. + await using var monitoringTaskCts = new CancelOnDisposeCancellationToken(); + + var monitor = new PowerShellStartupMonitor(workspace.WorkingDirectory, powerShellStartupTimeout, log, taskId); + + var monitoringTask = monitor.WaitForStartup(monitoringTaskCts.Token); + var scriptTask = Task.Run(() => RunScript(shellPath, writer, scriptTaskCts.Token), scriptTaskCts.Token); + + var completedTask = await Task.WhenAny(monitoringTask, scriptTask); + + if (completedTask == monitoringTask) + { + var startupStatus = await monitoringTask; + + if (startupStatus == PowerShellStartupStatus.NeverStarted) + { + // PowerShell never started - exit immediately with appropriate code + writer.WriteOutput(ProcessOutputSource.StdErr, + $"{shellPath} process did not start within {powerShellStartupTimeout.TotalMinutes} minutes. Script execution aborted."); + + return ScriptExitCodes.PowerShellNeverStartedExitCode; + } + } + + var exitCode = await scriptTask; + + return exitCode; + } + void RecordScriptHasStarted(IScriptLogWriter writer) { try @@ -170,7 +215,7 @@ void RecordScriptHasCompleted(int exitCode) } } - int RunScript(string shellPath, IScriptLogWriter writer) + int RunScript(string shellPath, IScriptLogWriter writer, CancellationToken cancellationToken) { try { @@ -182,7 +227,7 @@ int RunScript(string shellPath, IScriptLogWriter writer) LogScriptOutputTo(writer, ProcessOutputSource.StdOut), LogScriptOutputTo(writer, ProcessOutputSource.StdErr), environmentVariables, - token); + cancellationToken); return exitCode; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs index 67203fc6f..0a200fc19 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs @@ -9,10 +9,10 @@ using Octopus.Tentacle.Core.Diagnostics; using Octopus.Tentacle.Core.Maintenance; using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Core.Services.Scripts.StateStore; using Octopus.Tentacle.Scripts; -using Octopus.Tentacle.Services; using Octopus.Tentacle.Services.Scripts; using Octopus.Tentacle.Util; @@ -28,6 +28,7 @@ public class ScriptServiceV2 : IAsyncScriptServiceV2, IRunningScriptReporter readonly ScriptIsolationMutex scriptIsolationMutex; readonly ConcurrentDictionary runningScripts = new(); readonly IReadOnlyDictionary environmentVariables; + readonly TimeSpan powerShellStartupTimeout; public ScriptServiceV2( IShell shell, @@ -35,13 +36,15 @@ public ScriptServiceV2( IScriptStateStoreFactory scriptStateStoreFactory, ScriptIsolationMutex scriptIsolationMutex, ISystemLog log, - IReadOnlyDictionary environmentVariables) + IReadOnlyDictionary environmentVariables, + TimeSpan powerShellStartupTimeout) { this.shell = shell; this.workspaceFactory = workspaceFactory; this.scriptStateStoreFactory = scriptStateStoreFactory; this.log = log; this.environmentVariables = environmentVariables; + this.powerShellStartupTimeout = powerShellStartupTimeout; this.scriptIsolationMutex = scriptIsolationMutex; } @@ -50,7 +53,7 @@ public ScriptServiceV2( IScriptWorkspaceFactory workspaceFactory, IScriptStateStoreFactory scriptStateStoreFactory, ScriptIsolationMutex scriptIsolationMutex, - ISystemLog log) : this(shell, workspaceFactory, scriptStateStoreFactory, scriptIsolationMutex, log, new Dictionary()) + ISystemLog log) : this(shell, workspaceFactory, scriptStateStoreFactory, scriptIsolationMutex, log, new Dictionary(), PowerShellStartupDetection.PowerShellStartupTimeout) { } @@ -96,7 +99,11 @@ public async Task StartScriptAsync(StartScriptCommandV2 runningScript.ScriptStateStore.Create(); } - var process = LaunchShell(command.ScriptTicket, command.TaskId, workspace, runningScript.ScriptStateStore, runningScript.CancellationToken); + var process = LaunchShell(command.ScriptTicket, + command.TaskId, + workspace, + runningScript.ScriptStateStore, + runningScript.CancellationToken); runningScript.Process = process; @@ -148,9 +155,8 @@ public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, Cancellat RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, CancellationToken cancellationToken) { - var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, log); - var thread = new Thread(runningScript.Execute) { Name = "Executing PowerShell runningScript for " + ticket.TaskId }; - thread.Start(); + var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, powerShellStartupTimeout, log); + _ = Task.Run(async () => await runningScript.Execute()); return runningScript; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs index cb346576a..2b32e4114 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs @@ -27,6 +27,8 @@ public override void BootstrapScript(string scriptBody) FileSystem.OverwriteFile(BootstrapScriptFilePath, scriptBody, Encoding.Default); } + public override bool ShouldMonitorPowerShellStartup() => false; + public static string GetBashBootstrapScriptFilePath(string workspaceDirectory) => Path.Combine(workspaceDirectory, BootstrapScriptFileName); } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs index ee6fdb849..c28d9e58c 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs @@ -1,5 +1,4 @@ using System; -using System.IO; using System.Threading; using System.Threading.Tasks; using Octopus.Tentacle.Contracts; @@ -16,6 +15,7 @@ public interface IScriptWorkspace ScriptIsolationLevel IsolationLevel { get; set; } TimeSpan ScriptMutexAcquireTimeout { get; set; } string? ScriptMutexName { get; set; } + bool ShouldMonitorPowerShellStartup(); void BootstrapScript(string scriptBody); string ResolvePath(string fileName); Task Delete(CancellationToken cancellationToken); diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs index 112e15e5d..f8ed13c4d 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs @@ -4,10 +4,11 @@ using System.Threading; using System.Threading.Tasks; using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Core.Services.Scripts; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Locking; using Octopus.Tentacle.Core.Services.Scripts.Logging; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; -using Octopus.Tentacle.Services.Scripts; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Scripts @@ -69,7 +70,7 @@ public TimeSpan ScriptMutexAcquireTimeout } public string? ScriptMutexName { get; set; } - + public string[]? ScriptArguments { get; set; } public string WorkingDirectory { get; } @@ -78,9 +79,23 @@ public TimeSpan ScriptMutexAcquireTimeout public virtual void BootstrapScript(string scriptBody) { + var (processedScriptBody, shouldMonitorPowerShellStartup) = PowerShellStartupDetection.InjectDetectionCode(scriptBody); + this.shouldMonitorPowerShellStartup = shouldMonitorPowerShellStartup; + + if (shouldMonitorPowerShellStartup) + { + // Create the "should run" file to signal that the script should proceed + var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); + FileSystem.OverwriteFile(shouldRunFile, ""); + } + // default is UTF8noBOM but powershell doesn't interpret that correctly - FileSystem.OverwriteFile(BootstrapScriptFilePath, scriptBody, Encoding.UTF8); + FileSystem.OverwriteFile(BootstrapScriptFilePath, processedScriptBody, Encoding.UTF8); } + + public virtual bool ShouldMonitorPowerShellStartup() => shouldMonitorPowerShellStartup; + + bool shouldMonitorPowerShellStartup; public string ResolvePath(string fileName) { diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs index 5fc53a003..7a3821a14 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs @@ -19,15 +19,25 @@ public class ScriptWorkspaceFactory : IScriptWorkspaceFactory readonly IOctopusFileSystem fileSystem; readonly IHomeDirectoryProvider home; readonly SensitiveValueMasker sensitiveValueMasker; + readonly bool useBashWorkspace; public ScriptWorkspaceFactory( IOctopusFileSystem fileSystem, IHomeDirectoryProvider home, - SensitiveValueMasker sensitiveValueMasker) + SensitiveValueMasker sensitiveValueMasker, + bool useBashWorkspace) { this.fileSystem = fileSystem; this.home = home; this.sensitiveValueMasker = sensitiveValueMasker; + this.useBashWorkspace = useBashWorkspace; + } + + public ScriptWorkspaceFactory( + IOctopusFileSystem fileSystem, + IHomeDirectoryProvider home, + SensitiveValueMasker sensitiveValueMasker) : this(fileSystem, home, sensitiveValueMasker, useBashWorkspace: !PlatformDetection.IsRunningOnWindows) + { } public IScriptWorkspace GetWorkspace(ScriptTicket ticket, WorkspaceReadinessCheck readinessCheck) @@ -107,7 +117,7 @@ string FindWorkingDirectory(ScriptTicket ticket) IScriptWorkspace CreateWorkspace(ScriptTicket scriptTicket, string workingDirectory) { - if (!PlatformDetection.IsRunningOnWindows) + if (useBashWorkspace) { return new BashScriptWorkspace(scriptTicket, workingDirectory, fileSystem, sensitiveValueMasker); } diff --git a/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs b/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs index 6a455bb2d..2b5f83a59 100644 --- a/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs +++ b/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs @@ -28,6 +28,7 @@ public static class EnvironmentVariables public const string TentacleEnableDataStreamLengthChecks = "TentacleEnableDataStreamLengthChecks"; public const string TentacleMachineConfigurationHomeDirectory = "TentacleMachineConfigurationHomeDirectory"; public const string TentaclePollingConnectionCount = "TentaclePollingConnectionCount"; + public const string TentaclePowerShellStartupTimeout = "TentaclePowerShellStartupTimeout"; public const string NfsWatchdogDirectory = "watchdog_directory"; public static string TentacleUseTcpNoDelay = "TentacleUseTcpNoDelay"; public static string TentacleUseAsyncListener = "TentacleUseAsyncListener"; diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs new file mode 100644 index 000000000..d06b6ac4f --- /dev/null +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -0,0 +1,440 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NSubstitute; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Configuration; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ScriptServiceV2; +using Octopus.Tentacle.Core.Services.Scripts; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; +using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; +using Octopus.Tentacle.Core.Services.Scripts.Shell; +using Octopus.Tentacle.Core.Services.Scripts.StateStore; +using Octopus.Tentacle.Scripts; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class PowerShellStartupDetectionTests : IntegrationTest + { + static (ScriptServiceV2 service, ScriptWorkspaceFactory workspaceFactory, ScriptStateStoreFactory stateStoreFactory, TemporaryDirectory tempDir) CreateScriptService( + TimeSpan? powerShellStartupTimeout = null) + { + var tempDir = new TemporaryDirectory(); + + var systemLog = new SerilogSystemLog(new SerilogLoggerBuilder().Build()); + + var homeConfiguration = Substitute.For(); + homeConfiguration.HomeDirectory.Returns(tempDir.DirectoryPath); + + var octopusPhysicalFileSystem = new OctopusPhysicalFileSystem(systemLog); + var workspaceFactory = new ScriptWorkspaceFactory(octopusPhysicalFileSystem, homeConfiguration, new SensitiveValueMasker(), + useBashWorkspace: false // Force the powershell workspace to be used + ); + var stateStoreFactory = new ScriptStateStoreFactory(octopusPhysicalFileSystem); + + var shell = GetShellForCurrentPlatform(); + + var service = new ScriptServiceV2( + shell, + workspaceFactory, + stateStoreFactory, + new ScriptIsolationMutex(), + systemLog, + new Dictionary(), + powerShellStartupTimeout ?? PowerShellStartupDetection.PowerShellStartupTimeout); + + + return (service, workspaceFactory, stateStoreFactory, tempDir); + } + + [Test] + public async Task WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_ScriptSucceeds() + { + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(60)); + using (tempDir) + { + var scriptBody = @" +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'Hello from PowerShell' +write-output 'Script completed successfully' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("Hello from PowerShell"); + allLogs.Should().Contain("Script completed successfully"); + } + } + + + [Test] + public async Task WhenPowerShellScriptHasDetectionComment_AndPowershellScriptRunsLongerThanThePowerShellStartupTimeout_ScriptSucceeds() + { + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(10)); + using (tempDir) + { + var scriptBody = @" +# TENTACLE-POWERSHELL-STARTUP-DETECTION +Start-Sleep -Seconds 20 +write-output 'Hello from PowerShell' +write-output 'Script completed successfully' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("Hello from PowerShell"); + allLogs.Should().Contain("Script completed successfully"); + } + } + + [Test] + public async Task WhenPowerShellNeverStarts_DetectionReportsFailure() + { + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + using (tempDir) + { + // Simulate PowerShell hanging before the detection code by sleeping for a long time + // This tests the scenario where PowerShell.exe starts but hangs before executing our script + var scriptBody = $@" +# Sleep for a long time to simulate PowerShell hanging before reaching detection code +Start-Sleep -Seconds 3600 +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("process did not start within"); + } + } + + [Test] + public async Task WhenPowerShellNeverStarts_WeShouldDetectTheScriptDidNotStart_AndAttemptToCancelTheScript() + { + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + using (tempDir) + { + var stillRunning = Path.Combine(tempDir.DirectoryPath, "stillRunning"); + // Simulate PowerShell hanging before the detection code by sleeping for a long time + // This tests the scenario where PowerShell.exe starts but hangs before executing our script + var scriptBody = $@" +while ($true) {{ + Add-Content -Path '{stillRunning}' -Value 'This is the appended text.' + Start-Sleep -Seconds 1 +}} +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + + await DeletePotentiallyInUseFile(stillRunning); + await Task.Delay(TimeSpan.FromSeconds(5)); + File.Exists(stillRunning).Should().BeFalse("Otherwise the script is still running and we made not effort to cancel it."); + } + } + + async Task DeletePotentiallyInUseFile(string file) + { + while (true) + { + this.CancellationToken.ThrowIfCancellationRequested(); + try + { + File.Delete(file); + break; + } + catch (Exception) { } + Logger.Information("Will re-attempt to delete {file} in 100ms", file); + await Task.Delay(TimeSpan.FromMilliseconds(100), this.CancellationToken); + } + } + + [Test] + public async Task WhenPowerShellScriptWithoutDetectionComment_NormalExecutionOccurs() + { + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + using (tempDir) + { + // If we have a long-running script that does not have the detection comment, + // then tentacle should not bother with any detection logic. This includes not terminating the script + // because it never reported as running. + var scriptBody = @" +Start-Sleep -Seconds 10 +write-output 'Hello from PowerShell without detection' +write-output 'Script completed successfully'"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + + allLogs.Should().NotContain("PowerShell startup detection"); + // PowerShell output might not be captured in all test environments + // The important thing is that it completes successfully + } + } + + + [Test] + public async Task WhenPowerShellNeverStarts_WeShouldDetectTheScriptDidNotStart_AndTheScriptShouldNotBeAbleToStartAgain() + { + var (service, workspaceFactory, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + using (tempDir) + { + var shouldSleep = Path.Combine(tempDir.DirectoryPath, "shouldSleep"); + File.WriteAllText(shouldSleep, ""); + + var scriptBody = $@" +while (Test-Path -Path '{shouldSleep}') {{ + Start-Sleep -Seconds 1 +}} +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (_, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + + // At this point the monitor has: + // - Created the "started" file (prevents PowerShell from creating it) + // - Deleted the "should-run" file (prevents PowerShell from running even if workspace is cleaned up) + + // Delete shouldSleep so the script can proceed past the loop when re-invoked directly + File.Delete(shouldSleep); + + // Re-invoke the bootstrap script directly - the detection code should block it from running + var workspace = workspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); + Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(workspace.BootstrapScriptFilePath)); + var shell = GetShellForCurrentPlatform(); + var args = shell.FormatCommandArguments(workspace.BootstrapScriptFilePath, null, allowInteractive: false); + + var directOutput = new List(); + var directExitCode = SilentProcessRunner.ExecuteCommand( + shell.GetFullPath(), + args, + workspace.WorkingDirectory, + _ => { }, + line => directOutput.Add(line), + line => directOutput.Add(line), + CancellationToken.None); + + var directOutputText = string.Join("\n", directOutput); + Logger.Information("Direct invocation output:\n{Output}", directOutputText); + + directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); + + // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running"); + } + } + } + + [Test] + public async Task WhenPowerShellNeverStarts_AndWorkspaceIsDeletedBeforeScriptRuns_TheScriptShouldStillNotBeAbleToStart() + { + var (service, workspaceFactory, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + using (tempDir) + { + var shouldSleep = Path.Combine(tempDir.DirectoryPath, "shouldSleep"); + File.WriteAllText(shouldSleep, ""); + + var scriptBody = $@" +while (Test-Path -Path '{shouldSleep}') {{ + Start-Sleep -Seconds 1 +}} +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (_, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + + // Delete shouldSleep so the script can proceed past the loop when re-invoked directly + File.Delete(shouldSleep); + + // Simulate the workspace being cleaned up while the script is still in memory: + // delete every file in the workspace except the bootstrap script itself + var workspace = workspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); + var bootstrapScriptFilePath = workspace.BootstrapScriptFilePath; + foreach (var file in Directory.GetFiles(workspace.WorkingDirectory)) + { + if (!string.Equals(file, bootstrapScriptFilePath, StringComparison.OrdinalIgnoreCase)) + File.Delete(file); + } + + Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(bootstrapScriptFilePath)); + + // Re-invoke the bootstrap script directly - even without the workspace files it should be blocked + var shell = GetShellForCurrentPlatform(); + var args = shell.FormatCommandArguments(bootstrapScriptFilePath, null, allowInteractive: false); + + var directOutput = new List(); + var directExitCode = SilentProcessRunner.ExecuteCommand( + shell.GetFullPath(), + args, + workspace.WorkingDirectory, + _ => { }, + line => directOutput.Add(line), + line => directOutput.Add(line), + CancellationToken.None); + + var directOutputText = string.Join("\n", directOutput); + Logger.Information("Direct invocation output:\n{Output}", directOutputText); + + directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); + + // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running even when the workspace files are gone"); + } + } + } + + static IShell GetShellForCurrentPlatform() + { + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + return new PowerShell(); + } + + // On Linux/Mac, try to use pwsh (PowerShell Core) + // First check if pwsh is available + try + { + var result = SilentProcessRunner.ExecuteCommand( + "which", + "pwsh", + Environment.CurrentDirectory, + _ => { }, + _ => { }, + _ => { }, + new Dictionary(), + CancellationToken.None); + + if (result == 0) + { + // pwsh is available, create a custom shell for it + return new TestPwshShell(); + } + } + catch + { + // pwsh not available + } + + // Fall back to bash (tests will be skipped for PowerShell-specific features) + Assert.Inconclusive("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); + return null!; + } + + static async Task<(List, ScriptStatusResponseV2)> RunUntilScriptCompletes(ScriptServiceV2 service, StartScriptCommandV2 startScriptCommand, ScriptStatusResponseV2 response) + { + var (logs, lastResponse) = await RunUntilScriptFinishes(service, startScriptCommand, response); + WriteLogsToConsole(logs); + return (logs, lastResponse); + } + + static async Task<(List logs, ScriptStatusResponseV2 response)> RunUntilScriptFinishes(ScriptServiceV2 service, StartScriptCommandV2 startScriptCommand, ScriptStatusResponseV2 response) + { + var logs = new List(response.Logs); + + while (response.State != ProcessState.Complete) + { + response = await service.GetStatusAsync(new ScriptStatusRequestV2(startScriptCommand.ScriptTicket, response.NextLogSequence), CancellationToken.None); + logs.AddRange(response.Logs); + + if (response.State != ProcessState.Complete) + { + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } + } + + return (logs, response); + } + + static void WriteLogsToConsole(List logs) + { + foreach (var log in logs) + { + TestContext.Out.WriteLine("{0:yyyy-MM-dd HH:mm:ss K}: {1}", log.Occurred.ToLocalTime(), log.Text); + } + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs index 6203db380..c0e3b05ad 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Threading; +using System.Threading.Tasks; using FluentAssertions; using NSubstitute; using NUnit.Framework; @@ -11,6 +12,7 @@ using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Core.Services.Scripts; using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Diagnostics; @@ -72,6 +74,7 @@ public void SetUpLocal() scriptIsolationMutex, cancellationTokenSource.Token, new Dictionary(), + PowerShellStartupDetection.PowerShellStartupTimeout, log); } @@ -84,19 +87,19 @@ public void TearDownLocal() [Test] [Retry(3)] - public void ExitCode_ShouldBeReturned() + public async Task ExitCode_ShouldBeReturned() { workspace.BootstrapScript("exit 99"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(99, "the exit code of the script should be returned"); } [Test] [Retry(3)] - public void WriteHost_WritesToStdOut_AndIsReturned() + public async Task WriteHost_WritesToStdOut_AndIsReturned() { workspace.BootstrapScript("echo Hello"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdErr.Length.Should().Be(0, "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf("Hello", "the message should have been written to stdout"); @@ -105,10 +108,10 @@ public void WriteHost_WritesToStdOut_AndIsReturned() [Test] [Retry(3)] [WindowsTest] - public void WriteDebug_DoesNotWriteAnywhere() + public async Task WriteDebug_DoesNotWriteAnywhere() { workspace.BootstrapScript("Write-Debug Hello"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdOut.ToString().Should().NotContain("Hello", "the script shouldn't have written to stdout"); scriptLog.StdErr.ToString().Should().NotContain("Hello", "the script shouldn't have written to stderr"); @@ -117,10 +120,10 @@ public void WriteDebug_DoesNotWriteAnywhere() [Test] [Retry(3)] [WindowsTest] - public void WriteOutput_WritesToStdOut_AndIsReturned() + public async Task WriteOutput_WritesToStdOut_AndIsReturned() { workspace.BootstrapScript("Write-Output Hello"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdErr.ToString().Should().NotContain("Hello", "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf("Hello", "the message should have been written to stdout"); @@ -128,11 +131,11 @@ public void WriteOutput_WritesToStdOut_AndIsReturned() [Test] [Retry(3)] - public void WriteError_WritesToStdErr_AndIsReturned() + public async Task WriteError_WritesToStdErr_AndIsReturned() { workspace.BootstrapScript(PlatformDetection.IsRunningOnWindows ? "Write-Error EpicFail" : "&2 echo EpicFail"); - runningScript.Execute(); + await runningScript.Execute(); if (PlatformDetection.IsRunningOnWindows) runningScript.ExitCode.Should().Be(1, "Write-Error causes the exit code to be 1"); else @@ -144,13 +147,13 @@ public void WriteError_WritesToStdErr_AndIsReturned() [Test] [Retry(3)] - public void RunAsCurrentUser_ShouldWork() + public async Task RunAsCurrentUser_ShouldWork() { var scriptBody = PlatformDetection.IsRunningOnWindows ? $"echo {EchoEnvironmentVariable("username")}" : "whoami"; workspace.BootstrapScript(scriptBody); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdErr.Length.Should().Be(0, "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf(TestEnvironmentHelper.EnvironmentUserName); @@ -158,7 +161,7 @@ public void RunAsCurrentUser_ShouldWork() [Test] [Retry(5)] - public void CancellationToken_ShouldKillTheProcess() + public async Task CancellationToken_ShouldKillTheProcess() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) { @@ -173,10 +176,11 @@ public void CancellationToken_ShouldKillTheProcess() scriptIsolationMutex, cts.Token, new Dictionary(), + PowerShellStartupDetection.PowerShellStartupTimeout, new InMemoryLog()); workspace.BootstrapScript($"echo Starting\n{sleepCommand} 30\necho Finito"); - script.Execute(); + await script.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have been canceled"); scriptLog.StdErr.ToString().Should().Be("", "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf("Starting", "the starting message should be written to stdout"); diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs b/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs new file mode 100644 index 000000000..40088bf8d --- /dev/null +++ b/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs @@ -0,0 +1,34 @@ +using System; +using Octopus.Tentacle.Core.Services.Scripts.Shell; + +namespace Octopus.Tentacle.Tests.Integration.Util +{ + public class TestPwshShell : IShell + { + public string Name => "pwsh"; + + public string GetFullPath() + { + return "pwsh"; + } + + public string FormatCommandArguments(string bootstrapFile, string[]? scriptArguments, bool allowInteractive) + { + var args = new System.Text.StringBuilder(); + + if (!allowInteractive) + args.Append("-NonInteractive "); + + args.Append("-NoProfile "); + args.Append("-NoLogo "); + args.Append("-ExecutionPolicy Unrestricted "); + + var escapedBootstrapFile = bootstrapFile.Replace("'", "''"); + args.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", + escapedBootstrapFile, + string.Join(" ", scriptArguments ?? new string[0])); + + return args.ToString(); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs index e8c15a636..e1b952202 100644 --- a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs +++ b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs @@ -9,6 +9,7 @@ using Octopus.Tentacle.Core.Services; using Octopus.Tentacle.Core.Services.Scripts; using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Maintenance; using Octopus.Tentacle.Scripts; @@ -92,9 +93,8 @@ public async Task CompleteScriptAsync(CompleteScriptComman RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, CancellationTokenSource cancel) { - var runningScript = new RunningScript(shell, workspace, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancel.Token, new Dictionary(), log); - var thread = new Thread(runningScript.Execute) { Name = "Executing PowerShell script for " + ticket.TaskId }; - thread.Start(); + var runningScript = new RunningScript(shell, workspace, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancel.Token, new Dictionary(), PowerShellStartupDetection.PowerShellStartupTimeout, log); + _ = Task.Run(async () => await runningScript.Execute()); return runningScript; }