Skip to content

Commit b096b43

Browse files
TingluoHuangTingluoHuang
authored
Merge pull request #322 from Microsoft/users/tihuang/more
Linux cancellation
2 parents a067fbe + c6941a1 commit b096b43

File tree

8 files changed

+105
-43
lines changed

8 files changed

+105
-43
lines changed

‎src/Agent.Listener/JobDispatcher.cs

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.IO;
77
using System.Threading;
88
using System.Threading.Tasks;
9+
using Microsoft.VisualStudio.Services.WebApi;
910

1011
namespace Microsoft.VisualStudio.Services.Agent.Listener
1112
{
@@ -478,6 +479,11 @@ private async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockTok
478479
return;
479480
}
480481
}
482+
else
483+
{
484+
Trace.Error("Catch exception during renew agent jobrequest.");
485+
Trace.Error(ex);
486+
}
481487

482488
// retry
483489
TimeSpan remainingTime = TimeSpan.Zero;
@@ -514,20 +520,33 @@ private async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockTok
514520
private async Task CompleteJobRequestAsync(int poolId, JobRequestMessage message, Guid lockToken, TaskResult result)
515521
{
516522
var agentServer = HostContext.GetService<IAgentServer>();
517-
try
523+
bool retrying = false;
524+
while (true)
518525
{
519-
using (var csFinishRequest = new CancellationTokenSource(ChannelTimeout))
526+
try
520527
{
521-
await agentServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, csFinishRequest.Token);
528+
using (var csFinishRequest = new CancellationTokenSource(ChannelTimeout))
529+
{
530+
await agentServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, csFinishRequest.Token);
531+
}
532+
break;
533+
}
534+
catch (TaskAgentJobNotFoundException)
535+
{
536+
Trace.Info("TaskAgentJobNotFoundException received, job is no longer valid.");
537+
break;
538+
}
539+
catch (TaskAgentJobTokenExpiredException)
540+
{
541+
Trace.Info("TaskAgentJobTokenExpiredException received, job is no longer valid.");
542+
break;
543+
}
544+
catch (VssServiceResponseException ex) when (!retrying && ex.InnerException != null && ex.InnerException is ArgumentNullException)
545+
{
546+
Trace.Error("Retry on ArgumentNullException due a dotnet core bug in Linux/Mac.");
547+
Trace.Error(ex);
548+
retrying = true;
522549
}
523-
}
524-
catch (TaskAgentJobNotFoundException)
525-
{
526-
Trace.Info("TaskAgentJobNotFoundException received, job is no longer valid.");
527-
}
528-
catch (TaskAgentJobTokenExpiredException)
529-
{
530-
Trace.Info("TaskAgentJobTokenExpiredException received, job is no longer valid.");
531550
}
532551

533552
// This should be the last thing to run so we don't notify external parties until actually finished

‎src/Agent.Listener/_project.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
"runtimes": {
5757
"win7-x64": { },
5858
"ubuntu.14.04-x64": { },
59+
"ubuntu.16.04-x64": { },
5960
"centos.7-x64": { },
6061
"rhel.7.2-x64": { },
6162
"osx.10.11-x64": { }

‎src/Agent.Worker/_project.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
"runtimes": {
5656
"win7-x64": { },
5757
"ubuntu.14.04-x64": { },
58+
"ubuntu.16.04-x64": { },
5859
"centos.7-x64": { },
5960
"rhel.7.2-x64": { },
6061
"osx.10.11-x64": { }

‎src/Microsoft.VisualStudio.Services.Agent/ProcessInvoker.cs

Lines changed: 69 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ public sealed class ProcessInvoker : AgentService, IProcessInvoker
5151
private readonly TaskCompletionSource<bool> _processExitedCompletionSource = new TaskCompletionSource<bool>();
5252
private readonly ConcurrentQueue<string> _errorData = new ConcurrentQueue<string>();
5353
private readonly ConcurrentQueue<string> _outputData = new ConcurrentQueue<string>();
54+
private readonly TimeSpan _sigintTimeout = TimeSpan.FromSeconds(10);
55+
private readonly TimeSpan _sigtermTimeout = TimeSpan.FromSeconds(5);
5456

5557
public event EventHandler<ProcessDataReceivedEventArgs> OutputDataReceived;
5658
public event EventHandler<ProcessDataReceivedEventArgs> ErrorDataReceived;
@@ -234,11 +236,40 @@ private void ProcessOutput()
234236
private async Task CancelAndKillProcessTree()
235237
{
236238
ArgUtil.NotNull(_proc, nameof(_proc));
239+
bool sigint_succeed = await SendSIGINT(_sigintTimeout);
240+
if (sigint_succeed)
241+
{
242+
Trace.Info("Process cancelled successfully through Ctrl+C/SIGINT.");
243+
return;
244+
}
245+
246+
bool sigterm_succeed = await SendSIGTERM(_sigtermTimeout);
247+
if (sigterm_succeed)
248+
{
249+
Trace.Info("Process terminate successfully through Ctrl+Break/SIGTERM.");
250+
return;
251+
}
252+
253+
Trace.Info("Kill entire process tree since both cancel and terminate signal has been ignored by the target process.");
254+
KillProcessTree();
255+
}
256+
257+
private async Task<bool> SendSIGINT(TimeSpan timeout)
258+
{
237259
#if OS_WINDOWS
238-
await WindowsCancelAndKillProcessTree();
260+
return await SendCtrlSignal(ConsoleCtrlEvent.CTRL_C, timeout);
239261
#else
240-
await NixCancelAndKillProcessTree();
241-
#endif
262+
return await SendSignal(Signals.SIGINT, timeout);
263+
#endif
264+
}
265+
266+
private async Task<bool> SendSIGTERM(TimeSpan timeout)
267+
{
268+
#if OS_WINDOWS
269+
return await SendCtrlSignal(ConsoleCtrlEvent.CTRL_BREAK, timeout);
270+
#else
271+
return await SendSignal(Signals.SIGTERM, timeout);
272+
#endif
242273
}
243274

244275
private void ProcessExitedHandler(object sender, EventArgs e)
@@ -293,26 +324,6 @@ private void KillProcessTree()
293324
}
294325

295326
#if OS_WINDOWS
296-
private async Task WindowsCancelAndKillProcessTree()
297-
{
298-
bool ctrl_c_succeed = await SendCtrlSignal(ConsoleCtrlEvent.CTRL_C, TimeSpan.FromSeconds(10));
299-
if (ctrl_c_succeed)
300-
{
301-
Trace.Info("Process cancelled successfully through Ctrl+C.");
302-
return;
303-
}
304-
305-
bool ctrl_break_succeed = await SendCtrlSignal(ConsoleCtrlEvent.CTRL_BREAK, TimeSpan.FromSeconds(5));
306-
if (ctrl_break_succeed)
307-
{
308-
Trace.Info("Process terminate successfully through Ctrl+Break.");
309-
return;
310-
}
311-
312-
Trace.Info("Kill entire process tree since both cancel and terminate signal has been ignored by the target process.");
313-
KillProcessTree();
314-
}
315-
316327
private async Task<bool> SendCtrlSignal(ConsoleCtrlEvent signal, TimeSpan timeout)
317328
{
318329
Trace.Info($"Sending {signal} to process {_proc.Id}.");
@@ -545,31 +556,57 @@ public uint Size
545556
// Delegate type to be used as the Handler Routine for SetConsoleCtrlHandler
546557
private delegate Boolean ConsoleCtrlDelegate(ConsoleCtrlEvent CtrlType);
547558
#else
548-
private async Task NixCancelAndKillProcessTree()
559+
private async Task<bool> SendSignal(Signals signal, TimeSpan timeout)
549560
{
550-
// TODO: replace Task.Delay(1) with Send SIGINT/SIGTERM/SIGKILL
551-
await Task.Delay(1);
552-
KillProcessTree();
561+
Trace.Info($"Sending {signal} to process {_proc.Id}.");
562+
int errorCode = kill(_proc.Id, (int)signal);
563+
if (errorCode != 0)
564+
{
565+
Trace.Info($"{signal} signal doesn't fire successfully.");
566+
Trace.Error($"Error code: {errorCode}.");
567+
return false;
568+
}
569+
570+
Trace.Info($"Successfully send {signal} to process {_proc.Id}.");
571+
Trace.Info($"Waiting for process exit or {timeout.TotalSeconds} seconds after {signal} signal fired.");
572+
var completedTask = await Task.WhenAny(Task.Delay(timeout), _processExitedCompletionSource.Task);
573+
if (completedTask == _processExitedCompletionSource.Task)
574+
{
575+
Trace.Info("Process exit successfully.");
576+
return true;
577+
}
578+
else
579+
{
580+
Trace.Info($"Process did not honor {signal} signal within {timeout.TotalSeconds} seconds.");
581+
return false;
582+
}
553583
}
554584

555585
private void NixKillProcessTree()
556586
{
557587
try
558588
{
559-
// TODO: Send Ctrl+C/Break to process group.
560589
if (!_proc.HasExited)
561590
{
562591
_proc.Kill();
563592
}
564593
}
565-
catch (InvalidOperationException)
594+
catch (InvalidOperationException ex)
566595
{
567-
// InvalidOperationException can occur if process got terminated by itself between
568-
// HasExited and Kill() calls above.
596+
Trace.Error("Ignore InvalidOperationException during Process.Kill().");
597+
Trace.Error(ex);
569598
}
570599
}
571-
#endif
572600

601+
private enum Signals : int
602+
{
603+
SIGINT = 2,
604+
SIGTERM = 15
605+
}
606+
607+
[DllImport("libc", SetLastError = true)]
608+
private static extern int kill(int pid, int sig);
609+
#endif
573610
}
574611

575612
public sealed class ProcessExitCodeException : Exception

‎src/Microsoft.VisualStudio.Services.Agent/_project.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
"runtimes": {
5555
"win7-x64": { },
5656
"ubuntu.14.04-x64": { },
57+
"ubuntu.16.04-x64": { },
5758
"centos.7-x64": { },
5859
"rhel.7.2-x64": { },
5960
"osx.10.11-x64": { }

‎src/Test/L0/ConstantGenerationL0.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public void BuildConstantGenerateSucceed()
1515
{
1616
"win7-x64",
1717
"ubuntu.14.04-x64",
18+
"ubuntu.16.04-x64",
1819
"centos.7-x64",
1920
"rhel.7.2-x64",
2021
"osx.10.11-x64"

‎src/Test/L0/Util/ApiUtilL0.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ public void VerifyUserAgentHasPlatformInfo()
8585
{
8686
"win7-x64",
8787
"ubuntu.14.04-x64",
88+
"ubuntu.16.04-x64",
8889
"centos.7-x64",
8990
"rhel.7.2-x64",
9091
"osx.10.11-x64"

‎src/Test/_project.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
"runtimes": {
5252
"win7-x64": { },
5353
"ubuntu.14.04-x64": { },
54+
"ubuntu.16.04-x64": { },
5455
"centos.7-x64": { },
5556
"rhel.7.2-x64": { },
5657
"osx.10.11-x64": { }

0 commit comments

Comments
 (0)