11package main
22
33import (
4+ "context"
45 "os"
56 "os/exec"
67 "runtime"
8+ "strings"
9+ "time"
710
811 log "github.com/newrelic/newrelic-diagnostics-cli/logger"
912)
1013
14+ const (
15+ dockerBuildTimeout = 10 * time .Minute
16+ dockerRunTimeout = 10 * time .Minute
17+ )
18+
19+ var dockerBuildRetryPatterns = []string {
20+ "hcs::CreateComputeSystem" ,
21+ "failed to connect to the docker API" ,
22+ "docker_engine" ,
23+ }
24+
1125func CreateDockerImage (imageName string , dockerFROM string , docker_cmd string , dockerLines []string ) error {
1226
1327 //Create the Dockerfile
@@ -19,18 +33,74 @@ func CreateDockerImage(imageName string, dockerFROM string, docker_cmd string, d
1933
2034 log .Debug ("Running docker build -f integrationDockerfile -t " , imageName , " ." )
2135
22- cmdBuild := exec .Command ("docker" , "build" , "-f" , dockerfile , "-t" , imageName , "." )
23-
24- output , cmdBuildErr := cmdBuild .CombinedOutput ()
36+ const maxAttempts = 3
37+ var output []byte
38+ var cmdBuildErr error
39+ var timedOut bool
40+ for attempt := 1 ; attempt <= maxAttempts ; attempt ++ {
41+ ctx , cancel := context .WithTimeout (context .Background (), dockerBuildTimeout )
42+ cmdBuild := exec .CommandContext (ctx , "docker" , "build" , "-f" , dockerfile , "-t" , imageName , "." )
43+ output , cmdBuildErr = cmdBuild .CombinedOutput ()
44+ timedOut = ctx .Err () == context .DeadlineExceeded
45+ cancel ()
46+
47+ if cmdBuildErr == nil {
48+ break
49+ }
50+ if timedOut {
51+ log .Info ("Docker build TIMED OUT for " , imageName , " after " , dockerBuildTimeout , " (attempt " , attempt , " of " , maxAttempts , ")" )
52+ log .Info ("Partial build output for " , imageName , ":\n " , string (output ))
53+ logDockerDiagnostics (imageName )
54+ }
55+ if attempt == maxAttempts || ! isTransientDockerError (string (output )) {
56+ break
57+ }
58+ log .Info ("Transient docker build error for " , imageName , " (attempt " , attempt , " of " , maxAttempts , "), retrying in " , attempt * 5 , "s" )
59+ time .Sleep (time .Duration (attempt * 5 ) * time .Second )
60+ }
2561
2662 if cmdBuildErr != nil {
2763 log .Info ("Error running docker build -" , cmdBuildErr )
2864 log .Info ("Error was " , string (output ))
2965 return cmdBuildErr
3066 }
67+ log .Info ("Docker build output for " , imageName , ":\n " , string (output ))
3168 return nil
3269}
3370
71+ func isTransientDockerError (output string ) bool {
72+ for _ , pattern := range dockerBuildRetryPatterns {
73+ if strings .Contains (output , pattern ) {
74+ return true
75+ }
76+ }
77+ return false
78+ }
79+
80+ func logDockerDiagnostics (imageName string ) {
81+ log .Info ("=== Docker diagnostics for " , imageName , " ===" )
82+
83+ psCtx , psCancel := context .WithTimeout (context .Background (), 30 * time .Second )
84+ psOut , psErr := exec .CommandContext (psCtx , "docker" , "ps" , "-a" ).CombinedOutput ()
85+ psCancel ()
86+ if psErr != nil {
87+ log .Info ("docker ps -a failed: " , psErr , " output: " , string (psOut ))
88+ } else {
89+ log .Info ("docker ps -a:\n " , string (psOut ))
90+ }
91+
92+ infoCtx , infoCancel := context .WithTimeout (context .Background (), 30 * time .Second )
93+ infoOut , infoErr := exec .CommandContext (infoCtx , "docker" , "info" ).CombinedOutput ()
94+ infoCancel ()
95+ if infoErr != nil {
96+ log .Info ("docker info failed: " , infoErr , " output: " , string (infoOut ))
97+ } else {
98+ log .Info ("docker info:\n " , string (infoOut ))
99+ }
100+
101+ log .Info ("=== End docker diagnostics for " , imageName , " ===" )
102+ }
103+
34104// CreateDockerfile - This builds the raw Dockerfile from the slice of tests
35105func CreateDockerfile (imageName string , dockerFROM string , dockerCMD string , dockerfileLines []string ) (string , error ) {
36106 f , _ := os .CreateTemp ("temp" , imageName )
@@ -47,7 +117,7 @@ func CreateDockerfile(imageName string, dockerFROM string, dockerCMD string, doc
47117 }
48118
49119 baseWindowsDockerFrom := []string {
50- "FROM mcr.microsoft.com/windows/servercore:ltsc2022 " ,
120+ "FROM mcr.microsoft.com/windows/servercore:ltsc2025 " ,
51121 `SHELL ["powershell"]` ,
52122 "RUN NET USER nrdiagadmin /add" ,
53123 "RUN NET LOCALGROUP administrators /add nrdiagadmin" ,
@@ -120,10 +190,18 @@ func RunDockerContainer(imageName string, hostsAdditions []string) (string, erro
120190 }
121191 }
122192 args = append (args , imageName )
123- cmd := exec .Command ("docker" , args ... )
193+
194+ ctx , cancel := context .WithTimeout (context .Background (), dockerRunTimeout )
195+ defer cancel ()
196+ cmd := exec .CommandContext (ctx , "docker" , args ... )
124197
125198 out , cmdErr := cmd .CombinedOutput ()
126199 if cmdErr != nil {
200+ if ctx .Err () == context .DeadlineExceeded {
201+ log .Info ("Docker run TIMED OUT for " , imageName , " after " , dockerRunTimeout )
202+ log .Info ("Partial run output for " , imageName , ":\n " , string (out ))
203+ logDockerDiagnostics (imageName )
204+ }
127205 // Docker daemon returns exit code 125 in jenkins but runs normally otherwise
128206 if cmdErr .Error () == "exit status 125" {
129207 return string (out [:]), nil
0 commit comments