Improves test runner process handling (#334)

Author: jmgomez

testrunner.nim

@@ -36,32 +36,6 @@ proc getFullPath*(entryPoint: string, workspaceRoot: string): string =
       return absolutePath
   return entryPoint
 
-proc listTests*(
-  entryPoint: string, 
-  nimPath: string,
-  workspaceRoot: string
-): Future[TestProjectInfo] {.async.} =
-  var entryPoint = getFullPath(entryPoint, workspaceRoot)
-  let process = await startProcess(
-    nimPath,
-    arguments = @["c", "-d:unittest2ListTests", "-r", "--listFullPaths", entryPoint],
-    options = {UsePath},
-    stderrHandle = AsyncProcess.Pipe,
-    stdoutHandle = AsyncProcess.Pipe,
-  )
-  try:
-    let res = await process.waitForExit(15.seconds)
-    if res != 0:
-      error "Failed to list tests", nimPath = nimPath, entryPoint = entryPoint, res = res    
-      let error = string.fromBytes(process.stderrStream.read().await)
-      error "An error occurred while listing tests", error = error
-      result = TestProjectInfo(error: some error)
-    else:
-      let rawOutput = string.fromBytes(process.stdoutStream.read().await)   
-      result = extractTestInfo(rawOutput)
-  finally:
-    await shutdownChildProcess(process)
-
 proc parseObject(obj: var object, node: XmlNode) = 
   for field, value in obj.fieldPairs:
     when value is string:
@@ -91,6 +65,37 @@ proc parseTestResults*(xmlContent: string): RunTestProjectResult =
     if suite.testResults.len > 0:
       result.suites.add(suite)
 
+proc listTests*(
+  entryPoint: string, 
+  nimPath: string,
+  workspaceRoot: string
+): Future[TestProjectInfo] {.async.} =
+  var entryPoint = getFullPath(entryPoint, workspaceRoot)
+  debug "Listing tests", entryPoint = entryPoint, exists = fileExists(entryPoint)
+  let args = @["c", "-d:unittest2ListTests", "-r", entryPoint]
+  let process = await startProcess(
+    nimPath,
+    arguments = args,
+    options = {UsePath},
+    stderrHandle = AsyncProcess.Pipe,
+    stdoutHandle = AsyncProcess.Pipe,
+  )
+  try:
+    let (error, res) = await readErrorOutputUntilExit(process, 15.seconds)
+    if res != 0:
+      error "Failed to list tests", nimPath = nimPath, entryPoint = entryPoint, res = res    
+      error "An error occurred while listing tests"
+      for line in error.splitLines:
+        error "Error line: ", line = line
+      error "Command args: ", args = args
+      result = TestProjectInfo(error: some error)
+    else:
+      let rawOutput = await readAllOutput(process.stdoutStream)   
+      debug "list test raw output", rawOutput = rawOutput
+      result = extractTestInfo(rawOutput)
+  finally:
+    await shutdownChildProcess(process)
+
 proc runTests*(
   entryPoint: string, 
   nimPath: string, 
@@ -120,19 +125,23 @@ proc runTests*(
   )
   ls.testRunProcess = some(process)
   try:
-    let res = await process.waitForExit(15.seconds)
-    let processOutput = string.fromBytes(process.stdoutStream.read().await)
+    removeFile(resultFile)
+    let (error, res) = await readErrorOutputUntilExit(process, 15.seconds)
+    if res != 0: #When a test fails, the process will exit with a non-zero code
+      if fileExists(resultFile):
+        result = parseTestResults(readFile(resultFile))
+        result.fullOutput = error
+        return result
 
-    if not fileExists(resultFile):
-      let processError = string.fromBytes(process.stderrStream.read().await)
-      error "Result file does not exist meaning tests were not run"
-      error "Output from process", output = processOutput
-      error "Error from process", error = processError
+      error "Failed to run tests", nimPath = nimPath, entryPoint = entryPoint, res = res    
+      error "An error occurred while running tests"
+      error "Error from process", error = error
+      result = RunTestProjectResult(fullOutput: error)
     else:
       let xmlContent = readFile(resultFile)
       # echo "XML CONTENT: ", xmlContent
       result = parseTestResults(xmlContent)
-      result.fullOutput = processOutput
+      result.fullOutput = error
       removeFile(resultFile)
   except Exception as e:
     let processOutput = string.fromBytes(process.stdoutStream.read().await)

utils.nim

@@ -3,6 +3,7 @@ import chronos, chronicles, chronos/asyncproc
 import "$nim/compiler/pathutils"
 import json_rpc/private/jrpc_sys
 import macros
+import stew/byteutils
 
 type
   FingerTable = seq[tuple[u16pos, offset: int]]
@@ -357,4 +358,31 @@ proc shutdownChildProcess*(p: AsyncProcessRef): Future[void]  {.async.} =
       writeStackTrace()
 
 macro getField*(obj: object, fld: string): untyped =
-  result = newDotExpr(obj, newIdentNode(fld.strVal))
+  result = newDotExpr(obj, newIdentNode(fld.strVal))
+
+
+proc readAllOutput*(stream: AsyncStreamReader): Future[string] {.async.} =
+  result = ""
+  while not stream.atEof:
+    let data = await stream.read()
+    result.add(string.fromBytes(data))
+
+proc readErrorOutputUntilExit*(process: AsyncProcessRef, duration: Duration): Future[tuple[output: string, code: int]] {.async.} =
+  var output = ""
+  var res = 0
+  while true:
+    if not process.stderrStream.atEof:
+      let data = await process.stderrStream.read()
+      output.add(string.fromBytes(data))
+    
+    let hasExited = try:
+      res = await process.waitForExit(duration)
+      true
+    except AsyncTimeoutError:
+      false
+    
+    if hasExited:
+      while not process.stderrStream.atEof:
+        let data = await process.stderrStream.read()
+        output.add(string.fromBytes(data))
+      return (output, res)