Testament hybrid runner

.github/workflows/test.yml

@@ -46,7 +46,7 @@ jobs:
           hg --version
       - name: Install dependencies
         run: nimble install -y
-      - name: Run nim c -r tester
+      - name: Run tests
         run: |
           nimble test
       - name: Install nimble

.gitignore

@@ -63,9 +63,11 @@ src/nimblepkg/version
 # Test procedure artifacts
 *.nims
 /buildTests
+/testresults/
 /tests/**/.gitignore
 /tests/**/nimble.lock
 /tests/nimdep/localnimbledeps/
+/tests/testresults/
 
 # Nimble Guide files
 nimble-guide/site/

nimble.nimble

@@ -22,18 +22,14 @@ when defined(nimdistros):
 before install:
   exec "git submodule update --init"
 
-proc runTester(extraFlags = "") =
-  var extraParams = ""
-  for i in 0 .. paramCount():
-    if "::" in paramStr(i):
-      extraParams = "test "
-      extraParams.addQuoted paramStr(i)
-
-  withDir "tests":
-    exec "nim c " & extraFlags & " -r tester " & extraParams
-
 task test, "Run the Nimble tester!":
-  runTester()
+  exec "testament --directory:tests --megatest:off --skipFrom:skip.txt pattern t*.nim"
+  exec "testament --directory:tests/tissues --megatest:off pattern tissue_*.nim"
+  exec "testament --directory:tests/sat --megatest:off pattern t*.nim"
+  exec "cd tests && nim c -r tester.nim"
 
 task cibenchmark, "Run tests with timing instrumentation":
-  runTester("-d:timedTests")
+  exec "testament --directory:tests --megatest:off --skipFrom:skip.txt pattern t*.nim -d:timedTests"
+  exec "testament --directory:tests/tissues --megatest:off pattern tissue_*.nim -d:timedTests"
+  exec "testament --directory:tests/sat --megatest:off pattern t*.nim -d:timedTests"
+  exec "cd tests && nim c -r -d:timedTests tester.nim"

tests/testscommon.nim → tests/common.nim

@@ -3,7 +3,7 @@
 
 {.used.}
 
-import sequtils, strutils, strformat, os, osproc, sugar, unittest, macros
+import sequtils, strutils, strformat, os, osproc, sugar, unittest, macros, std/times
 import pkg/checksums/sha1
 
 import nimblepkg/cli
@@ -21,7 +21,7 @@ const
   pkgMultiBetaUrl* = &"{pkgMultiUrl}?subdir=beta"
 
 let
-  rootDir = getCurrentDir().parentDir
+  rootDir = currentSourcePath().parentDir.parentDir
   nimblePath* = rootDir / "src" / addFileExt("nimble", ExeExt)
   nimbleCompilePath = rootDir / "src" / "nimble.nim"
   installDir* = rootDir / "tests" / "nimbleDir"
@@ -39,7 +39,7 @@ proc execNimble*(args: varargs[string]): ProcessOutput =
   quotedArgs.insert(nimblePath)
   quotedArgs = quotedArgs.map((x: string) => x.quoteShell)
 
-  let path {.used.} = getCurrentDir().parentDir() / "src"
+  let path {.used.} = rootDir / "src"
 
   var cmd =
     when not defined(windows):
@@ -214,11 +214,15 @@ putEnv("NIMBLE_TEST_BINARY_PATH", nimblePath)
 setVerbosity(MediumPriority)
 setShowColor(false)
 
-# Always recompile.
+# Compile nimble if binary is missing or source is newer.
 block:
-  # Verbose name is used for exit code so assert is clearer
-  let (output, nimbleCompileExitCode) = execCmdEx("nim c " & nimbleCompilePath)
-  doAssert nimbleCompileExitCode == QuitSuccess, output
+  let nimbleBin = rootDir / "src" / addFileExt("nimble", ExeExt)
+  let needsCompile = not nimbleBin.fileExists or
+                     nimbleBin.getLastModificationTime < nimbleCompilePath.getLastModificationTime
+  if needsCompile:
+    # Verbose name is used for exit code so assert is clearer
+    let (output, nimbleCompileExitCode) = execCmdEx("nim c " & nimbleCompilePath)
+    doAssert nimbleCompileExitCode == QuitSuccess, output
 
 # Test timing instrumentation — compile with -d:timedTests to enable
 when defined(timedTests):

tests/declarativeparserfixture/declarativeparserfixture.nimble

@@ -0,0 +1,14 @@
+version       = "0.1.0"
+author        = "test"
+description   = "Test fixture for declarative parser"
+license       = "MIT"
+srcDir        = "src"
+bin           = @["declarativeparserfixture"]
+
+requires "nim >= 2.0.0"
+requires "json_rpc"
+requires "with"
+requires "chronicles"
+requires "serialization"
+requires "stew"
+requires "regex"

tests/declarativeparserfixture/jesterfixture/jesterfixture.nimble

@@ -0,0 +1,9 @@
+version       = "0.1.0"
+author        = "test"
+description   = "Test fixture with nested requires"
+license       = "MIT"
+
+when defined(windows):
+  requires "winapi"
+else:
+  requires "posix"

tests/declarativeparserfixture/nim_1_6_12/compiler/compilation.nim

@@ -0,0 +1,4 @@
+const
+  NimMajor* = 1
+  NimMinor* = 6
+  NimPatch* = 12

tests/declarativeparserfixture/nim_1_6_12/nim.nimble

@@ -0,0 +1,8 @@
+version       = "1.6.12"
+author        = "Andreas Rumpf"
+description   = "Fixture mimicking nim nimble file"
+license       = "MIT"
+
+include "compiler/compilation.nim"
+
+requires "nim >= 1.6.0"

tests/declarativeparserfixture/nim_2_2_0/compiler/compilation.nim

@@ -0,0 +1,4 @@
+const
+  NimMajor* = 2
+  NimMinor* = 2
+  NimPatch* = 0

tests/declarativeparserfixture/nim_2_2_0/nim.nimble

@@ -0,0 +1,8 @@
+version       = "2.2.0"
+author        = "Andreas Rumpf"
+description   = "Fixture mimicking nim nimble file"
+license       = "MIT"
+
+include "compiler/compilation.nim"
+
+requires "nim >= 1.6.0"

tests/declarativeparserfixture/src/declarativeparserfixture.nim

@@ -0,0 +1 @@
+echo "fixture"