diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1a43c48..5105c2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,7 +30,8 @@ jobs: fetch-depth: 0 - name: Get version id: tag - uses: anothrNick/github-tag-action@1.73.0 + #uses: anothrNick/github-tag-action@1.73.0 + uses: rorybartie/github-tag-action@bugfix/duplicate-prefix-on-prerelease env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG_PREFIX: v @@ -86,12 +87,14 @@ jobs: dotnet build --configuration Release --no-restore -p:Version=${VERSION:1} - name: Add NuGet source + if: ${{ !github.event.pull_request }} env: NUGET_USERNAME: ${{ vars.NUGET_USERNAME }} NUGET_PASSWORD: ${{ secrets.NUGET_PASSWORD }} run: dotnet nuget add source --username ${NUGET_USERNAME} --password ${NUGET_PASSWORD} --store-password-in-clear-text --name nuget "https://nuget.pkg.github.com/${NUGET_USERNAME}/index.json" - name: Publish package + if: ${{ !github.event.pull_request }} env: VERSION: ${{ needs.get-version.outputs.tag }} NUGET_APIKEY: ${{ secrets.NUGET_APIKEY }} diff --git a/Readme.md b/Readme.md index 50e486f..1027104 100644 --- a/Readme.md +++ b/Readme.md @@ -9,25 +9,32 @@ [![Build](https://github.com/dlidstrom/Sprout/actions/workflows/build.yml/badge.svg)](https://github.com/dlidstrom/Sprout/actions/workflows/build.yml) [![NuGet version](https://badge.fury.io/nu/dlidstrom.Sprout.svg)](https://badge.fury.io/nu/dlidstrom.Sprout) +๐Ÿ”ฅ Latest news + +- Refactoring for improved extensibility. + - Custom runner support has been added. + - Test ordering is now customizable. + - See [Extending Sprout](#-extending-sprout) for more details. + ## โœ… Features -* Minimalist & expressive BDD-style syntax -* Nestable `describe` blocks for organizing tests to any depth or complexity -* Computation expressions for `it`, `beforeEach`, `afterEach` -* Asynchronous blocks supported -* Pending tests support -* Logging for improved traceability -* Built-in assertions: `shouldEqual`, `shouldBeTrue`, `shouldBeFalse` - * These go along way to making your tests readable and expressive due to the +- Minimalist & expressive BDD-style syntax +- Nestable `describe` blocks for organizing tests to any depth or complexity +- Computation expressions for `it`, `beforeEach`, `afterEach` +- Asynchronous blocks supported +- Pending tests support +- Logging for improved traceability +- Built-in assertions: `shouldEqual`, `shouldBeTrue`, `shouldBeFalse` + - These go along way to making your tests readable and expressive due to the descriptive text that goes along with each `describe` block and `it` test case. Note that you may use any exception-based constraints library if desired ([FsUnit.Light](https://github.com/Lanayx/FsUnit.Light) works beautifully). -* Support for parameterized tests using normal F# loops within `describe` blocks -* Pluggable reporters (console, silent, TAP, JSON) -* Built for F# โ€” simple and idiomatic - * No need for `<|` or functions wrapped in parentheses! - * No need for complex combination of attributes - it's just all F# code! +- Support for parameterized tests using normal F# loops within `describe` blocks +- Pluggable reporters (console, silent, TAP, JSON) +- Built for F# โ€” simple and idiomatic + - No need for `<|` or functions wrapped in parentheses! + - No need for complex combination of attributes - it's just all F# code! Sprout provides a clean API for structuring tests. There are a minimum of abstractions. `describe` blocks may be nested to any suitable depth. It is all @@ -47,6 +54,8 @@ dotnet add package dlidstrom.Sprout ### ๐Ÿงช Example Test ```fsharp +#load "Sprout.fs" + open Sprout let suite = describe "A test suite" { @@ -116,8 +125,11 @@ let suite = describe "A test suite" { } } -// Run the test suite asynchronously -runTestSuite suite +runTestSuiteCustom + // id is used to order the tests (it blocks) + // you can specify a custom ordering function if needed + (DefaultRunner(Reporters.ConsoleReporter("v", "x", "?", " "), id)) + suite |> Async.RunSynchronously ``` @@ -135,16 +147,16 @@ Output: | `it` | Imperative | Any F# expressions, but typically exception-based assertions | | `beforeEach`, `afterEach` | Imperative | Hook functions that run before and after test cases, respectively | -* **`describe`** - used to define a test suite, or a group of tests. Use the +- **`describe`** - used to define a test suite, or a group of tests. Use the descriptive text to create descriptive sentences of expected behaviour. -* **`it`** - used to define test assertions, along with a descriptive text to +- **`it`** - used to define test assertions, along with a descriptive text to define the expected behaviour -* **`beforeEach`/`afterEach`** - hook functions that run before and after test +- **`beforeEach`/`afterEach`** - hook functions that run before and after test cases, respectively -* **`pending`** - used to denote a pending test case -* **`info`/`debug`** - tracing functions that may be used inside hook functions +- **`pending`** - used to denote a pending test case +- **`info`/`debug`** - tracing functions that may be used inside hook functions and `it` blocks -* **`Info`/`Debug`** - constructor functions that provide tracing inside +- **`Info`/`Debug`** - constructor functions that provide tracing inside `describe` blocks > Tracing works slightly different in `describe` blocks due to limitations of me @@ -158,24 +170,28 @@ You can plug in your own reporter: ```fsharp type MyCustomReporter() = - interface ITestReporter with - member _.Begin(totalCount) = ... - member _.BeginSuite(name, path) = ... - member _.ReportResult(result, path) = ... - member _.EndSuite(name, path) = ... - member _.Info(message, path) = ... - member _.Debug(message, path) = ... - member _.End(testResults) = ... + inherit TestReporter() + with + override _.Begin(totalCount) = ... + override _.BeginSuite(name, path) = ... + override _.ReportResult(result, path) = ... + override _.EndSuite(name, path) = ... + override _.Info(message, path) = ... + override _.Debug(message, path) = ... + override _.End(testResults) = ... ``` Use it like this: ```fsharp -let reporter = MyCustomReporter() :> ITestReporter +let reporter = MyCustomReporter() + let failedCount = - runTestSuiteWithContext + runTestSuiteCustom + // id is used to order the tests (it blocks) + // you can specify a custom ordering function if needed + (DefaultRunner(reporter, id)) suite - { TestContext.New with Reporter = reporter } |> Async.RunSynchronously ``` @@ -188,6 +204,30 @@ Available reporters (available in the `Sprout.Reporters` namespace): --- +It is also possible to modify the default or create your own runner. A simple +example is to modify the default runner to run tests in parallel: + +```fsharp +let parallelRunner = { + new DefaultRunner(Reporters.TapReporter(), id) with + override _.SequenceAsync args = Async.Parallel args } +runTestSuiteCustom + parallelRunner + suite +|> Async.RunSynchronously +``` + +--- + +### ๐Ÿ—๏ธ Testing Sprout Itself + +Sprout is tested using [Bash Automated Testing System](https://github.com/bats-core/bats-core). +You can run the tests using the following command: + +```bash +bats . +``` + ### ๐Ÿ“ฆ Package Info | | | diff --git a/Sprout.fs b/Sprout.fs index f7256d1..ebedd5b 100644 --- a/Sprout.fs +++ b/Sprout.fs @@ -6,6 +6,10 @@ let mutable debug: string -> unit = ignore type LogLevel = Debug of string | Info of string type HookFunction = unit -> Async +type HookFunctions = { + Before: HookFunction list + After: HookFunction list +} type EachFunction = | Before of f: HookFunction | After of f: HookFunction @@ -40,6 +44,18 @@ with (d.Steps |> List.sumBy (function ItStep _ -> 1 | _ -> 0)) + (d.Children |> List.sumBy count) count this + member this.HookFunctions = + let beforeHooks, afterHooks = + this.Each + |> List.fold (fun (be, af) hook -> + match hook with + | Before hookFunction -> hookFunction :: be, af + | After hookFunction -> be, hookFunction :: af + ) ([], []) + { + Before = List.rev beforeHooks + After = List.rev afterHooks + } static member New name = { Name = name @@ -58,12 +74,28 @@ with member this.Value = match this with | Path p -> p -type TestResult = +type TestOutcome = | Passed of Path * string | Failed of Path * string * exn | Pending of Path * string +type TestResult = { + Outcome: TestOutcome + Logs: LogLevel list +} + +type CollectedStep = + | CollectedIt of Path * HookFunctions * It + | CollectedLog of Path * LogLevel -type ITestReporter = +type CollectedDescribe = { + Name: string + Path: Path + Steps: CollectedStep list + Children: CollectedDescribe list +} + +[] +type TestReporter() = abstract Begin : totalCount:int -> unit abstract BeginSuite : name:string * path:Path -> unit abstract ReportResult : result:TestResult * path:Path -> unit @@ -143,22 +175,21 @@ module Reporters = let reset = "\u001b[0m" type ConsoleReporter(passedChar, failedChar, pendingChar, indentString) = - let sw = Stopwatch.StartNew() - let indent (path: Path) = String.replicate (path.Length - 1) indentString - - new() = - ConsoleReporter("โœ…", "โŒ", "โ”", " ") + inherit TestReporter() with + let sw = Stopwatch.StartNew() + let indent (path: Path) = String.replicate (path.Length - 1) indentString - interface ITestReporter with - member _.Begin totalCount = + new() = + ConsoleReporter("โœ…", "โŒ", "โ”", " ") + override _.Begin totalCount = sw.Restart() - member _.BeginSuite(name, path) = + override _.BeginSuite(name, path) = let indent = indent path - printfn $"%s{indent}{AnsiColours.green}{name}{AnsiColours.reset}" + printfn $"%s{indent}%s{AnsiColours.green}%s{name}%s{AnsiColours.reset}" - member _.ReportResult(result, path) = + override _.ReportResult(result, path) = let indent = indent path - match result with + match result.Outcome with | Passed (_, name) -> printfn $"%s{indent}%s{AnsiColours.green} %s{passedChar} passed: %s{name}%s{AnsiColours.reset}" | Failed (_, name, ex) -> @@ -166,49 +197,50 @@ module Reporters = | Pending (_, name) -> printfn $"%s{indent}%s{AnsiColours.grey} %s{pendingChar} pending: %s{name}%s{AnsiColours.reset}" - member _.EndSuite(_, _) = () - member _.Debug(message: string, path: Path): unit = + override _.EndSuite(_, _) = () + override _.Debug(message: string, path: Path): unit = let indent = indent path printfn $"%s{indent}%s{AnsiColours.grey}%s{message}%s{AnsiColours.reset}" - member _.Info(message: string, path: Path): unit = + override _.Info(message: string, path: Path): unit = let indent = indent path printfn $"%s{indent}%s{AnsiColours.white}%s{message}%s{AnsiColours.reset}" - member _.End(testResults: TestResult []): unit = - let testFailures = testResults |> Array.filter (function Failed _ -> true | _ -> false) + override _.End(testResults: TestResult []): unit = + let testFailures = testResults |> Array.filter _.Outcome.IsFailed if Array.isEmpty testFailures then printfn $"All tests passed!" else printfn $"There were %d{Array.length testFailures} test failures:" - testResults |> Array.iter (function + testResults |> Array.iter (fun tr -> + match tr.Outcome with | Failed (path, name, ex) -> let pathString = String.concat " / " path.Value printfn $"- %s{AnsiColours.red}%s{pathString} / %s{name} - %s{ex.Message}%s{AnsiColours.reset}" | _ -> ()) // Count results - let passedCount = testResults |> Array.filter (function Passed _ -> true | _ -> false) |> Array.length - let failedCount = testResults |> Array.filter (function Failed _ -> true | _ -> false) |> Array.length - let pendingCount = testResults |> Array.filter (function Pending _ -> true | _ -> false) |> Array.length + let passedCount = testResults |> Seq.filter _.Outcome.IsPassed |> Seq.length + let failedCount = testResults |> Seq.filter _.Outcome.IsFailed |> Seq.length + let pendingCount = testResults |> Seq.filter _.Outcome.IsPending |> Seq.length - printfn $"Summary: {passedCount} passed, {failedCount} failed, {pendingCount} pending" + printfn $"Summary: %d{passedCount} passed, %d{failedCount} failed, %d{pendingCount} pending" printfn $"Total time: %s{sw.Elapsed.ToString()}" type TapReporter() = - interface ITestReporter with - member this.Begin(totalCount: int): unit = + inherit TestReporter() with + override this.Begin(totalCount: int): unit = printfn "TAP version 14" printfn "1..%d" totalCount - member this.BeginSuite(name: string, path: Path): unit = + override this.BeginSuite(name: string, path: Path): unit = + () + override this.Debug(message: string, path: Path): unit = () - member this.Debug(message: string, path: Path): unit = + override this.End(arg1: TestResult []): unit = () + override this.EndSuite(name: string, path: Path): unit = () - member this.End(arg1: TestResult []): unit = () - member this.EndSuite(name: string, path: Path): unit = - printfn "" - member this.Info(message: string, path: Path): unit = + override this.Info(message: string, path: Path): unit = () - member this.ReportResult(result: TestResult, path: Path): unit = - match result with + override this.ReportResult(result: TestResult, path: Path): unit = + match result.Outcome with | Passed (_, name) -> printf "ok %s\n" name | Failed (_, name, ex) -> @@ -219,33 +251,63 @@ module Reporters = | Pending (_, name) -> printf "ok %s # SKIP\n" name -type TestContext = { - Path: Path - ParentBeforeHooks: HookFunction list - ParentAfterHooks: HookFunction list - Reporter: ITestReporter - Log: string -> unit -} -with - static member New = { - Path = Path [] - ParentBeforeHooks = [] - ParentAfterHooks = [] - Reporter = Reporters.ConsoleReporter() :> ITestReporter - Log = printfn "%s" - } +[] +type Runner() = + abstract member Run: Describe -> Async + abstract member CollectDescribes: Describe -> CollectedDescribe + abstract member RunTestCase: Path -> It -> HookFunctions -> Async + abstract member RunCollectedDescribe: CollectedDescribe -> Async + abstract member SequenceAsync: Async<'T> list -> Async<'T array> + +type StepsOrderingDelegate = CollectedStep list -> CollectedStep list +type LogDelegate = string -> unit + +type DefaultRunner(reporter: TestReporter, order: StepsOrderingDelegate) = + inherit Runner() + override _.SequenceAsync xs = Async.Sequential xs + override this.Run suite = + async { + reporter.Begin suite.TotalCount + let collected = this.CollectDescribes suite + let! testResults = this.RunCollectedDescribe collected + reporter.EndSuite(suite.Name, Path [suite.Name]) + reporter.End testResults + return testResults + } + override _.CollectDescribes describe = + let rec loop parentPath parentHookFunctions (d: Describe) = + let hookFunctions = d.HookFunctions + let hookFunctions' = { + Before = parentHookFunctions.Before @ hookFunctions.Before + After = hookFunctions.After @ parentHookFunctions.After + } + let path = parentPath @ [d.Name] + let steps = + d.Steps + |> List.map (function + | ItStep it -> CollectedIt (Path path, hookFunctions', it) + | LogStatementStep log -> CollectedLog (Path path, log)) + let children = + d.Children + |> List.map (loop path hookFunctions') + { + Name = d.Name + Path = Path path + Steps = steps + Children = children + } + loop [] { Before = []; After = [] } describe -module Runner = - let private runTestCase path (testCase: It) beforeHooks afterHooks: Async> = + override _.RunTestCase(path: Path) (testCase: It) hookFunctions: Async = async { // setup logging functions let info', debug' = info, debug use _ = { new System.IDisposable with member _.Dispose() = info <- info'; debug <- debug' } - let logs = ResizeArray() - info <- fun s -> logs.Add (Info s) - debug <- fun s -> logs.Add (Debug s) - for hookFunction in beforeHooks do + let mutable logs = [] + info <- fun s -> logs <- Info s :: logs + debug <- fun s -> logs <- Debug s :: logs + for hookFunction in hookFunctions.Before do do! hookFunction() let! result = match testCase.Body with @@ -261,84 +323,62 @@ module Runner = async { return Pending (path, testCase.Name) } - for hookFunction in afterHooks do + for hookFunction in hookFunctions.After do do! hookFunction() - return result, logs + return { + Outcome = result + Logs = List.rev logs + } } - let rec doRunTestSuite (suite: Describe) (context: TestContext): Async = + override this.RunCollectedDescribe(cd: CollectedDescribe): Async = async { - context.Reporter.BeginSuite(suite.Name, context.Path) - - let beforeHooks, afterHooks = - suite.Each - |> List.fold (fun (be, af) hook -> - match hook with - | Before hookFunction -> hookFunction :: be, af - | After hookFunction -> be, hookFunction :: af - ) ([], []) - let beforeHooks = List.rev beforeHooks |> List.append context.ParentBeforeHooks - let afterHooks = context.ParentAfterHooks |> List.append (List.rev afterHooks) - - let! testResults = - suite.Steps + reporter.BeginSuite(cd.Name, cd.Path) + let! stepResults = + cd.Steps + |> order |> List.map (function - | ItStep itCase -> - async { - let! s, i = runTestCase context.Path itCase beforeHooks afterHooks - return Some (s, i) - } - | LogStatementStep (Info message) -> + | CollectedIt (path, hookFunctions, it) -> async { - context.Reporter.Info(message, context.Path) - return None + let! result = this.RunTestCase path it hookFunctions + for log in result.Logs do + match log with + | Info message -> reporter.Info(message, path) + | Debug message -> reporter.Debug(message, path) + reporter.ReportResult(result, path) + return Some result } - | LogStatementStep (Debug message) -> + | CollectedLog (path, log) -> async { - context.Reporter.Debug(message, context.Path) + match log with + | Info message -> reporter.Info(message, path) + | Debug message -> reporter.Debug(message, path) return None }) - |> Async.Sequential - - let itResults = testResults |> Array.choose id - for result, logs in itResults do - for log in logs do - match log with - | Info message -> context.Reporter.Info(message, context.Path) - | Debug message -> context.Reporter.Debug(message, context.Path) - context.Reporter.ReportResult(result, context.Path) - - let! childrenResults = - suite.Children - |> Seq.map (fun child -> - let childContext = - { context with - ParentBeforeHooks = beforeHooks - ParentAfterHooks = afterHooks - Path = Path (context.Path.Value @ [child.Name]) } - doRunTestSuite - child - childContext) - |> Async.Sequential - let head = itResults |> Array.map fst - let tail = Array.concat childrenResults - let allResults = Array.concat [| head; tail |] + |> this.SequenceAsync + let! childResults = + cd.Children + |> List.map this.RunCollectedDescribe + |> this.SequenceAsync + reporter.EndSuite(cd.Name, cd.Path) + let allResults = + Array.concat [ + Array.choose id stepResults + Array.concat childResults + ] return allResults } -let runTestSuiteWithContext (context: TestContext) (sb: Describe) = +let runTestSuiteCustom (runner: Runner) (describe: Describe) = async { - context.Reporter.Begin sb.TotalCount - let! testResults = Runner.doRunTestSuite sb { context with Path = Path (context.Path.Value @ [sb.Name]) } - context.Reporter.EndSuite(sb.Name, context.Path) - context.Reporter.End testResults - return testResults |> Array.sumBy (function Failed _ -> 1 | _ -> 0) + return! runner.Run describe } let runTestSuite (describe: Describe) = - runTestSuiteWithContext - TestContext.New - describe + async { + let reporter = Reporters.ConsoleReporter() + return! runTestSuiteCustom (DefaultRunner(reporter, id)) describe + } [] module Constraints = diff --git a/Sprout.fsproj b/Sprout.fsproj index f6120f5..c8a0e38 100644 --- a/Sprout.fsproj +++ b/Sprout.fsproj @@ -16,6 +16,8 @@ embedded true true + + 3579 diff --git a/Tests.fsx b/Tests.fsx index 407a5e4..6c2d627 100644 --- a/Tests.fsx +++ b/Tests.fsx @@ -2,6 +2,12 @@ open Sprout +// This file contains a set of tests that demonstrate the features of the Sprout +// testing framework. It includes both synchronous and asynchronous tests, nested +// suites, and various assertions. The tests are designed to showcase the +// capabilities of the framework, including setup and teardown functions, logging, +// and custom runners. + let s1 = describe "Suite 1" {} let s2 = describe "Suite 2" { beforeEach { @@ -12,7 +18,6 @@ let s2 = describe "Suite 2" { info "This test passes in Suite 2" } } -runTestSuite (describe "Main Suite" { s1; s2 }) let suite = describe "A larger test suite" { Info "Top level info message" @@ -84,10 +89,29 @@ let asyncSuite = describe "Async Tests" { } [ + // run a suite on the fly, this one references the above suites + runTestSuite (describe "Main Suite" { s1; s2 }) + + // run the suite with a console reporter runTestSuite suite - runTestSuiteWithContext - { TestContext.New with Reporter = Reporters.TapReporter() } + + // run the suite with a tap reporter + runTestSuiteCustom + (DefaultRunner(Reporters.TapReporter(), id)) suite + + // create a custom runner that runs tests in parallel + let silentTapReporter = { + new Reporters.TapReporter() with + override _.ReportResult(_, _) = () } + let parallelRunner = { + new DefaultRunner(silentTapReporter, id) with + override _.SequenceAsync args = Async.Parallel args } + runTestSuiteCustom + parallelRunner + suite + + // run async tests runTestSuite asyncSuite ] |> Async.Sequential diff --git a/expected.log b/expected.log index 6d6e5ea..6acdab5 100644 --- a/expected.log +++ b/expected.log @@ -1,3 +1,12 @@ +Main Suite + Suite 1 + Suite 2 + Before each test in Suite 2 + This test passes in Suite 2 +  โœ… passed: should pass in Suite 2 +All tests passed! +Summary: 1 passed, 0 failed, 0 pending +Total time: 00:00:00.0078899 A larger test suite Top level info message Before each test @@ -33,7 +42,7 @@ There were 2 test failures: - A larger test suite / should fail - Intentional failure - A larger test suite / Arithmetic / Faulty Addition / should fail when adding incorrect numbers - Expected 5 but got 4 Summary: 4 passed, 2 failed, 1 pending -Total time: 00:00:00.0124887 +Total time: 00:00:00.0049374 TAP version 14 1..7 ok should pass @@ -51,7 +60,8 @@ not ok should fail when adding incorrect numbers message: Expected 5 but got 4 severity: fail ... - +TAP version 14 +1..7 Async Tests Before each async test  โœ… passed: should run an async test @@ -60,4 +70,4 @@ not ok should fail when adding incorrect numbers There were 1 test failures: - Async Tests / should handle async failure - Intentional async failure Summary: 1 passed, 1 failed, 0 pending -Total time: 00:00:00.4115251 +Total time: 00:00:00.4129414 diff --git a/out.png b/out.png index fc19f14..5cf09d3 100644 Binary files a/out.png and b/out.png differ diff --git a/sample.fsx b/sample.fsx index 0a7bdcc..67c0c82 100644 --- a/sample.fsx +++ b/sample.fsx @@ -69,7 +69,9 @@ let suite = describe "A test suite" { } } -runTestSuiteWithContext - { TestContext.New with Reporter = Reporters.ConsoleReporter("v", "x", "?", " ") :> ITestReporter } +runTestSuiteCustom + // id is used to order the tests (it blocks) + // you can specify a custom ordering function if needed + (DefaultRunner(Reporters.ConsoleReporter("v", "x", "?", " "), id)) suite |> Async.RunSynchronously