Create structured output from evaluation

Project.toml

@@ -1,7 +1,7 @@
 name = "TestPicker"
 uuid = "a64165b9-4409-4de6-85cd-a4e0953bae44"
 authors = ["theogf <[email protected]> and contributors"]
-version = "2.0.1"
+version = "2.1.0"
 
 [deps]
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

src/TestPicker.jl

@@ -47,6 +47,17 @@ struct EvalTest
     info::TestInfo
 end
 
+"""
+    EvalResult{T}
+
+Result from evaluating a given `EvalTest`.
+"""
+struct EvalResult{T}
+    success::Bool
+    info::TestInfo
+    result::T
+end
+
 """
     LATEST_EVAL
 

src/precompilation.jl

@@ -6,7 +6,7 @@
 
         # Test file operations
         try
-            root, test_files = get_test_files(pkg_spec)
+            root, testfiles = get_testfiles(pkg_spec)
 
             # Test block operations
             interface = StdTestset()
@@ -24,19 +24,19 @@
             end
 
             # Get testblocks from a test file
-            if !isempty(test_files)
-                test_file = joinpath(root, first(test_files))
-                if isfile(test_file)
-                    get_testblocks([interface], test_file)
+            if !isempty(testfiles)
+                testfile = joinpath(root, first(testfiles))
+                if isfile(testfile)
+                    get_testblocks([interface], testfile)
                 end
             end
 
             # File matching operations
-            get_matching_files("test", test_files)
+            get_matching_files("test", testfiles)
 
             # Build info to syntax
-            if !isempty(test_files)
-                matched_files = [first(test_files)]
+            if !isempty(testfiles)
+                matched_files = [first(testfiles)]
                 build_info_to_syntax([interface], root, matched_files)
             end
 
@@ -51,8 +51,7 @@
 
             # Construct pipeline object without executing
             test_pipeline = pipeline(
-                Cmd(fzf_cmd; ignorestatus=true, dir=root);
-                stdin=IOBuffer("test\n")
+                Cmd(fzf_cmd; ignorestatus=true, dir=root); stdin=IOBuffer("test\n")
             )
 
             # Precompile bat path construction

src/repl.jl

@@ -69,7 +69,7 @@ function REPL.complete_line(::TestModeCompletionProvider, s::LineEdit.PromptStat
     # Try to get test files - if it fails, return empty completions
     try
         pkg = current_pkg()
-        _, files = get_test_files(pkg)
+        _, files = get_testfiles(pkg)
 
         # Extract just the base file names (without path, but keep .jl extension)
         file_names = unique([basename(f) for f in files])

src/testblock.jl

@@ -75,20 +75,20 @@ function get_testblocks!(
 end
 
 """
-    get_matching_files(file_query::AbstractString, test_files::AbstractVector{<:AbstractString}) -> Vector{String}
+    get_matching_files(file_query::AbstractString, testfiles::AbstractVector{<:AbstractString}) -> Vector{String}
 
 Filter test files using fzf's non-interactive filtering based on the given query.
 
 Uses `fzf --filter` to perform fuzzy matching on the provided list of test files,
 returning only those that match the query pattern.
 """
 function get_matching_files(
-    file_query::AbstractString, test_files::AbstractVector{<:AbstractString}
+    file_query::AbstractString, testfiles::AbstractVector{<:AbstractString}
 )
     return readlines(
         pipeline(
             Cmd(`$(fzf()) --filter $(file_query)`; ignorestatus=true);
-            stdin=IOBuffer(join(test_files, '\n')),
+            stdin=IOBuffer(join(testfiles, '\n')),
         ),
     )
 end
@@ -173,14 +173,7 @@ function pick_testblock(
 )
     if !interactive
         # Non-interactive mode: use fzf --filter to get matching test blocks
-        args = [
-            "--filter",
-            testset_query,
-            "-d",
-            "$(separator())",
-            "--nth",
-            "1",
-        ]
+        args = ["--filter", testset_query, "-d", "$(separator())", "--nth", "1"]
         cmd = Cmd(`$(fzf()) $(args)`; ignorestatus=true, dir=root)
         return readlines(pipeline(cmd; stdin=IOBuffer(join(keys(tabled_keys), '\n'))))
     end
@@ -221,7 +214,7 @@ function testblock_list(
     info_to_syntax::Dict{TestBlockInfo,SyntaxBlock},
     display_to_info::Dict{String,TestBlockInfo},
     pkg::PackageSpec,
-)
+)::Vector{EvalTest}
     map(choices) do choice
         blockinfo = display_to_info[choice]
         syntax_block = info_to_syntax[blockinfo]
@@ -270,8 +263,9 @@ function fzf_testblock_from_files(
         tests = testblock_list(choices, info_to_syntax, display_to_info, pkg)
         clean_results_file(pkg)
         LATEST_EVAL[] = tests
-        for test in tests
-            eval_in_module(test, pkg)
+        map(tests) do test
+            result = eval_in_module(test, pkg)
+            EvalResult(isnothing(result), test.info, result)
         end
     end
 end
@@ -295,8 +289,10 @@ function fzf_testblock(
     interactive::Bool=true,
 )
     pkg = current_pkg()
-    root, test_files = get_test_files(pkg)
+    root, testfiles = get_testfiles(pkg)
     # We fetch all valid test files.
-    matched_files = get_matching_files(fuzzy_file, test_files)
-    fzf_testblock_from_files(interfaces, matched_files, fuzzy_testset, pkg, root; interactive)
+    matched_files = get_matching_files(fuzzy_file, testfiles)
+    return fzf_testblock_from_files(
+        interfaces, matched_files, fuzzy_testset, pkg, root; interactive
+    )
 end

src/testfile.jl

@@ -1,5 +1,5 @@
 """
-    select_test_files(query::AbstractString, pkg::PackageSpec=current_pkg(); interactive::Bool=true) -> (Symbol, String, Vector{String})
+    select_testfiles(query::AbstractString, pkg::PackageSpec=current_pkg(); interactive::Bool=true) -> (Symbol, String, Vector{String})
 
 Select test files using fzf based on a fuzzy search query.
 
@@ -13,8 +13,10 @@ Returns a tuple of (mode, root, files) where:
 - root is the test directory path
 - files are relative paths (not joined with root yet)
 """
-function select_test_files(query::AbstractString, pkg::PackageSpec=current_pkg(); interactive::Bool=true)
-    root, files = get_test_files(pkg)
+function select_testfiles(
+    query::AbstractString, pkg::PackageSpec=current_pkg(); interactive::Bool=true
+)
+    root, files = get_testfiles(pkg)
 
     if !interactive
         # Non-interactive mode: use fzf --filter to get matching files
@@ -73,14 +75,14 @@ function select_test_files(query::AbstractString, pkg::PackageSpec=current_pkg()
 end
 
 """
-    get_test_files(pkg::PackageSpec=current_pkg()) -> (String, Vector{String})
+    get_testfiles(pkg::PackageSpec=current_pkg()) -> (String, Vector{String})
 
 Discover and return all Julia test files for a package.
 
 Recursively searches the package's test directory to find all `.jl` files,
 returning both the test directory path and the collection of relative file paths.
 """
-function get_test_files(pkg::PackageSpec=current_pkg())
+function get_testfiles(pkg::PackageSpec=current_pkg())
     test_dir = get_test_dir_from_pkg(pkg)
     # Recursively get a list of julia files.
     return test_dir,
@@ -112,7 +114,7 @@ test files based on the query.
 """
 function fzf_testfile(query::AbstractString; interactive::Bool=true)
     pkg = current_pkg()
-    mode, root, files = select_test_files(query, pkg; interactive)
+    mode, root, files = select_testfiles(query, pkg; interactive)
 
     if mode == :testblock
         # User pressed ctrl-b, switch to testblock mode (only possible in interactive mode)
@@ -122,43 +124,64 @@ function fzf_testfile(query::AbstractString; interactive::Bool=true)
         # Normal file execution mode
         # Convert relative paths to absolute paths for file execution
         absolute_files = joinpath.(Ref(root), files)
-        return run_test_files(absolute_files, pkg)
+        return run_testfiles(absolute_files, pkg)
     end
 end
 
 """
-    run_test_files(files::AbstractVector{<:AbstractString}, pkg::PackageSpec) -> Nothing
+File to evaluate was empty.
+"""
+struct EmptyFile end
+
+"""
+Provided file could not be found.
+"""
+struct MissingFileException <: Exception
+    file::String
+end
+
+function Base.showerror(io::IO, (; file)::MissingFileException)
+    return println(
+        io,
+        "File $(file) could not be found, this sounds like a bug, please report it on https://github.com/theogf/TestPicker.jl/issues/new.",
+    )
+end
+
+"""
+    run_testfiles(files::AbstractVector{<:AbstractString}, pkg::PackageSpec) -> Nothing
 
 Execute a collection of test files in the package test environment.
 
 Runs each provided test file in sequence, handling errors gracefully and updating
 the test evaluation state. Each file is wrapped in a testset and executed in isolation.
 """
-function run_test_files(files::AbstractVector{<:AbstractString}, pkg::PackageSpec)
+function run_testfiles(files::AbstractVector{<:AbstractString}, pkg::PackageSpec)
     # We return early to not empty the LATEST_EVAL
     isempty(files) && return nothing
     # Reset the latest eval data.
     LATEST_EVAL[] = EvalTest[]
     clean_results_file(pkg)
-    for file in files
+    map(files) do file
         if isempty(file)
+            EmptyFile()
         elseif !isfile(file)
             @error "File $(file) could not be found, this sounds like a bug, please report it on https://github.com/theogf/TestPicker.jl/issues/new."
+            MissingFileException(file)
         else
-            run_test_file(file, pkg)
+            run_testfile(file, pkg)
         end
     end
 end
 
 """
-    run_test_file(file::AbstractString, pkg::PackageSpec) -> Any
+    run_testfile(file::AbstractString, pkg::PackageSpec) -> Any
 
 Execute a single test file in an isolated testset within the package test environment.
 
 Wraps the test file in a testset named after the package and file, handles test
 failures gracefully, and updates the global test state for later inspection.
 """
-function run_test_file(file::AbstractString, pkg::PackageSpec)
+function run_testfile(file::AbstractString, pkg::PackageSpec)
     testset_name = "$(pkg.name) - $(file)"
     test_info = TestInfo(file, "", 0)
     ex = quote
@@ -179,5 +202,7 @@ function run_test_file(file::AbstractString, pkg::PackageSpec)
     else
         LATEST_EVAL[] = [test]
     end
-    return eval_in_module(test, pkg)
+    result = eval_in_module(test, pkg)
+    # result is nothing when the testset is a success.
+    return EvalResult(isnothing(result), test_info, result)
 end

test/testblock.jl

@@ -1,14 +1,15 @@
 using Test
 using JuliaSyntax
 using TestPicker
-using TestPicker: TestBlockInfo, StdTestset, SyntaxBlock
+using TestPicker: TestBlockInfo, StdTestset, SyntaxBlock, EvalResult
 using TestPicker:
     get_testblocks,
-    get_test_files,
+    get_testfiles,
     get_matching_files,
     build_info_to_syntax,
     pick_testblock,
-    istestblock
+    istestblock,
+    fzf_testblock
 
 function no_indentation(s::AbstractString)
     return replace(s, r"^\s+"m => "")
@@ -137,3 +138,12 @@ end
     choices = pick_testblock(tabled_keys_c, "Second", root_subdir; interactive=false)
     @test length(choices) == 1
 end
+
+@testset "fzf_testblock return type" begin
+    interfaces = [StdTestset()]
+
+    # Test that fzf_testblock returns a vector when testblocks match
+    result = fzf_testblock(interfaces, "test-a", "testset"; interactive=false)
+    @test result isa Vector
+    @test all(r -> r isa EvalResult, result)
+end

test/testfile.jl

@@ -1,15 +1,15 @@
 using Test
 using Pkg.Types: PackageSpec
 using TestPicker
-using TestPicker: EvalTest, get_test_files, run_test_file, select_test_files
+using TestPicker: EvalTest, EvalResult, fzf_testfile, get_testfiles, run_testfile, select_testfiles
 
 @testset "Get test files" begin
     path = pkgdir(TestPicker)
     pkg_spec = PackageSpec(; name="TestPicker", path)
-    root, test_files = get_test_files(pkg_spec)
+    root, testfiles = get_testfiles(pkg_spec)
     @test root == joinpath(path, "test")
     @test issetequal(
-        filter(startswith("sandbox"), test_files),
+        filter(startswith("sandbox"), testfiles),
         [
             "sandbox/test-a.jl",
             "sandbox/test-b.jl",
@@ -23,7 +23,7 @@ end
     TestPicker.LATEST_EVAL[] = nothing
     path = pkgdir(TestPicker)
     pkg_spec = PackageSpec(; name="TestPicker", path)
-    @test_logs (:info, "Executing test file sandbox/test-a.jl") run_test_file(
+    @test_logs (:info, "Executing test file sandbox/test-a.jl") run_testfile(
         "sandbox/test-a.jl", pkg_spec
     )
     @test TestPicker.LATEST_EVAL[] isa Vector{EvalTest}
@@ -36,21 +36,36 @@ end
     pkg_spec = PackageSpec(; name="TestPicker", path)
 
     # Test selecting files with a query that matches multiple files
-    mode, root, files = select_test_files("test-a", pkg_spec; interactive=false)
+    mode, root, files = select_testfiles("test-a", pkg_spec; interactive=false)
     @test mode == :file
     @test root == joinpath(path, "test")
     @test "sandbox/test-a.jl" in files
 
     # Test selecting files with a query that matches no files
-    mode, root, files = select_test_files("nonexistent-file-xyz", pkg_spec; interactive=false)
+    mode, root, files = select_testfiles(
+        "nonexistent-file-xyz", pkg_spec; interactive=false
+    )
     @test mode == :file
     @test root == joinpath(path, "test")
     @test isempty(files)
 
     # Test selecting files with a broader query
-    mode, root, files = select_test_files("sandbox", pkg_spec; interactive=false)
+    mode, root, files = select_testfiles("sandbox", pkg_spec; interactive=false)
     @test mode == :file
     @test root == joinpath(path, "test")
     @test length(files) >= 3  # At least test-a, test-b, and weird-name
     @test any(startswith("sandbox/"), files)
 end
+
+@testset "fzf_testfile return type" begin
+    path = pkgdir(TestPicker)
+
+    # Test that fzf_testfile returns a vector when files match
+    result = fzf_testfile("test-a"; interactive=false)
+    @test result isa Vector
+    @test all(r -> r isa EvalResult, result)
+
+    # Test that fzf_testfile returns nothing when no files match
+    result = fzf_testfile("nonexistent-xyz"; interactive=false)
+    @test isnothing(result)
+end