From 8a1832ae0b0fd934d7d5a331cc066acd6cd678b5 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sun, 27 Aug 2023 20:46:19 -0400 Subject: [PATCH 1/9] remove name from at-compact --- src/compact.jl | 42 +++---------------------------- test/compact.jl | 66 +++++++++++++++++-------------------------------- 2 files changed, 25 insertions(+), 83 deletions(-) diff --git a/src/compact.jl b/src/compact.jl index cd758c1..dc54ad3 100644 --- a/src/compact.jl +++ b/src/compact.jl @@ -68,20 +68,6 @@ for epoch in 1:1000 Flux.train!((m,x,y) -> (m(x) - y)^2, model, data, optim) end ``` - -You may also specify a `name` for the model, which will -be used instead of the default printout, which gives a verbatim -representation of the code used to construct the model: - -``` -model = @compact(w=rand(3), name="Linear(3 => 1)") do x - sum(w .* x) -end -println(model) # "Linear(3 => 1)" -``` - -This can be useful when using `@compact` to hierarchically construct -complex models to be used inside a `Chain`. """ macro compact(_exs...) # check inputs, extracting function expression fex and unprocessed keyword arguments _kwexs @@ -108,16 +94,6 @@ macro compact(_exs...) kwexs2 = map(ex -> Expr(:kw, ex.args...), _kwexs) # handle keyword arguments provided before semicolon kwexs = (kwexs1..., kwexs2...) - # check if user has named layer: - name = findfirst(ex -> ex.args[1] == :name, kwexs) - if name !== nothing && kwexs[name].args[2] !== nothing - length(kwexs) == 1 && error("expects keyword arguments") - name_str = kwexs[name].args[2] - # remove name from kwexs (a tuple) - kwexs = (kwexs[1:name-1]..., kwexs[name+1:end]...) - name = name_str - end - # make strings layer = "@compact" setup = NamedTuple(map(ex -> Symbol(string(ex.args[1])) => string(ex.args[2]), kwexs)) @@ -136,7 +112,7 @@ macro compact(_exs...) fex = supportself(fex, vars) # assemble - return esc(:($CompactLayer($fex, $name, ($layer, $input, $block), $setup; $(kwexs...)))) + return esc(:($CompactLayer($fex, ($layer, $input, $block), $setup; $(kwexs...)))) end function supportself(fex::Expr, vars) @@ -155,12 +131,11 @@ end struct CompactLayer{F,NT1<:NamedTuple,NT2<:NamedTuple} fun::F - name::Union{String,Nothing} strings::NTuple{3,String} setup_strings::NT1 variables::NT2 end -CompactLayer(f::Function, name::Union{String,Nothing}, str::Tuple, setup_str::NamedTuple; kw...) = CompactLayer(f, name, str, setup_str, NamedTuple(kw)) +CompactLayer(f::Function, str::Tuple, setup_str::NamedTuple; kw...) = CompactLayer(f, str, setup_str, NamedTuple(kw)) (m::CompactLayer)(x...) = m.fun(m.variables, x...) CompactLayer(args...) = error("CompactLayer is meant to be constructed by the macro") Flux.@functor CompactLayer @@ -179,19 +154,9 @@ end function Flux._big_show(io::IO, obj::CompactLayer, indent::Int=0, name=nothing) setup_strings = obj.setup_strings - local_name = obj.name - has_explicit_name = local_name !== nothing - if has_explicit_name - if indent != 0 || length(Flux.params(obj)) <= 2 - _just_show_params(io, local_name, obj, indent) - else # indent == 0 - print(io, local_name) - Flux._big_finale(io, obj) - end - else # no name, so print normally layer, input, block = obj.strings pre, post = ("(", ")") - println(io, " "^indent, isnothing(name) ? "" : "$name = ", layer, pre) + println(io, " "^indent, "@compact", pre) for k in keys(obj.variables) v = obj.variables[k] if Flux._show_leaflike(v) @@ -220,7 +185,6 @@ function Flux._big_show(io::IO, obj::CompactLayer, indent::Int=0, name=nothing) else println(io, ",") end - end end # Modified from src/layers/show.jl diff --git a/test/compact.jl b/test/compact.jl index 88697e9..a1bfe83 100644 --- a/test/compact.jl +++ b/test/compact.jl @@ -118,15 +118,6 @@ end @test similar_strings(get_model_string(model), expected_string) end - @testset "Custom naming" begin - model = @compact(w=Dense(32, 32), name="Linear(...)") do x, y - tmp = sum(w(x)) - return tmp + y - end - expected_string = "Linear(...) # 1_056 parameters" - @test similar_strings(get_model_string(model), expected_string) - end - @testset "Hierarchical models" begin model1 = @compact(w1=Dense(32=>32, relu), w2=Dense(32=>32, relu)) do x w2(w1(x)) @@ -148,6 +139,28 @@ end @test similar_strings(get_model_string(model2), expected_string) end +#= # This test is broken: + +julia> model1 = @compact(w1=Dense(32=>32, relu), w2=Dense(32=>32, relu)) do x + w2(w1(x)); + +julia> model2 = @compact(w1=model1, w2=Dense(32=>32, relu)) do x + w2(w1(x)) + end +@compact( + @compact( + w1 = Dense(32 => 32, relu), # 1_056 parameters + w2 = Dense(32 => 32, relu), # 1_056 parameters + ) do x + w2(w1(x)) + end, + w2 = Dense(32 => 32, relu), # 1_056 parameters +) do x + w2(w1(x)) +end # Total: 6 arrays, 3_168 parameters, 13.239 KiB. + +=# + @testset "Array parameters" begin model = @compact(x=randn(32), w=Dense(32=>32)) do s w(x .* s) @@ -161,41 +174,6 @@ end @test similar_strings(get_model_string(model), expected_string) end - @testset "Hierarchy with inner model named" begin - model = @compact( - w1=@compact(w1=randn(32, 32), name="Model(32)") do x - w1 * x - end, - w2=randn(32, 32), - w3=randn(32), - ) do x - w2 * w1(x) - end - expected_string = """@compact( - Model(32), # 1_024 parameters - w2 = randn(32, 32), # 1_024 parameters - w3 = randn(32), # 32 parameters - ) do x - w2 * w1(x) - end # Total: 3 arrays, 2_080 parameters, 17.089 KiB.""" - @test similar_strings(get_model_string(model), expected_string) - end - - @testset "Hierarchy with outer model named" begin - model = @compact( - w1=@compact(w1=randn(32, 32)) do x - w1 * x - end, - w2=randn(32, 32), - w3=randn(32), - name="Model(32)" - ) do x - w2 * w1(x) - end - expected_string = """Model(32) # Total: 3 arrays, 2_080 parameters, 17.057KiB.""" - @test similar_strings(get_model_string(model), expected_string) - end - @testset "Dependent initializations" begin # Test that initialization lines cannot depend on each other @test_throws UndefVarError @compact(y = 3, z = y^2) do x From 6734e896839b0807a77abdf65131b88efab74a99 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sun, 27 Aug 2023 22:25:57 -0400 Subject: [PATCH 2/9] add NoShow as an (orthogonalised) alternative --- src/Fluxperimental.jl | 3 +++ src/compact.jl | 2 +- src/noshow.jl | 60 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 src/noshow.jl diff --git a/src/Fluxperimental.jl b/src/Fluxperimental.jl index 893dc46..d2ae839 100644 --- a/src/Fluxperimental.jl +++ b/src/Fluxperimental.jl @@ -13,6 +13,9 @@ include("chain.jl") include("compact.jl") +include("noshow.jl") +export NoShow + include("new_recur.jl") end # module Fluxperimental diff --git a/src/compact.jl b/src/compact.jl index dc54ad3..9a1b33a 100644 --- a/src/compact.jl +++ b/src/compact.jl @@ -156,7 +156,7 @@ function Flux._big_show(io::IO, obj::CompactLayer, indent::Int=0, name=nothing) setup_strings = obj.setup_strings layer, input, block = obj.strings pre, post = ("(", ")") - println(io, " "^indent, "@compact", pre) + println(io, " "^indent, isnothing(name) ? "" : "$name = ", layer, pre) for k in keys(obj.variables) v = obj.variables[k] if Flux._show_leaflike(v) diff --git a/src/noshow.jl b/src/noshow.jl new file mode 100644 index 0000000..0317114 --- /dev/null +++ b/src/noshow.jl @@ -0,0 +1,60 @@ + +""" + NoShow(layer) + NoShow(string, layer) + +This alters printing (for instance at the REPL prompt) to let you hide the complexity +of some part of a Flux model. It has no effect on the actual running of the model. + +By default it prints `NoShow(...)` instead of the given layer. +If you provide a string, it prints that instead -- it can be anything, +but it may make sense to print the name of a function which will +re-create the same structure. + +# Examples + +```jldoctest +julia> Chain(Dense(2 => 3), NoShow(Parallel(vcat, Dense(3 => 4), Dense(3 => 5))), Dense(9 => 10)) +Chain( + Dense(2 => 3), # 9 parameters + NoShow(...), # 36 parameters + Dense(9 => 10), # 100 parameters +) # Total: 8 arrays, 145 parameters, 1.191 KiB. + +julia> PseudoLayer((i,o)::Pair) = Parallel(+, Dense(i => o, relu), Dense(i => o, tanh)); + +julia> mid = PseudoLayer(3 => 10); + +julia> Chain(Dense(2 => 3), NoShow("PseudoLayer(3 => 10)", mid), Dense(9 => 10)) +Chain( + Dense(2 => 3), # 9 parameters + PseudoLayer(3 => 10), # 80 parameters + Dense(9 => 10), # 100 parameters +) # Total: 8 arrays, 189 parameters, 1.379 KiB. +``` +""" +struct NoShow{T} + str::String + layer::T +end + +NoShow(layer) = NoShow("", layer) + +Flux.@functor NoShow + +(no::NoShow)(x...) = no.layer(x...) + +Base.show(io::IO, no::NoShow) = print(io, isempty(no.str) ? "NoShow(...)" : no.str) + +Flux._show_leaflike(::NoShow) = true # I think this is right +Flux._show_children(::NoShow) = (;) # Seems to be needed? + +function Base.show(io::IO, ::MIME"text/plain", m::NoShow) + if get(io, :typeinfo, nothing) === nothing # e.g., top level of REPL + Flux._big_show(io, m) + elseif !get(io, :compact, false) # e.g., printed inside a Vector, but not a matrix + Flux._layer_show(io, m) + else + show(io, m) + end +end From 57b085886b6c393721095ee94220084e7bb3fb33 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 11 Oct 2023 19:03:11 -0400 Subject: [PATCH 3/9] add tests --- test/compact.jl | 32 ++++++++++---------------------- test/noshow.jl | 28 ++++++++++++++++++++++++++++ test/runtests.jl | 1 + 3 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 test/noshow.jl diff --git a/test/compact.jl b/test/compact.jl index a1bfe83..b3f1e42 100644 --- a/test/compact.jl +++ b/test/compact.jl @@ -139,28 +139,6 @@ end @test similar_strings(get_model_string(model2), expected_string) end -#= # This test is broken: - -julia> model1 = @compact(w1=Dense(32=>32, relu), w2=Dense(32=>32, relu)) do x - w2(w1(x)); - -julia> model2 = @compact(w1=model1, w2=Dense(32=>32, relu)) do x - w2(w1(x)) - end -@compact( - @compact( - w1 = Dense(32 => 32, relu), # 1_056 parameters - w2 = Dense(32 => 32, relu), # 1_056 parameters - ) do x - w2(w1(x)) - end, - w2 = Dense(32 => 32, relu), # 1_056 parameters -) do x - w2(w1(x)) -end # Total: 6 arrays, 3_168 parameters, 13.239 KiB. - -=# - @testset "Array parameters" begin model = @compact(x=randn(32), w=Dense(32=>32)) do s w(x .* s) @@ -212,3 +190,13 @@ end # Total: 6 arrays, 3_168 parameters, 13.239 KiB. end end + +@testset "Custom naming of @compact with NoShow" begin + _model = @compact(w=Dense(32, 32)) do x, y + tmp = sum(w(x)) + return tmp + y + end + model = NoShow(_model) + expected_string = "NoShow(...) # 1_056 parameters" + @test similar_strings(get_model_string(model), expected_string) +end diff --git a/test/noshow.jl b/test/noshow.jl new file mode 100644 index 0000000..496270f --- /dev/null +++ b/test/noshow.jl @@ -0,0 +1,28 @@ + +@testset "NoShow" begin + d23 = Dense(2 => 3) + d34 = Dense(3 => 4, tanh) + d35 = Dense(3 => 5, relu) + d910 = Dense(9 => 10) + + model = Chain(d23, Parallel(vcat, d34, d35), d910) + m_no = Chain(d23, NoShow(Parallel(vcat, d34, NoShow("zzz", d35))), d910) + + @test sum(length, Flux.params(model)) == sum(length, Flux.params(m_no)) + + xin = randn(Float32, 2, 7) + @test model(xin) ≈ m_no(xin) + + # gradients + grad = gradient(m -> m(xin)[1], model)[1] + g_no = gradient(m -> m(xin)[1], m_no)[1] + + @test grad.layers[2].layers[1].bias ≈ g_no.layers[2].layer.layers[1].bias + @test grad.layers[2].layers[2].bias ≈ g_no.layers[2].layer.layers[2].layer.bias + + # printing -- see also compact.jl for another test + @test !contains(string(model), "NoShow(...)") + @test contains(string(m_no), "NoShow(...)") + @test !contains(string(m_no), "3 => 4") +end + diff --git a/test/runtests.jl b/test/runtests.jl index 5291a8a..0efa3ed 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,7 @@ using Flux, Fluxperimental include("chain.jl") include("compact.jl") + include("noshow.jl") include("new_recur.jl") From 78cec0387497540631a9d5c465230e8bf62e0275 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 11 Oct 2023 19:04:05 -0400 Subject: [PATCH 4/9] silence a warning --- test/compact.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/compact.jl b/test/compact.jl index b3f1e42..9b76e02 100644 --- a/test/compact.jl +++ b/test/compact.jl @@ -101,7 +101,7 @@ end (1, 128), (1,), ] - @test size(model(randn(n_in, 32))) == (1, 32) + @test size(model(randn(Float32, n_in, 32))) == (1, 32) end @testset "String representations" begin From 75744f92878863eb490110ad52b4550a9481c1f7 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 11 Oct 2023 19:14:02 -0400 Subject: [PATCH 5/9] improve example --- src/noshow.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/noshow.jl b/src/noshow.jl index 0317114..7cc6445 100644 --- a/src/noshow.jl +++ b/src/noshow.jl @@ -21,11 +21,13 @@ Chain( Dense(9 => 10), # 100 parameters ) # Total: 8 arrays, 145 parameters, 1.191 KiB. -julia> PseudoLayer((i,o)::Pair) = Parallel(+, Dense(i => o, relu), Dense(i => o, tanh)); +julia> PseudoLayer((i,o)::Pair) = NoShow( + "PseudoLayer(\$i => \$o)", + Parallel(+, Dense(i => o, relu), Dense(i => o, tanh)), + ) +PseudoLayer (generic function with 1 method) -julia> mid = PseudoLayer(3 => 10); - -julia> Chain(Dense(2 => 3), NoShow("PseudoLayer(3 => 10)", mid), Dense(9 => 10)) +julia> Chain(Dense(2 => 3), PseudoLayer(3 => 10), Dense(9 => 10)) Chain( Dense(2 => 3), # 9 parameters PseudoLayer(3 => 10), # 80 parameters From f9a28cd73cd9416d578ca874e12aa35c5d1e82f2 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:57:05 -0400 Subject: [PATCH 6/9] lowercase + simplify --- src/noshow.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/noshow.jl b/src/noshow.jl index 7cc6445..fdc89aa 100644 --- a/src/noshow.jl +++ b/src/noshow.jl @@ -21,16 +21,16 @@ Chain( Dense(9 => 10), # 100 parameters ) # Total: 8 arrays, 145 parameters, 1.191 KiB. -julia> PseudoLayer((i,o)::Pair) = NoShow( - "PseudoLayer(\$i => \$o)", +julia> pseudolayer((i,o)::Pair) = NoShow( + "pseudolayer(\$i => \$o)", Parallel(+, Dense(i => o, relu), Dense(i => o, tanh)), ) -PseudoLayer (generic function with 1 method) +pseudolayer (generic function with 1 method) -julia> Chain(Dense(2 => 3), PseudoLayer(3 => 10), Dense(9 => 10)) +julia> Chain(Dense(2 => 3), pseudolayer(3 => 10), Dense(9 => 10)) Chain( Dense(2 => 3), # 9 parameters - PseudoLayer(3 => 10), # 80 parameters + pseudolayer(3 => 10), # 80 parameters Dense(9 => 10), # 100 parameters ) # Total: 8 arrays, 189 parameters, 1.379 KiB. ``` @@ -40,13 +40,13 @@ struct NoShow{T} layer::T end -NoShow(layer) = NoShow("", layer) +NoShow(layer) = NoShow("NoShow(...)", layer) Flux.@functor NoShow (no::NoShow)(x...) = no.layer(x...) -Base.show(io::IO, no::NoShow) = print(io, isempty(no.str) ? "NoShow(...)" : no.str) +Base.show(io::IO, no::NoShow) = print(io, no.str) Flux._show_leaflike(::NoShow) = true # I think this is right Flux._show_children(::NoShow) = (;) # Seems to be needed? From 38d6b029e129b3a87cdf0af4cde7e47a54ddff73 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Thu, 12 Oct 2023 16:56:37 -0400 Subject: [PATCH 7/9] Update compact.jl Co-authored-by: Gaurav Arya --- test/compact.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/compact.jl b/test/compact.jl index 9b76e02..5888c53 100644 --- a/test/compact.jl +++ b/test/compact.jl @@ -199,4 +199,6 @@ end model = NoShow(_model) expected_string = "NoShow(...) # 1_056 parameters" @test similar_strings(get_model_string(model), expected_string) + model2 = NoShow("test", _model) + @test similar_strings(get_model_string(model2), "test") end From 5adf64e38271952021006a11f3b9fee37afc0219 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Thu, 12 Oct 2023 16:56:53 -0400 Subject: [PATCH 8/9] Update compact.jl Co-authored-by: Gaurav Arya --- src/compact.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compact.jl b/src/compact.jl index 9a1b33a..c871dc3 100644 --- a/src/compact.jl +++ b/src/compact.jl @@ -68,6 +68,7 @@ for epoch in 1:1000 Flux.train!((m,x,y) -> (m(x) - y)^2, model, data, optim) end ``` +To specify a custom printout for the model, you may find [`NoShow`](@ref) useful. """ macro compact(_exs...) # check inputs, extracting function expression fex and unprocessed keyword arguments _kwexs From 345ef7a4e7f264e4edb9871ce35bbc52630f5b9c Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Thu, 12 Oct 2023 17:09:50 -0400 Subject: [PATCH 9/9] Update test/compact.jl --- test/compact.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/compact.jl b/test/compact.jl index 5888c53..e27f971 100644 --- a/test/compact.jl +++ b/test/compact.jl @@ -200,5 +200,5 @@ end expected_string = "NoShow(...) # 1_056 parameters" @test similar_strings(get_model_string(model), expected_string) model2 = NoShow("test", _model) - @test similar_strings(get_model_string(model2), "test") + @test contains(get_model_string(model2), "test") end