[JuliaLowering] get macro name in ctx.world; fix lowering iterator (#60168)

Evaluating a macro's name should be done in the same world we pass into
lowering and use for expansion. Fix this for all reasonable macro names.

The lowering iterator (currently very cold code only used for
module/toplevel expressions that JuliaLowering "controls") also needs to
update the expansion world between steps.

Author: mlechu

JuliaLowering/src/eval.jl

@@ -33,50 +33,42 @@ end
 # how we end up putting this into Base.
 
 struct LoweringIterator{GraphType}
-    ctx::MacroExpansionContext{GraphType}
+    expr_compat_mode::Bool # later stored in module?
     todo::Vector{Tuple{SyntaxTree{GraphType}, Bool, Int}}
 end
 
-function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_mode::Bool=false)
-    graph = ensure_macro_attributes(syntax_graph(ex))
-    ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
-    ex = reparent(ctx, ex)
-    LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)])
+function lower_init(ex::SyntaxTree{T};
+                    expr_compat_mode::Bool=false) where {T}
+    LoweringIterator{T}(expr_compat_mode, [(ex, false, 0)])
 end
 
-function lower_step(iter, push_mod=nothing)
-    if !isnothing(push_mod)
-        push_layer!(iter.ctx, push_mod, false)
-    end
-
+function lower_step(iter, mod, world=Base.get_world_counter())
     if isempty(iter.todo)
         return Core.svec(:done)
     end
 
-    ex, is_module_body, child_idx = pop!(iter.todo)
+    top_ex, is_module_body, child_idx = pop!(iter.todo)
     if child_idx > 0
-        next_child = child_idx + 1
-        if child_idx <= numchildren(ex)
-            push!(iter.todo, (ex, is_module_body, next_child))
-            ex = ex[child_idx]
+        if child_idx <= numchildren(top_ex)
+            push!(iter.todo, (top_ex, is_module_body, child_idx + 1))
+            ex = top_ex[child_idx]
+        elseif is_module_body
+            return Core.svec(:end_module)
         else
-            if is_module_body
-                pop_layer!(iter.ctx)
-                return Core.svec(:end_module)
-            else
-                return lower_step(iter)
-            end
+            return lower_step(iter, mod)
         end
+    else
+        ex = top_ex
     end
 
     k = kind(ex)
     if !(k in KSet"toplevel module")
-        ex = expand_forms_1(iter.ctx, ex)
+        ctx1, ex = expand_forms_1(mod, ex, iter.expr_compat_mode, world)
         k = kind(ex)
     end
     if k == K"toplevel"
         push!(iter.todo, (ex, false, 1))
-        return lower_step(iter)
+        return lower_step(iter, mod)
     elseif k == K"module"
         name = ex[1]
         if kind(name) != K"Identifier"
@@ -93,7 +85,7 @@ function lower_step(iter, push_mod=nothing)
         return Core.svec(:begin_module, newmod_name, std_defs, loc)
     else
         # Non macro expansion parts of lowering
-        ctx2, ex2 = expand_forms_2(iter.ctx, ex)
+        ctx2, ex2 = expand_forms_2(ctx1, ex)
         ctx3, ex3 = resolve_scopes(ctx2, ex2)
         ctx4, ex4 = convert_closures(ctx3, ex3)
         ctx5, ex5 = linearize_ir(ctx4, ex4)
@@ -457,7 +449,7 @@ end
 @fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree;
                                 macro_world::UInt=Base.get_world_counter(),
                                 opts...)
-    iter = lower_init(ex, mod, macro_world; opts...)
+    iter = lower_init(ex; opts...)
     _eval(mod, iter)
 end
 
@@ -466,69 +458,32 @@ function eval(mod::Module, ex; opts...)
     eval(mod, expr_to_syntaxtree(ex); opts...)
 end
 
-if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604
-
 function _eval(mod, iter)
-    modules = Module[]
-    new_mod = nothing
+    modules = Module[mod]
     result = nothing
     while true
-        thunk = lower_step(iter, new_mod)::Core.SimpleVector
-        new_mod = nothing
+        thunk = lower_step(iter, modules[end])::Core.SimpleVector
         type = thunk[1]::Symbol
         if type == :done
             break
         elseif type == :begin_module
-            push!(modules, mod)
             filename = something(thunk[4].file, :none)
-            mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint,
-                                             filename::Cstring, thunk[4].line::Cint)::Module
-            new_mod = mod
-        elseif type == :end_module
-            @ccall jl_end_new_module(mod::Module)::Cvoid
-            result = mod
-            mod = pop!(modules)
-        else
-            @assert type == :thunk
-            result = Core.eval(mod, thunk[2])
-        end
-    end
-    @assert isempty(modules)
-    return result
-end
-
-else
-
-function _eval(mod, iter, new_mod=nothing)
-    in_new_mod = !isnothing(new_mod)
-    result = nothing
-    while true
-        thunk = lower_step(iter, new_mod)::Core.SimpleVector
-        new_mod = nothing
-        type = thunk[1]::Symbol
-        if type == :done
-            @assert !in_new_mod
-            break
-        elseif type == :begin_module
-            name = thunk[2]::Symbol
-            std_defs = thunk[3]
-            result = Core.eval(mod,
-                Expr(:module, std_defs, name,
-                     Expr(:block, thunk[4], Expr(:call, m->_eval(m, iter, m), name)))
-            )
+            mod = @ccall jl_begin_new_module(
+                modules[end]::Any, thunk[2]::Symbol, thunk[3]::Cint,
+                filename::Cstring, thunk[4].line::Cint)::Module
+            push!(modules, mod)
         elseif type == :end_module
-            @assert in_new_mod
-            return mod
+            @ccall jl_end_new_module(modules[end]::Module)::Cvoid
+            result = pop!(modules)
         else
             @assert type == :thunk
-            result = Core.eval(mod, thunk[2])
+            result = Core.eval(modules[end], thunk[2])
         end
     end
+    @assert length(modules) === 1
     return result
 end
 
-end
-
 """
     include(mod::Module, path::AbstractString)
 

JuliaLowering/src/macro_expansion.jl

@@ -158,25 +158,49 @@ function fixup_macro_name(ctx::MacroExpansionContext, ex::SyntaxTree)
     end
 end
 
-function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex::SyntaxTree)
-    # `ex1` might contain a nontrivial mix of scope layers so we can't just
-    # `eval()` it, as it's already been partially lowered by this point.
-    # Instead, we repeat the latter parts of `lower()` here.
-    ex1 = expand_forms_1(ctx, fixup_macro_name(ctx, ex))
-    ctx2, ex2 = expand_forms_2(ctx, ex1)
-    ctx3, ex3 = resolve_scopes(ctx2, ex2)
-    ctx4, ex4 = convert_closures(ctx3, ex3)
-    ctx5, ex5 = linearize_ir(ctx4, ex4)
+function _eval_dot(world::UInt, mod, ex::SyntaxTree)
+    if kind(ex) === K"."
+        mod = _eval_dot(world, mod, ex[1])
+        ex = ex[2]
+    end
+    kind(ex) in KSet"Identifier Symbol" && mod isa Module ?
+        Base.invoke_in_world(world, getproperty, mod, Symbol(ex.name_val)) :
+        nothing
+end
+
+# If macroexpand(ex[1]) is an identifier or dot-expression, we can simply grab
+# it from the scope layer's module in ctx.macro_world.  Otherwise, we need to
+# eval arbitrary code (which, TODO: does not use the correct world age, and it
+# isn't clear the language is meant to support this).
+function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex0::SyntaxTree)
     mod = current_layer(ctx).mod
-    expr_form = to_lowered_expr(ex5)
+    ex = fixup_macro_name(ctx, expand_forms_1(ctx, ex0))
     try
-        # Using Core.eval here fails when precompiling packages since we hit the
-        # user-facing error (in `jl_check_top_level_effect`) that warns that
-        # effects won't persist when eval-ing into a closed module.
-        # `jl_invoke_julia_macro` bypasses this by calling `jl_toplevel_eval` on
-        # the macro name.  This is fine assuming the first argument to the
-        # macrocall is effect-free.
-        ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form)
+        if kind(ex) === K"Value"
+            !(ex.value isa GlobalRef) ? ex.value :
+                Base.invoke_in_world(ctx.macro_world, getglobal,
+                                     ex.value.mod, ex.value.name)
+        elseif kind(ex) === K"Identifier"
+            layer = get(ex, :scope_layer, nothing)
+            if !isnothing(layer)
+                mod = ctx.scope_layers[layer].mod
+            end
+            Base.invoke_in_world(ctx.macro_world, getproperty,
+                                 mod, Symbol(ex.name_val))
+        elseif kind(ex) === K"." &&
+                (ed = _eval_dot(ctx.macro_world, mod, ex); !isnothing(ed))
+            ed
+        else
+            # `ex` might contain a nontrivial mix of scope layers so we can't
+            # just `eval()` it, as it's already been partially lowered by this
+            # point.  Instead, we repeat the latter parts of `lower()` here.
+            ctx2, ex2 = expand_forms_2(ctx, ex)
+            ctx3, ex3 = resolve_scopes(ctx2, ex2)
+            ctx4, ex4 = convert_closures(ctx3, ex3)
+            ctx5, ex5 = linearize_ir(ctx4, ex4)
+            expr_form = to_lowered_expr(ex5)
+            ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form)
+        end
     catch err
         throw(MacroExpansionError(mctx, ex, "Macro not found", :all, err))
     end

JuliaLowering/test/macros.jl

@@ -515,5 +515,27 @@ end
 """)
 code = JuliaLowering.include_string(test_mod, """Mod1.@indirect_MODULE()""")
 @test JuliaLowering.eval(test_mod, code) === test_mod # !== test_mod.Mod1
+# the lowering/eval iterator needs to expand in the correct world age (currently
+# the only way to hit this from user code is macros producing toplevel)
+
+@testset "macros defining macros" begin
+    @eval test_mod macro make_and_use_macro_toplevel()
+        Expr(:toplevel,
+             esc(:(macro from_toplevel_expansion()
+                   :(123)
+               end)),
+             esc(:(@from_toplevel_expansion())))
+    end
+
+    @test JuliaLowering.include_string(
+        test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=true) === 123
+
+    if isdefined(test_mod, Symbol("@from_toplevel_expansion"))
+        Base.delete_binding(test_mod, Symbol("@from_toplevel_expansion"))
+    end
+
+    @test JuliaLowering.include_string(
+        test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=false) === 123
+end
 
 end

JuliaLowering/test/macros_ir.jl

@@ -147,7 +147,7 @@ _never_exist = @m_not_exist 42
 #---------------------
 MacroExpansionError while expanding @m_not_exist in module Main.TestMod:
 _never_exist = @m_not_exist 42
-#              └──────────┘ ── Macro not found
+#               └─────────┘ ── Macro not found
 Caused by:
 UndefVarError: `@m_not_exist` not defined in `Main.TestMod`
 Suggestion: check for spelling errors or missing imports.