Skip to content

Bypass SyntaxNode in JuliaLowering; fix kw bug #60162

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Bypass SyntaxNode in JuliaLowering; fix kw bug #60162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f14996d
Bypass `SyntaxNode` in JuliaLowering; fix `kw` bug
mlechu Nov 17, 2025
f962f9e
Fixes I misplaced
mlechu Nov 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 43 additions & 29 deletions JuliaLowering/src/compat.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@ end
graph = syntax_graph(ctx)
toplevel_src = if isnothing(lnn)
# Provenance sinkhole for all nodes until we hit a linenode
dummy_src = SourceRef(
SourceFile("No source for expression"),
1, JS.GreenNode(K"None", 0))
dummy_src = SourceRef(SourceFile("No source for expression"), 1, 0)
_insert_tree_node(graph, K"None", dummy_src)
else
lnn
Expand All @@ -46,18 +44,24 @@ end
return out
end

function _expr_replace!(@nospecialize(e), replace_pred::Function, replacer!::Function,
function _expr_replace(@nospecialize(e), replace_pred::Function, replacer::Function,
recurse_pred=(@nospecialize e)->true)
if replace_pred(e)
replacer!(e)
end
if e isa Expr && recurse_pred(e)
for a in e.args
_expr_replace!(a, replace_pred, replacer!, recurse_pred)
end
replacer(e)
elseif e isa Expr && recurse_pred(e)
Expr(e.head, [_expr_replace(a, replace_pred, replacer, recurse_pred) for a in e.args]...)
else
e
end
end

function _eq_to_kw(@nospecialize(e))
_expr_replace(e,
(x)->x isa Expr && x.head === :(=),
(x)->Expr(:kw, x.args...),
(x)->x.head in (:escape, :var"hygienic-scope"))
end

function _to_iterspec(exs::Vector, is_generator::Bool)
if length(exs) === 1 && exs[1].head === :filter
@assert length(exs[1].args) >= 2
Expand Down Expand Up @@ -85,19 +89,25 @@ Parameters are expected to be at `e.args[pos]`.
e.g. orderings of (a,b,c;d;e;f):
Expr: (tuple (parameters (parameters (parameters f) e) d) a b c)
SyntaxTree: (tuple a b c (parameters d) (parameters e) (parameters f))

`ensure_kw` converts `=` to `kw` within parameters blocks (needed for ref,
curly, vect, and braces).
"""
function collect_expr_parameters(e::Expr, pos::Int)
function collect_expr_parameters(e::Expr, pos::Int, ensure_kw::Bool)
params = expr_parameters(e, pos)
isnothing(params) && return copy(e.args)
args = Any[e.args[1:pos-1]..., e.args[pos+1:end]...]
return _flatten_params!(args, params)
return _flatten_params!(args, params, ensure_kw)
end
function _flatten_params!(out::Vector{Any}, params::Expr)
function _flatten_params!(out::Vector{Any}, params::Expr, ensure_kw)
p,p_esc = unwrap_esc(params)
p1 = expr_parameters(p, 1)
if !isnothing(p1)
push!(out, p_esc(Expr(:parameters, p.args[2:end]...)))
_flatten_params!(out, p_esc(p1))
p_args = ensure_kw ? map(_eq_to_kw, p.args[2:end]) : p.args[2:end]
push!(out, p_esc(Expr(:parameters, p_args...)))
_flatten_params!(out, p_esc(p1), ensure_kw)
elseif ensure_kw && params isa Expr && params.head === :parameters
push!(out, Expr(:parameters, map(_eq_to_kw, params.args)...))
else
push!(out, params::Any)
end
Expand Down Expand Up @@ -262,7 +272,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
elseif e.head === :macrocall
@assert nargs >= 2
a1,a1_esc = unwrap_esc(e.args[1])
child_exprs = collect_expr_parameters(e, 3)
child_exprs = collect_expr_parameters(e, 3, false)
if child_exprs[2] isa LineNumberNode
src = child_exprs[2]
end
Expand Down Expand Up @@ -297,7 +307,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
a2, a2_esc = unwrap_esc(e.args[2])
if a2 isa Expr && a2.head === :tuple
st_k = K"dotcall"
tuple_exprs = collect_expr_parameters(a2_esc(a2), 1)
tuple_exprs = collect_expr_parameters(a2_esc(a2), 1, false)
child_exprs = pushfirst!(tuple_exprs, e.args[1])
elseif a2 isa QuoteNode
child_exprs[2] = a2_esc(a2.value)
Expand All @@ -311,10 +321,16 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
if !(e2 isa Expr && e2.head === :braces)
child_exprs = Any[e.args[1], Expr(:braces, e.args[2:end]...)]
end
elseif e.head in (:tuple, :vect, :braces)
child_exprs = collect_expr_parameters(e, 1)
elseif e.head in (:curly, :ref)
child_exprs = collect_expr_parameters(e, 2)
elseif e.head === :ref
child_exprs = collect_expr_parameters(e, 2, true)
elseif e.head === :curly
child_exprs = collect_expr_parameters(e, 2, true)
map!(_eq_to_kw, child_exprs[2:end])
elseif e.head in (:vect, :braces)
child_exprs = collect_expr_parameters(e, 1, true)
elseif e.head === :tuple
child_exprs = collect_expr_parameters(e, 1, false)
map!(_eq_to_kw, child_exprs)
elseif e.head === :try
child_exprs = Any[e.args[1]]
# Expr:
Expand Down Expand Up @@ -368,15 +384,15 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
a isa LineNumberNode && continue
a isa Expr && a.head === :(=) ? push!(lam_eqs, a) : push!(lam_args, a)
end
!isempty(lam_eqs) && push!(lam_args, Expr(:parameters, lam_eqs...))
!isempty(lam_eqs) && push!(lam_args, Expr(:parameters, map(_eq_to_kw, lam_eqs)...))
child_exprs[1] = a1_esc(Expr(:tuple, lam_args...))
elseif !(a1 isa Expr && (a1.head in (:tuple, :where)))
child_exprs[1] = a1_esc(Expr(:tuple, a1))
end
src = maybe_extract_lnn(e.args[2], src)
child_exprs[2] = maybe_unwrap_arg(e.args[2])
elseif e.head === :call
child_exprs = collect_expr_parameters(e, 2)
child_exprs = collect_expr_parameters(e, 2, false)
a1,a1_esc = unwrap_esc(child_exprs[1])
if a1 isa Symbol
a1s = string(a1)
Expand Down Expand Up @@ -408,7 +424,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
# (do (call f args...) (-> (tuple lam_args...) (block ...)))
# SyntaxTree:
# (call f args... (do (tuple lam_args...) (block ...)))
callargs = collect_expr_parameters(e.args[1], 2)
callargs = collect_expr_parameters(e.args[1], 2, false)
if e.args[1].head === :macrocall
st_k = K"macrocall"
if callargs[2] isa LineNumberNode
Expand All @@ -433,11 +449,9 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
child_exprs = child_exprs[2:end]
# TODO handle docstrings after refactor
elseif (e.head === :using || e.head === :import)
_expr_replace!(e,
(e)->(e isa Expr && e.head === :.),
(e)->(e.head = :importpath))
elseif e.head === :kw
st_k = K"="
e2 = _expr_replace(e, (e)->(e isa Expr && e.head === :.),
(e)->Expr(:importpath, e.args...))
child_exprs = e2.args
elseif e.head in (:local, :global) && nargs > 1
# Possible normalization
# child_exprs = Any[Expr(:tuple, child_exprs...)]
Expand Down
28 changes: 17 additions & 11 deletions JuliaLowering/src/desugaring.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ function check_no_parameters(ex::SyntaxTree, msg)
end

function check_no_assignment(exs, msg="misplaced assignment statement in `[ ... ]`")
i = findfirst(kind(e) == K"=" for e in exs)
i = findfirst(kind(e) == K"=" || kind(e) == K"kw" for e in exs)
if !isnothing(i)
throw(LoweringError(exs[i], msg))
end
Expand Down Expand Up @@ -1622,7 +1622,7 @@ function expand_named_tuple(ctx, ex, kws;
# x ==> x = x
name = to_symbol(ctx, kw)
value = kw
elseif k == K"="
elseif k == K"kw"
# x = a
if kind(kw[1]) != K"Identifier" && kind(kw[1]) != K"Placeholder"
throw(LoweringError(kw[1], "invalid $field_name name"))
Expand Down Expand Up @@ -1864,7 +1864,7 @@ function remove_kw_args!(ctx, args::SyntaxList)
for i in 1:length(args)
arg = args[i]
k = kind(arg)
if k == K"="
if k == K"kw"
if isnothing(kws)
kws = SyntaxList(ctx)
end
Expand Down Expand Up @@ -2263,7 +2263,7 @@ end
function expand_function_arg(ctx, body_stmts, arg, is_last_arg, is_kw)
ex = arg

if kind(ex) == K"="
if kind(ex) == K"kw"
default = ex[2]
ex = ex[1]
else
Expand Down Expand Up @@ -2824,8 +2824,10 @@ function keyword_function_defs(ctx, srcref, callex_srcref, name_str, typevar_nam
kwcall_body_tail
]
else
scope_nest(ctx, kw_names, kw_values, kwcall_body_tail)
scope_nest(ctx, has_kw_slurp ? kw_names[1:end-1] : kw_names,
kw_values, kwcall_body_tail)
end

main_kwcall_typevars = trim_used_typevars(ctx, kwcall_arg_types, typevar_names, typevar_stmts)
push!(kwcall_method_defs,
method_def_expr(ctx, srcref, callex_srcref, kwcall_mtable,
Expand Down Expand Up @@ -3212,9 +3214,13 @@ function expand_arrow_arglist(ctx, arglist, arrowname)
# https://github.com/JuliaLang/JuliaSyntax.jl/pull/522
if k == K"block"
@chk numchildren(arglist) == 2
kw = kind(arglist[2]) === K"=" ?
@ast(ctx, arglist[2], [K"kw" children(arglist[2])...]) :
arglist[2]

arglist = @ast ctx arglist [K"tuple"
arglist[1]
[K"parameters" arglist[2]]
[K"parameters" kw]
]
elseif k != K"tuple"
arglist = @ast ctx arglist [K"tuple"
Expand Down Expand Up @@ -3795,7 +3801,7 @@ function _rewrite_ctor_new_calls(ctx, ex, struct_name, global_struct_name, ctor_
)
end
# Rewrite a call to new()
kw_arg_i = findfirst(e->(k = kind(e); k == K"=" || k == K"parameters"), children(ex))
kw_arg_i = findfirst(e->(k = kind(e); k == K"kw" || k == K"parameters"), children(ex))
if !isnothing(kw_arg_i)
throw(LoweringError(ex[kw_arg_i], "`new` does not accept keyword arguments"))
end
Expand Down Expand Up @@ -4113,7 +4119,7 @@ function expand_struct_def(ctx, ex, docs)
struct_name
isnothing(docs) ? nothing_(ctx, ex) : docs[1]
::K"SourceLocation"(ex)
[K"="
[K"kw"
"field_docs"::K"Identifier"
[K"call" "svec"::K"core" field_docs...]
]
Expand Down Expand Up @@ -4160,7 +4166,7 @@ end
function expand_curly(ctx, ex)
@assert kind(ex) == K"curly"
check_no_parameters(ex, "unexpected semicolon in type parameter list")
check_no_assignment(children(ex), "misplace assignment in type parameter list")
check_no_assignment(children(ex), "misplaced assignment in type parameter list")

typevar_stmts = SyntaxList(ctx)
type_args = SyntaxList(ctx)
Expand Down Expand Up @@ -4479,7 +4485,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
throw(LoweringError(ex[end], "unexpected semicolon in tuple - use `,` to separate tuple elements"))
end
expand_forms_2(ctx, expand_named_tuple(ctx, ex, children(ex[1])))
elseif any_assignment(children(ex))
elseif any(kind(c) == K"kw" for c in children(ex))
expand_forms_2(ctx, expand_named_tuple(ctx, ex, children(ex)))
else
expand_forms_2(ctx, @ast ctx ex [K"call"
Expand Down Expand Up @@ -4524,7 +4530,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
ctx.mod ::K"Value"
[K"inert" ex]
[K"parameters"
[K"="
[K"kw"
"expr_compat_mode"::K"Identifier"
ctx.expr_compat_mode::K"Bool"
]
Expand Down
4 changes: 2 additions & 2 deletions JuliaLowering/src/hooks.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ function core_lowering_hook(@nospecialize(code), mod::Module,
file = file isa Ptr{UInt8} ? unsafe_string(file) : file
line = !(line isa Int) ? Int(line) : line

local st0 = nothing
local st0, st1 = nothing, nothing
try
st0 = code isa Expr ? expr_to_syntaxtree(code, LineNumberNode(line, file)) : code
if kind(st0) in KSet"toplevel module"
Expand All @@ -32,7 +32,7 @@ function core_lowering_hook(@nospecialize(code), mod::Module,
ex = to_lowered_expr(st5)
return Core.svec(ex, st5, ctx5)
catch exc
@info("JuliaLowering threw given input:", code=code, st0=st0, file=file, line=line, mod=mod)
@info("JuliaLowering threw given input:", code=code, st0=st0, st1=st1, file=file, line=line, mod=mod)
rethrow(exc)

# TODO: Re-enable flisp fallback once we're done collecting errors
Expand Down
5 changes: 3 additions & 2 deletions JuliaLowering/src/macro_expansion.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,8 +436,9 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
# TODO: Upstream should set a general flag for detecting parenthesized
# expressions so we don't need to dig into `green_tree` here. Ugh!
plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) &&
kind(ex[1]) == K"Identifier" &&
(sr = sourceref(ex); sr isa SourceRef && kind(sr.green_tree[2]) != K"parens")
kind(ex[1]) == K"Identifier" && (
prov = flattened_provenance(ex);
length(prov) >= 1 && kind(prov[end][end]) != K"parens")
if plain_symbol
# As a compromise for compatibility, we treat non-parenthesized
# colon quoted identifiers like `:x` as plain Symbol literals
Expand Down
Loading