r
/
Runic.jl
mirror of https://github.com/fredrikekre/Runic.jl
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Better syntax tree normalization 

This patch changes the parsed syntax tree normalization with the goal of
simplifying the formatting code. JuliaSyntax is not very consistent with
where whitespace ends up which make formatting tricky since you always
have to "peek" into the next node (sometimes multiple levels) to check
if the next leaf is e.g. whitespace. This patch reworks the
normalization to "bubble up" whitespace so that all nodes start and end
with a non-whitespace node.

For example, given `a + b * c`, the output from JuliaSyntax is:

```
[call]
  Identifier
  Whitespace
  +
  [call]
    Whitespace
    Identifier
    Whitespace
    *
    Whitespace
    Identifier
```

and now after normalization the leading whitespace of the `*`-call is
bubbled up:

```
[call]
  Identifier
  Whitespace
  +
  Whitespace
  [call]
    Identifier
    Whitespace
    *
    Whitespace
    Identifier
```

As seen from the diff, this make it possible to remove a lot of
complicated code that were necessary to handle the random whitespace
placement. In particular, there is no need to peek into the nodes to
check for whitespace.

As a bonus, this fixes two issues:
 - `global\n\n x = 1` would previously not indent correctly because we
   had to double-peek (first into the `=` node, then into the LHS node).
 - `runic: (off|on) toggling within array literals. Previously the
   toggle comments would end up inside the `row` nodes so in the tree
   they weren't on the same "level" which is required for the toggling.
pull/101/head
Fredrik Ekre 1 year ago
parent
91fff169a1
commit
041fcbfd82
No known key found for this signature in database
GPG Key ID: DE82E6D5E364C0A2
3 changed files with 293 additions and 449 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 179
      src/chisels.jl
  2. 459
      src/runestone.jl
  3. 44
      test/runtests.jl

179
src/chisels.jl
Unescape View File

@ -77,81 +77,97 @@ function stringify_flags(node::Node)
return String(take!(io)) return String(take!(io))
end end
function pop_if_whitespace!(popf!::F, node) where {F}
if !is_leaf(node) && begin
kids = verified_kids(node)
length(kids) > 0 &&
JuliaSyntax.is_whitespace(popf! === popfirst! ? kids[1] : kids[end])
end
ws = popf!(kids)
@assert JuliaSyntax.is_whitespace(ws)
node′ = make_node(node, kids)
@assert span(node) == span(node′) + span(ws)
return node′, ws
end
return nothing, nothing
end
# The parser is somewhat inconsistent(?) with where e.g. whitespace nodes end up so in order # The parser is somewhat inconsistent(?) with where e.g. whitespace nodes end up so in order
# to simplify the formatting code we normalize some things. # to simplify the formatting code we normalize some things.
function normalize_tree!(node) function normalize_tree!(node)
is_leaf(node) && return is_leaf(node) && return node
kids = verified_kids(node) kids = verified_kids(node)
# Move standalone K"NewlineWs" (and other??) nodes from between the var block and the # For K"let" the token (K"NewlineWs" or K";") separating the vars block from the
# body block in K"let" nodes. # body ends up in between the two K"block" nodes. Move it into the body block.
# Note that this happens before the whitespace into block normalization below because
# for let we want to move it to the subsequent block instead.
if kind(node) === K"let" if kind(node) === K"let"
varsidx = findfirst(x -> kind(x) === K"block", kids)::Int varsidx = findfirst(x -> kind(x) === K"block", kids)::Int
bodyidx = findnext(x -> kind(x) === K"block", kids, varsidx + 1)::Int blockidx = findnext(x -> kind(x) === K"block", kids, varsidx + 1)::Int
r = (varsidx + 1):(bodyidx - 1) while blockidx != varsidx + 1
if length(r) > 0 grandkid = popat!(kids, blockidx - 1)
items = kids[r] blockidx -= 1
deleteat!(kids, r) block = kids[blockidx]
bodyidx -= length(r) pushfirst!(verified_kids(block), grandkid)
body = kids[bodyidx] kids[blockidx] = make_node(block, verified_kids(block))
prepend!(verified_kids(body), items)
kids[bodyidx] = make_node(body, verified_kids(body))
end end
@assert span(node) == mapreduce(span, +, kids; init = 0)
end end
# Normalize K"Whitespace" nodes in blocks. For example in `if x y end` the space will be # Normalize K"Whitespace" nodes in blocks. For example in `if x y end` the space
# outside the block just before the K"end" node, but in `if x\ny\nend` the K"NewlineWs" # will be outside the block just before the K"end" node, but in `if x\ny\nend` the
# will end up inside the block. # K"NewlineWs" will end up inside the block.
if kind(node) in KSet"function if elseif for while try do macro module baremodule let struct module" if kind(node) in KSet"function if elseif for while try do macro module baremodule let struct module"
blockidx = findfirst(x -> kind(x) === K"block", kids) blockidx = 0
while blockidx !== nothing && blockidx < length(kids) while let
if kind(kids[blockidx + 1]) !== K"Whitespace"
# TODO: This repeats the computation below...
blockidx = findnext(x -> kind(x) === K"block", kids, blockidx + 1) blockidx = findnext(x -> kind(x) === K"block", kids, blockidx + 1)
blockidx !== nothing && blockidx < length(kids)
end
if kind(kids[blockidx + 1]) !== K"Whitespace"
continue continue
end end
# Pop the ws and push it into the block instead
block = kids[blockidx] block = kids[blockidx]
blockkids = verified_kids(block) blockkids = verified_kids(block)
@assert !(kind(blockkids[end]) in KSet"Whitespace NewlineWs") @assert !(kind(blockkids[end]) in KSet"Whitespace NewlineWs")
push!(blockkids, popat!(kids, blockidx + 1)) push!(blockkids, popat!(kids, blockidx + 1))
# Remake the block to recompute the span
kids[blockidx] = make_node(block, blockkids) kids[blockidx] = make_node(block, blockkids)
# Find next block
blockidx = findnext(x -> kind(x) === K"block", kids, blockidx + 1)
end end
@assert span(node) == mapreduce(span, +, kids; init = 0)
end end
# Normalize K"Whitespace" nodes in if-elseif-else chains where the node needs to move # Normalize K"Whitespace" nodes in if-elseif-else chains where the node needs to
# many steps into the last else block... # move many steps into the last else block.
if kind(node) === K"if" if kind(node) === K"if"
elseifidx = findfirst(x -> kind(x) === K"elseif", kids) elseifidx = findfirst(x -> kind(x) === K"elseif", kids)
if elseifidx !== nothing if elseifidx !== nothing
endidx = findnext(x -> kind(x) === K"end", kids, elseifidx + 1)::Int endidx = findnext(x -> kind(x) === K"end", kids, elseifidx + 1)::Int
if elseifidx + 2 == endidx && kind(kids[elseifidx + 1]) === K"Whitespace" if elseifidx + 2 == endidx && kind(kids[elseifidx + 1]) === K"Whitespace"
# Pop the ws and push it into the last block instead
ws = popat!(kids, elseifidx + 1) ws = popat!(kids, elseifidx + 1)
elseifnode = insert_into_last_else_block(kids[elseifidx], ws) elseifnode = insert_into_last_else_block(kids[elseifidx], ws)
@assert elseifnode !== nothing @assert elseifnode !== nothing
kids[elseifidx] = elseifnode kids[elseifidx] = elseifnode
end end
end end
@assert span(node) == mapreduce(span, +, kids; init = 0)
end end
# Normalize K"Whitespace" nodes in try-catch-finally-else # Normalize K"Whitespace" nodes in try-catch-finally-else
if kind(node) === K"try" if kind(node) === K"try"
catchidx = findfirst(x -> kind(x) in KSet"catch finally else", kids) catchidx = 0
while catchidx !== nothing while let
catchidx = findnext(
x -> kind(x) in KSet"catch finally else", kids, catchidx + 1
)
catchidx !== nothing
end
@assert catchidx + 1 <= length(kids)
if kind(kids[catchidx + 1]) === K"Whitespace" if kind(kids[catchidx + 1]) === K"Whitespace"
ws = popat!(kids, catchidx + 1) ws = popat!(kids, catchidx + 1)
catchnode = insert_into_last_catchlike_block(kids[catchidx], ws) catchnode = insert_into_last_catchlike_block(kids[catchidx], ws)
@assert catchnode !== nothing @assert catchnode !== nothing
kids[catchidx] = catchnode kids[catchidx] = catchnode
end end
catchidx = findnext(x -> kind(x) in KSet"catch finally else", kids, catchidx + 1)
end end
@assert span(node) == mapreduce(span, +, kids; init = 0)
end end
# Normalize K"NewlineWs" nodes in empty do-blocks # Normalize K"NewlineWs" nodes in empty do-blocks
@ -161,7 +177,7 @@ function normalize_tree!(node)
@assert tupleidx + 1 == blockidx @assert tupleidx + 1 == blockidx
tuple = kids[tupleidx] tuple = kids[tupleidx]
tuplekids = verified_kids(tuple) tuplekids = verified_kids(tuple)
if kind(tuplekids[end]) === K"NewlineWs" if length(tuplekids) > 0 && kind(tuplekids[end]) === K"NewlineWs"
# If the tuple ends with a K"NewlineWs" node we move it into the block # If the tuple ends with a K"NewlineWs" node we move it into the block
block = kids[blockidx] block = kids[blockidx]
blockkids = verified_kids(block) blockkids = verified_kids(block)
@ -173,13 +189,63 @@ function normalize_tree!(node)
kids[tupleidx] = make_node(tuple, tuplekids) kids[tupleidx] = make_node(tuple, tuplekids)
kids[blockidx] = make_node(block, blockkids) kids[blockidx] = make_node(block, blockkids)
end end
@assert span(node) == mapreduce(span, +, kids; init = 0)
end end
@assert span(node) == mapreduce(span, +, kids; init = 0)
@assert kids === verified_kids(node) @assert kids === verified_kids(node)
for kid in kids
ksp = span(kid) # Loop over the kids to bubble up whitespace
normalize_tree!(kid) i = 0
@assert span(kid) == ksp while i < lastindex(kids)
i += 1
kid = kids[i]
# Recursively normalize the kid to bubble up whitespace to the front. This should
# never change the node kind or the span, just internally reorganize the tokens.
kid′ = normalize_tree!(kid)
@assert kid === kid′
is_leaf(kid) ||
@assert span(kid) == mapreduce(span, +, verified_kids(kid); init = 0)
# If the kid is a K"block" we don't bubble up whitespace since it is more convenient
# to keep it inside the block (see e.g. `indent_block` which relies on this for
# making sure blocks starts/ends with a K"NewlineWs").
if kind(kid) === K"block"
# If this is a begin-block (or quote-block) we can still bubble up the
# whitespace since the begin and end tokens are inside of the block. I don't
# think this currently happens though so for now we just assert.
if is_begin_block(kid)
@assert kind(verified_kids(kid)[1]) === K"begin"
@assert kind(verified_kids(kid)[end]) === K"end"
elseif is_begin_block(kid, K"quote")
@assert kind(verified_kids(kid)[1]) === K"quote"
@assert kind(verified_kids(kid)[end]) === K"end"
elseif is_paren_block(kid)
@assert kind(verified_kids(kid)[1]) === K"("
@assert kind(verified_kids(kid)[end]) === K")"
end
continue
end
# Extract any leading whitespace and insert it before the node.
while ((kid′, ws) = pop_if_whitespace!(popfirst!, kid); ws !== nothing)
@assert kid′ !== nothing
@assert kind(kid′) === kind(kid)
kids[i] = kid = kid′
insert!(kids, i, ws)
i += 1
end
# Extract any trailing whitespace and insert it after the node.
while ((kid′, ws) = pop_if_whitespace!(pop!, kid); ws !== nothing)
# Trailing whitespace have only been seen in the wild in these three cases
@assert kind(kid) in KSet"row nrow parameters"
@assert kid′ !== nothing
@assert kind(kid′) === kind(kid)
kids[i] = kid = kid′
insert!(kids, i + 1, ws)
end
end end
# We only move around things inside this node so the span should be unchanged # We only move around things inside this node so the span should be unchanged
@assert span(node) == mapreduce(span, +, kids; init = 0) @assert span(node) == mapreduce(span, +, kids; init = 0)
@ -378,21 +444,6 @@ function nth_leaf(node::Node, nth::Int, n_seen::Int)
end end
end end
function second_leaf(node::Node)
return nth_leaf(node, 2)
end
# TODO: This should probably be merged with kmatch or something...
function peek_leafs(node::Node, leaf_kinds::Tuple)
for (i, leaf_kind) in pairs(leaf_kinds)
ith = nth_leaf(node, i)
if ith === nothing || kind(ith) !== leaf_kind
return false
end
end
return true
end
# Return number of non-whitespace kids, basically the length the equivalent # Return number of non-whitespace kids, basically the length the equivalent
# (expr::Expr).args # (expr::Expr).args
function meta_nargs(node::Node) function meta_nargs(node::Node)
@ -482,27 +533,6 @@ function second_last_leaf(node::Node, n_seen::Int)
return nothing, n_seen return nothing, n_seen
end end
function has_newline_after_non_whitespace(node::Node)
if is_leaf(node)
@assert kind(node) !== K"NewlineWs"
return false
else
kids = verified_kids(node)
idx = findlast(!JuliaSyntax.is_whitespace, kids)
if idx === nothing
unreachable()
# Everything is whitespace...
return any(x -> kind(x) === K"NewlineWs", kids)
end
return any(x -> kind(x) === K"NewlineWs", kids[(idx + 1):end]) ||
has_newline_after_non_whitespace(kids[idx])
# if idx === nothing
# # All is whitespace, check if any of the kids is a newline
# return any(x -> kind(x) === K"NewlineWs", kids)
# end
end
end
function is_assignment(node::Node) function is_assignment(node::Node)
return JuliaSyntax.is_prec_assignment(node) return JuliaSyntax.is_prec_assignment(node)
end end
@ -622,9 +652,10 @@ function first_non_whitespace_kid(node::Node)
return kids[idx] return kids[idx]
end end
function is_begin_block(node::Node) function is_begin_block(node::Node, token = K"begin")
@assert token in KSet"begin quote"
return kind(node) === K"block" && length(verified_kids(node)) > 0 && return kind(node) === K"block" && length(verified_kids(node)) > 0 &&
kind(verified_kids(node)[1]) === K"begin" kind(verified_kids(node)[1]) === token
end end
function is_paren_block(node::Node) function is_paren_block(node::Node)

459
src/runestone.jl
Unescape View File

@ -178,38 +178,8 @@ function spaces_around_x(ctx::Context, node::Node, is_x::F, n_leaves_per_x::Int
replace_bytes!(ctx, " ", span(kid)) replace_bytes!(ctx, " ", span(kid))
accept_node!(ctx, ws) accept_node!(ctx, ws)
looking_for_whitespace = false looking_for_whitespace = false
elseif !is_leaf(kid) && kind(first_leaf(kid)) === K"Whitespace"
# Whitespace found at the beginning of next kid.
kid_ws = first_leaf(kid)
looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace"
@assert !is_x(kid)::Bool
looking_for_x = true
if span(kid_ws) == 1
# Accept the node
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
else
# Replace the whitespace node of the kid
kid′ = replace_first_leaf(kid, ws)
@assert span(kid′) == span(kid) - span(kid_ws) + 1
replace_bytes!(ctx, " ", span(kid_ws))
accept_node!(ctx, kid′)
any_changes = true
if kids′ === kids
kids′ = kids[1:(i - 1)]
end
push!(kids′, kid′)
end
elseif !is_leaf(kid) && kind(first_leaf(kid)) === K"NewlineWs"
# NewlineWs have to be accepted as is
# @info " ... kids first leaf is NewlineWs I'll take it"
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace"
@assert !is_x(kid)::Bool
looking_for_x = true
else else
# @info " ... no whitespace, inserting" kind(kid) @assert !(first_leaf(kid) in KSet"Whitespace NewlineWs")
# Not a whitespace node, insert one # Not a whitespace node, insert one
any_changes = true any_changes = true
if kids′ === kids if kids′ === kids
@ -399,7 +369,7 @@ function spaces_in_listlike(ctx::Context, node::Node)
elseif n_items > 0 elseif n_items > 0
require_trailing_comma = any( require_trailing_comma = any(
x -> kind(x) === K"NewlineWs", @view(kids[(last_item_idx + 1):(closing_leaf_idx - 1)]) x -> kind(x) === K"NewlineWs", @view(kids[(last_item_idx + 1):(closing_leaf_idx - 1)])
) || has_newline_after_non_whitespace(kids[last_item_idx]) )
end end
# Helper to compute the new state after a given item # Helper to compute the new state after a given item
@ -440,14 +410,8 @@ function spaces_in_listlike(ctx::Context, node::Node)
for i in (opening_leaf_idx + 1):(closing_leaf_idx - 1) for i in (opening_leaf_idx + 1):(closing_leaf_idx - 1)
kid′ = kids[i] kid′ = kids[i]
this_kid_changed = false this_kid_changed = false
first_item_in_implicit_tuple = implicit_tuple && i == opening_leaf_idx + 1
if state === :expect_item if state === :expect_item
if first_item_in_implicit_tuple && kind(kid′) === K"Whitespace" && peek(i) !== K"Comment" if kind(kid′) === K"Whitespace" && peek(i) !== K"Comment"
# Not allowed to touch this one I think?
accept_node!(ctx, kid′)
any_kid_changed && push!(kids′, kid′)
elseif kind(kid′) === K"Whitespace" && peek(i) !== K"Comment"
@assert !first_item_in_implicit_tuple # Unreachable?
# Delete whitespace unless followed by a comment # Delete whitespace unless followed by a comment
replace_bytes!(ctx, "", span(kid′)) replace_bytes!(ctx, "", span(kid′))
this_kid_changed = true this_kid_changed = true
@ -456,37 +420,18 @@ function spaces_in_listlike(ctx::Context, node::Node)
end end
elseif kind(kid′) === K"NewlineWs" || elseif kind(kid′) === K"NewlineWs" ||
(kind(kid′) === K"Whitespace" && peek(i) === K"Comment") (kind(kid′) === K"Whitespace" && peek(i) === K"Comment")
@assert !first_item_in_implicit_tuple # Unreachable?
# Newline here can happen if this kid is just after the opening leaf or if # Newline here can happen if this kid is just after the opening leaf or if
# there is an empty line between items. No state change. # there is an empty line between items. No state change.
accept_node!(ctx, kid′) accept_node!(ctx, kid′)
any_kid_changed && push!(kids′, kid′) any_kid_changed && push!(kids′, kid′)
elseif kind(kid′) === K"Comment" elseif kind(kid′) === K"Comment"
@assert !first_item_in_implicit_tuple # Unreachable?
accept_node!(ctx, kid′) accept_node!(ctx, kid′)
any_kid_changed && push!(kids′, kid′) any_kid_changed && push!(kids′, kid′)
state = :expect_space # To ensure space after the comment state = :expect_space # To ensure space after the comment
else else
# This is an item (probably?). # This is an item (probably?).
# Make sure it doesn't have leading or trailing whitespace. @assert !JuliaSyntax.is_whitespace(first_leaf(kid′))
if kind(first_leaf(kid′)) === K"Whitespace" && kind(second_leaf(kid′)) !== K"Comment" && !first_item_in_implicit_tuple @assert !JuliaSyntax.is_whitespace(last_leaf(kid′))
# Delete the whitespace leaf
kid_ws = first_leaf(kid′)
replace_bytes!(ctx, "", span(kid_ws))
kid′ = replace_first_leaf(kid′, nullnode)
this_kid_changed = true
end
if kind(last_leaf(kid′)) === K"Whitespace"
# Delete the whitespace leaf
kid_ws = last_leaf(kid′)
let pos = position(ctx.fmt_io)
seek(ctx.fmt_io, pos + span(kid′) - span(kid_ws))
replace_bytes!(ctx, "", span(kid_ws))
seek(ctx.fmt_io, pos)
end
kid′ = replace_last_leaf(kid′, nullnode)
this_kid_changed = true
end
if kind(kid′) === K"parameters" && require_trailing_comma && if kind(kid′) === K"parameters" && require_trailing_comma &&
i == last_item_idx && !has_tag(kid′, TAG_TRAILING_COMMA) i == last_item_idx && !has_tag(kid′, TAG_TRAILING_COMMA)
# Tag the node to require a trailing comma # Tag the node to require a trailing comma
@ -504,22 +449,15 @@ function spaces_in_listlike(ctx::Context, node::Node)
x -> !(JuliaSyntax.is_whitespace(x) || kind(x) in KSet", ;"), x -> !(JuliaSyntax.is_whitespace(x) || kind(x) in KSet", ;"),
verified_kids(kid′) verified_kids(kid′)
) == 0 ) == 0
# If kid is K"parameters" without items and we don't want a trailing # If kid is K"parameters" without items and we don't want the
# comma/semicolon we need to eat any whitespace kids (e.g. comments) # K"parameters" node for the trailing semicolon we drop the entire node
grandkids = verified_kids(kid′) grandkids = verified_kids(kid′)
semi_idx = 1 @assert length(grandkids) == 1 && kind(grandkids[1]) === K";"
@assert kind(grandkids[semi_idx]) === K";"
ws_idx = something(findnext(x -> kind(x) !== K"Whitespace", grandkids, semi_idx + 1), lastindex(grandkids) + 1)
any_kid_changed = true any_kid_changed = true
if kids′ === kids if kids′ === kids
kids′ = kids[1:(i - 1)] kids′ = kids[1:(i - 1)]
end end
replace_bytes!(ctx, "", mapreduce(span, +, grandkids[1:(ws_idx - 1)]; init = 0)) replace_bytes!(ctx, "", span(kid′))
for j in ws_idx:length(grandkids)
grandkid = grandkids[j]
accept_node!(ctx, grandkid)
push!(kids′, grandkid)
end
else else
# Kid is now acceptable # Kid is now acceptable
any_kid_changed |= this_kid_changed any_kid_changed |= this_kid_changed
@ -589,16 +527,7 @@ function spaces_in_listlike(ctx::Context, node::Node)
elseif kind(kid′) === K"parameters" elseif kind(kid′) === K"parameters"
# Note that some of these are not valid Julia syntax still parse # Note that some of these are not valid Julia syntax still parse
@assert kind(node) in KSet"call dotcall macrocall curly tuple vect ref braces" @assert kind(node) in KSet"call dotcall macrocall curly tuple vect ref braces"
if kind(first_leaf(kid′)) === K"Whitespace" @assert !JuliaSyntax.is_whitespace(first_leaf(kid′))
# Delete the whitespace leaf
kid_ws = first_leaf(kid′)
replace_bytes!(ctx, "", span(kid_ws))
kid′ = replace_first_leaf(kid′, nullnode)
this_kid_changed = true
# if kids′ === kids
# kids′ = kids[1:i - 1]
# end
end
if require_trailing_comma && !has_tag(kid′, TAG_TRAILING_COMMA) if require_trailing_comma && !has_tag(kid′, TAG_TRAILING_COMMA)
# Tag the parameters node to require a trailing comma # Tag the parameters node to require a trailing comma
kid′ = add_tag(kid′, TAG_TRAILING_COMMA) kid′ = add_tag(kid′, TAG_TRAILING_COMMA)
@ -617,33 +546,15 @@ function spaces_in_listlike(ctx::Context, node::Node)
x -> !(JuliaSyntax.is_whitespace(x) || kind(x) in KSet", ;"), x -> !(JuliaSyntax.is_whitespace(x) || kind(x) in KSet", ;"),
verified_kids(kid′) verified_kids(kid′)
) == 0 ) == 0
# If kid is K"parameters" without items and we don't want a trailing # If kid is K"parameters" without items and we don't want the
# comma/semicolon we need to eat any whitespace kids (e.g. comments) # K"parameters" node for the trailing semicolon we drop the entire node
grandkids = verified_kids(kid′) grandkids = verified_kids(kid′)
semi_idx = 1 @assert length(grandkids) == 1 && kind(grandkids[1]) === K";"
@assert kind(grandkids[semi_idx]) === K";"
ws_idx = findnext(x -> kind(x) !== K"Whitespace", grandkids, semi_idx + 1)
if ws_idx !== nothing
any_kid_changed = true
if kids′ === kids
kids′ = kids[1:(i - 1)]
end
replace_bytes!(ctx, "", mapreduce(span, +, grandkids[1:(ws_idx - 1)]; init = 0))
for j in ws_idx:length(grandkids)
grandkid = grandkids[j]
accept_node!(ctx, grandkid)
push!(kids′, grandkid)
end
else
# Nothing in the parameter node needed, overwrite it fully
any_kid_changed = true any_kid_changed = true
replace_bytes!(ctx, "", span(kid′)) replace_bytes!(ctx, "", span(kid′))
if any_kid_changed
if kids′ === kids if kids′ === kids
kids′ = kids[1:(i - 1)] kids′ = kids[1:(i - 1)]
end end
end
end
else else
# TODO: Tag for requiring trailing comma. # TODO: Tag for requiring trailing comma.
any_kid_changed |= this_kid_changed any_kid_changed |= this_kid_changed
@ -686,40 +597,10 @@ function spaces_in_listlike(ctx::Context, node::Node)
accept_node!(ctx, kid′) accept_node!(ctx, kid′)
any_kid_changed && push!(kids′, kid′) any_kid_changed && push!(kids′, kid′)
state = :expect_item state = :expect_item
elseif kind(kid′) === K"Comment"
# Comments are accepted, state stays the same
# TODO: Make sure there is a space before the comment? Maybe that's not the
# responsibility of this function though.
accept_node!(ctx, kid′)
any_kid_changed && push!(kids′, kid′)
else else
# Probably a list item, look for leading whitespace, or insert. # Probably a list item, insert a space before it
@assert !(kind(kid′) in KSet", ;") @assert !(kind(kid′) in KSet", ;")
if kind(first_leaf(kid′)) === K"NewlineWs" || @assert !JuliaSyntax.is_whitespace(first_leaf(kid′))
kind(first_leaf(kid′)) === K"Comment" ||
(kind(first_leaf(kid′)) === K"Whitespace" && kind(second_leaf(kid′)) === K"Comment")
# Newline, comment, or whitespace followed by comment
accept_node!(ctx, kid′)
any_kid_changed && push!(kids′, kid′)
state = state_after_item(i, last_item_idx, require_trailing_comma)
elseif kind(first_leaf(kid′)) === K"Whitespace"
ws_node = first_leaf(kid′)
if span(ws_node) == 1
accept_node!(ctx, kid′)
any_kid_changed && push!(kids′, kid′)
else
kid′ = replace_first_leaf(kid′, ws)
this_kid_changed = true
if kids′ === kids
kids′ = kids[1:(i - 1)]
end
replace_bytes!(ctx, " ", span(ws_node))
accept_node!(ctx, kid′)
push!(kids′, kid′)
end
state = state_after_item(i, last_item_idx, require_trailing_comma)
else
# Insert a standalone space kid and then accept the current node
this_kid_changed = true this_kid_changed = true
if kids′ === kids if kids′ === kids
kids′ = kids[1:(i - 1)] kids′ = kids[1:(i - 1)]
@ -732,7 +613,6 @@ function spaces_in_listlike(ctx::Context, node::Node)
# Here we inserted a space and consumed the next item, moving on to comma # Here we inserted a space and consumed the next item, moving on to comma
state = state_after_item(i, last_item_idx, require_trailing_comma) state = state_after_item(i, last_item_idx, require_trailing_comma)
end end
end
else else
@assert state === :expect_closing @assert state === :expect_closing
if (kind(kid′) === K"," && !allow_trailing_comma) || if (kind(kid′) === K"," && !allow_trailing_comma) ||
@ -867,11 +747,10 @@ function no_spaces_around_x(ctx::Context, node::Node, is_x::F) where {F}
end end
for (i, kid) in pairs(kids) for (i, kid) in pairs(kids)
if (i == 1 || i == length(kids)) && kind(kid) === K"Whitespace" if kind(kid) === K"Whitespace"
# Leave any leading and trailing whitespace # Leading and trailing whitespace should not be dropped but normalization should
accept_node!(ctx, kid) # make sure these never exist.
any_changes && push!(kids′, kid) @assert 1 < i < length(kids)
elseif kind(kid) === K"Whitespace"
# Ignore it but need to copy kids and re-write bytes # Ignore it but need to copy kids and re-write bytes
any_changes = true any_changes = true
if kids′ === kids if kids′ === kids
@ -882,23 +761,6 @@ function no_spaces_around_x(ctx::Context, node::Node, is_x::F) where {F}
@assert !JuliaSyntax.is_whitespace(kid) # Filtered out above @assert !JuliaSyntax.is_whitespace(kid) # Filtered out above
if looking_for_x if looking_for_x
@assert is_x(kid)::Bool @assert is_x(kid)::Bool
else
if i > first_x_idx
# Remove leading whitespace
ws_kid = first_leaf(kid)
if kind(ws_kid) === K"Whitespace"
kid = replace_first_leaf(kid, nullnode)
replace_bytes!(ctx, "", span(ws_kid))
any_changes = true
end
end
if i < last_x_idx
# Remove trailing whitespace
ws_kid = last_leaf(kid)
if kind(ws_kid) === K"Whitespace"
unreachable()
end
end
end end
if any_changes if any_changes
if kids === kids′ if kids === kids′
@ -948,14 +810,13 @@ function spaces_in_export_public(ctx::Context, node::Node)
any_changes && push!(kids′, kid) any_changes && push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
elseif kind(kid) === K"Whitespace" elseif kind(kid) === K"Whitespace"
kid′ = replace_first_leaf(kid, spacenode) replace_bytes!(ctx, " ", span(kid))
replace_bytes!(ctx, " ", span(first_leaf(kid)))
any_changes = true any_changes = true
if kids′ === kids if kids′ === kids
kids′ = kids[1:(i - 1)] kids′ = kids[1:(i - 1)]
end end
accept_node!(ctx, kid′) accept_node!(ctx, spacenode)
push!(kids′, kid′) push!(kids′, spacenode)
elseif kind(kid) === K"Comment" elseif kind(kid) === K"Comment"
any_changes && push!(kids′, kid) any_changes && push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
@ -1021,35 +882,17 @@ function spaces_in_export_public(ctx::Context, node::Node)
return any_changes ? make_node(node, kids′) : nothing return any_changes ? make_node(node, kids′) : nothing
end end
# Used in spaces_in_import_using. Well formatted importpath should have a single leading # Used in `spaces_in_import_using` and `format_as`
# space or a newline.
function format_importpath(ctx::Context, node::Node) function format_importpath(ctx::Context, node::Node)
@assert kind(node) === K"importpath" @assert kind(node) === K"importpath"
pos = position(ctx.fmt_io) @assert !JuliaSyntax.is_whitespace(first_leaf(node))
spacebar = Node(JuliaSyntax.SyntaxHead(K"Whitespace", JuliaSyntax.TRIVIA_FLAG), 1) return nothing
if kind(first_leaf(node)) === K"NewlineWs" ||
(kind(first_leaf(node)) === K"Whitespace" && span(first_leaf(node)) == 1)
# Newline or whitespace with correct span
node′ = nothing
elseif kind(first_leaf(node)) === K"Whitespace"
# Whitespace with incorrect span; replace with a single space
replace_bytes!(ctx, " ", span(first_leaf(node)))
node′ = replace_first_leaf(node, spacebar)
else
# No whitespace, insert
@assert kind(first_leaf(node)) in KSet"Identifier @ Comment" ||
JuliaSyntax.is_operator(first_leaf(node))
kids′ = copy(verified_kids(node))
pushfirst!(kids′, spacebar)
replace_bytes!(ctx, " ", 0)
node′ = make_node(node, kids′)
end
# Reset stream
seek(ctx.fmt_io, pos)
return node′
end end
# Used in spaces_in_import_using. # Used in `spaces_in_import_using`
# TODO: This doesn't handle comments and newlines which can be present on either side of the
# `as`. However, the asserts don't trigger on any julia package so can be fixed
# whenever someone files an issue about this :^)
function format_as(ctx::Context, node::Node) function format_as(ctx::Context, node::Node)
@assert kind(node) === K"as" @assert kind(node) === K"as"
kids = verified_kids(node) kids = verified_kids(node)
@ -1062,11 +905,7 @@ function format_as(ctx::Context, node::Node)
kid′ = kids[idx] kid′ = kids[idx]
@assert kind(kid′) === K"importpath" @assert kind(kid′) === K"importpath"
kid′′ = format_importpath(ctx, kid′) kid′′ = format_importpath(ctx, kid′)
if kid′′ !== nothing @assert kid′′ === nothing
any_changes = true
kid′ = kid′′
kids′ = [kid′]
end
accept_node!(ctx, kid′) accept_node!(ctx, kid′)
# space before `as` # space before `as`
idx += 1 idx += 1
@ -1110,7 +949,7 @@ function format_as(ctx::Context, node::Node)
accept_node!(ctx, spacebar) accept_node!(ctx, spacebar)
push!(kids′, spacebar) push!(kids′, spacebar)
end end
# Alias-identifier # Alias-identifier (RHS of the `as`)
idx += 1 idx += 1
kid = kids[idx] kid = kids[idx]
@assert kind(kid) in KSet"Identifier $ @" @assert kind(kid) in KSet"Identifier $ @"
@ -1134,6 +973,7 @@ function format_as(ctx::Context, node::Node)
return any_changes ? make_node(node, kids′) : nothing return any_changes ? make_node(node, kids′) : nothing
end end
# TODO: This method is very similar to `spaces_in_export_public`
function spaces_in_import_using(ctx::Context, node::Node) function spaces_in_import_using(ctx::Context, node::Node)
if !(kind(node) in KSet"import using" && !is_leaf(node)) if !(kind(node) in KSet"import using" && !is_leaf(node))
return nothing return nothing
@ -1154,16 +994,19 @@ function spaces_in_import_using(ctx::Context, node::Node)
@assert kind(kids[1]) in KSet"import using" @assert kind(kids[1]) in KSet"import using"
accept_node!(ctx, kids[1]) accept_node!(ctx, kids[1])
state = :expect_item spacebar = Node(JuliaSyntax.SyntaxHead(K"Whitespace", JuliaSyntax.TRIVIA_FLAG), 1)
state = :expect_space
i = 2 i = 2
while i <= length(kids) while i <= length(kids)
kid = kids[i] kid = kids[i]
if state === :expect_item if state === :expect_item
@assert kind(kid) in KSet"importpath as" state = :expect_comma
if kind(kid) in KSet"importpath as"
if kind(kid) === K"importpath" if kind(kid) === K"importpath"
kid′ = format_importpath(ctx, kid) kid′ = format_importpath(ctx, kid)
@assert kid′ === nothing
else else
@assert kind(kid) === K"as"
kid′ = format_as(ctx, kid) kid′ = format_as(ctx, kid)
end end
if kid′ !== nothing if kid′ !== nothing
@ -1177,9 +1020,15 @@ function spaces_in_import_using(ctx::Context, node::Node)
accept_node!(ctx, kid) accept_node!(ctx, kid)
any_changes && push!(kids′, kid) any_changes && push!(kids′, kid)
end end
state = :expect_comma elseif kind(kid) in KSet"Comment NewlineWs"
any_changes && push!(kids′, kid)
accept_node!(ctx, kid)
state = kind(kid) === K"Comment" ? (:expect_space) : (:expect_item)
else else
@assert state === :expect_comma unreachable()
end
elseif state === :expect_comma
state = :expect_space
if kind(kid) === K"Whitespace" if kind(kid) === K"Whitespace"
# Drop this node # Drop this node
any_changes = true any_changes = true
@ -1187,11 +1036,44 @@ function spaces_in_import_using(ctx::Context, node::Node)
if kids′ === kids if kids′ === kids
kids′ = kids[1:(i - 1)] kids′ = kids[1:(i - 1)]
end end
state = :expect_comma
else else
@assert kind(kid) in KSet": ," @assert kind(kid) in KSet": ,"
accept_node!(ctx, kid) accept_node!(ctx, kid)
any_changes && push!(kids′, kid) any_changes && push!(kids′, kid)
end
else
@assert state === :expect_space
state = :expect_item state = :expect_item
if kind(kid) === K"NewlineWs" || (kind(kid) === K"Whitespace" && span(kid) == 1)
# Newline or whitespace with correct span
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
elseif kind(kid) === K"Whitespace"
# Whitespace with incorrect span; replace with a single space
any_changes = true
replace_bytes!(ctx, " ", span(kid))
if kids′ === kids
kids′ = kids[1:(i - 1)]
end
accept_node!(ctx, spacebar)
push!(kids′, spacebar)
elseif kind(kid) === K"Comment"
any_changes && push!(kids′, kid)
accept_node!(ctx, kid)
state = :expect_space
else
# No whitespace, insert
@assert kind(kid) in KSet"Identifier importpath as"
@assert !JuliaSyntax.is_whitespace(first_leaf(kid))
any_changes = true
replace_bytes!(ctx, " ", 0)
if kids′ === kids
kids′ = kids[1:(i - 1)]
end
accept_node!(ctx, spacebar)
push!(kids′, spacebar)
continue # Skip increment of i
end end
end end
i += 1 i += 1
@ -1279,9 +1161,9 @@ function spaces_around_keywords(ctx::Context, node::Node)
state = :looking_for_space state = :looking_for_space
# `do` should only be followed by space if the argument-tuple is non-empty # `do` should only be followed by space if the argument-tuple is non-empty
if kind(node) === K"do" if kind(node) === K"do"
nkid = kids[i + 1] tupleidx = findnext(x -> kind(x) === K"tuple", kids, i + 1)::Int
@assert kind(nkid) === K"tuple" tuple = kids[tupleidx]
if !any(x -> !(JuliaSyntax.is_whitespace(x) || kind(x) === K";"), verified_kids(nkid)) if !any(x -> !(JuliaSyntax.is_whitespace(x) || kind(x) === K";"), verified_kids(tuple))
state = :closing state = :closing
end end
end end
@ -1298,7 +1180,7 @@ function spaces_around_keywords(ctx::Context, node::Node)
end end
elseif state === :looking_for_space elseif state === :looking_for_space
if (kind(kid) === K"Whitespace" && span(kid) == 1) || if (kind(kid) === K"Whitespace" && span(kid) == 1) ||
kind(kid) === K"NewlineWs" || kind(first_leaf(kid)) === K"NewlineWs" kind(kid) === K"NewlineWs"
if kind(kid) === K"NewlineWs" if kind(kid) === K"NewlineWs"
# Is a newline instead of a space accepted for any other case? # Is a newline instead of a space accepted for any other case?
@assert kind(node) in KSet"where local global const" @assert kind(node) in KSet"where local global const"
@ -1314,25 +1196,8 @@ function spaces_around_keywords(ctx::Context, node::Node)
replace_bytes!(ctx, " ", span(kid)) replace_bytes!(ctx, " ", span(kid))
push!(kids′, ws) push!(kids′, ws)
accept_node!(ctx, ws) accept_node!(ctx, ws)
elseif space_after && kind(first_leaf(kid)) === K"Whitespace"
kid_ws = first_leaf(kid)
if span(kid_ws) == 1
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
else
kid′ = replace_first_leaf(kid, ws)
@assert span(kid′) == span(kid) - span(kid_ws) + 1
replace_bytes!(ctx, " ", span(kid_ws))
accept_node!(ctx, kid′)
any_changes = true
if kids′ === kids
kids′ = kids[1:(i - 1)]
end
push!(kids′, kid′)
end
elseif !space_after && kind(last_leaf(kid)) === K"Whitespace"
unreachable()
else else
@assert kind(first_leaf(kid)) !== K"Whitespace"
# Reachable in e.g. `T where{T}`, `if(`, ... insert space # Reachable in e.g. `T where{T}`, `if(`, ... insert space
@assert kind(node) in KSet"where if elseif while do function return local global module baremodule" @assert kind(node) in KSet"where if elseif while do function return local global module baremodule"
any_changes = true any_changes = true
@ -1416,11 +1281,8 @@ function replace_with_in_filter(ctx::Context, node::Node)
@assert kind(node) === K"filter" && !is_leaf(node) @assert kind(node) === K"filter" && !is_leaf(node)
pos = position(ctx.fmt_io) pos = position(ctx.fmt_io)
kids = verified_kids(node) kids = verified_kids(node)
idx = findfirst(x -> kind(x) in KSet"= cartesian_iterator" && !is_leaf(x), kids)::Int kid = kids[1]
for i in 1:(idx - 1) @assert kind(kid) in KSet"= cartesian_iterator" && !is_leaf(kid)
accept_node!(ctx, kids[i])
end
kid = kids[idx]
if kind(kid) === K"=" if kind(kid) === K"="
kid′ = replace_with_in(ctx, kid) kid′ = replace_with_in(ctx, kid)
else else
@ -1432,7 +1294,7 @@ function replace_with_in_filter(ctx::Context, node::Node)
return nothing return nothing
else else
kids = copy(kids) kids = copy(kids)
kids[idx] = kid′ kids[1] = kid′
return make_node(node, kids) return make_node(node, kids)
end end
end end
@ -1490,10 +1352,13 @@ function for_loop_use_in(ctx::Context, node::Node)
for i in next_index:for_index for i in next_index:for_index
accept_node!(ctx, kids[i]) accept_node!(ctx, kids[i])
end end
# The for loop specification node can be either K"=" or K"cartesian_iterator" # The for loop specification node can be one of K"=", K"cartesian_iterator", or
for_spec_index = for_index + 1 # K"filter"
for_spec_index = findnext(x -> kind(x) in KSet"= cartesian_iterator filter", kids, for_index + 1)::Int
for_spec_node = kids[for_spec_index] for_spec_node = kids[for_spec_index]
@assert kind(for_spec_node) in KSet"= cartesian_iterator filter" for i in (for_index + 1):(for_spec_index - 1)
accept_node!(ctx, kids[i])
end
if kind(for_spec_node) === K"=" if kind(for_spec_node) === K"="
for_spec_node′ = replace_with_in(ctx, for_spec_node) for_spec_node′ = replace_with_in(ctx, for_spec_node)
elseif kind(for_spec_node) === K"filter" elseif kind(for_spec_node) === K"filter"
@ -1544,6 +1409,7 @@ function braces_around_where_rhs(ctx::Context, node::Node)
pos = position(ctx.fmt_io) pos = position(ctx.fmt_io)
where_idx = findfirst(x -> is_leaf(x) && kind(x) === K"where", kids)::Int where_idx = findfirst(x -> is_leaf(x) && kind(x) === K"where", kids)::Int
rhs_idx = findnext(!JuliaSyntax.is_whitespace, kids, where_idx + 1)::Int rhs_idx = findnext(!JuliaSyntax.is_whitespace, kids, where_idx + 1)::Int
@assert rhs_idx == lastindex(kids)
rhs = kids[rhs_idx] rhs = kids[rhs_idx]
if kind(rhs) === K"braces" if kind(rhs) === K"braces"
return nothing return nothing
@ -1566,11 +1432,6 @@ function braces_around_where_rhs(ctx::Context, node::Node)
accept_node!(ctx, rhs) accept_node!(ctx, rhs)
replace_bytes!(ctx, "}", 0) replace_bytes!(ctx, "}", 0)
accept_node!(ctx, closing_brace) accept_node!(ctx, closing_brace)
# Accept any remaining kids
for i in (rhs_idx + 1):length(kids)
accept_node!(ctx, kids[i])
push!(kids′, kids[i])
end
# Reset stream and return # Reset stream and return
seek(ctx.fmt_io, pos) seek(ctx.fmt_io, pos)
return make_node(node, kids′) return make_node(node, kids′)
@ -1594,18 +1455,15 @@ function parens_around_op_calls_in_colon(ctx::Context, node::Node)
end end
grandkids = verified_kids(kid) grandkids = verified_kids(kid)
first_non_ws = findfirst(!JuliaSyntax.is_whitespace, grandkids)::Int first_non_ws = findfirst(!JuliaSyntax.is_whitespace, grandkids)::Int
@assert first_non_ws == firstindex(grandkids)
last_non_ws = findlast(!JuliaSyntax.is_whitespace, grandkids)::Int last_non_ws = findlast(!JuliaSyntax.is_whitespace, grandkids)::Int
# Extract whitespace grandkids to become kids @assert last_non_ws == lastindex(grandkids)
for j in 1:(first_non_ws - 1)
accept_node!(ctx, grandkids[j])
push!(kids′, grandkids[j])
end
# Create the parens node # Create the parens node
opening_paren = Node(JuliaSyntax.SyntaxHead(K"(", 0), 1) opening_paren = Node(JuliaSyntax.SyntaxHead(K"(", 0), 1)
replace_bytes!(ctx, "(", 0) replace_bytes!(ctx, "(", 0)
accept_node!(ctx, opening_paren) accept_node!(ctx, opening_paren)
parens_kids = [opening_paren] parens_kids = [opening_paren]
kid′_kids = grandkids[first_non_ws:last_non_ws] kid′_kids = copy(grandkids)
kid′ = make_node(kid, kid′_kids) kid′ = make_node(kid, kid′_kids)
accept_node!(ctx, kid′) accept_node!(ctx, kid′)
push!(parens_kids, kid′) push!(parens_kids, kid′)
@ -1615,10 +1473,6 @@ function parens_around_op_calls_in_colon(ctx::Context, node::Node)
push!(parens_kids, closing_paren) push!(parens_kids, closing_paren)
parens = Node(JuliaSyntax.SyntaxHead(K"parens", 0), parens_kids) parens = Node(JuliaSyntax.SyntaxHead(K"parens", 0), parens_kids)
push!(kids′, parens) push!(kids′, parens)
for j in (last_non_ws + 1):length(grandkids)
accept_node!(ctx, grandkids[j])
push!(kids′, grandkids[j])
end
any_changes = true any_changes = true
else else
accept_node!(ctx, kid) accept_node!(ctx, kid)
@ -1895,6 +1749,8 @@ function indent_begin(ctx::Context, node::Node, block_kind = K"begin")
return any_kid_changed ? make_node(node, kids) : nothing return any_kid_changed ? make_node(node, kids) : nothing
end end
# This function ensures that the block start, and ends, with a newline, and make sure that
# the trailing newline is tagged with TAG_PRE_DEDENT.
function indent_block( function indent_block(
ctx::Context, node::Node; allow_empty::Bool = true, do_indent::Bool = true ctx::Context, node::Node; allow_empty::Bool = true, do_indent::Bool = true
) )
@ -2032,12 +1888,6 @@ function indent_block(
if kind(kids′[insert_idx]) === K"Whitespace" if kind(kids′[insert_idx]) === K"Whitespace"
kids′, ws = popatview!(kids′, insert_idx) kids′, ws = popatview!(kids′, insert_idx)
wsspn = span(ws) wsspn = span(ws)
elseif !is_leaf(kids′[insert_idx]) &&
kind(first_leaf(kids′[insert_idx])) === K"Whitespace"
ws = first_leaf(kids′[insert_idx])
kids′[insert_idx] = replace_first_leaf(kids′[insert_idx], nullnode)
any_kid_changed = true
wsspn = span(ws)
end end
# If we end up in this code path we are most likely splitting a single line block # If we end up in this code path we are most likely splitting a single line block
# into multiples lines. This means that we haven't yet updated the indent level for # into multiples lines. This means that we haven't yet updated the indent level for
@ -2373,9 +2223,7 @@ function indent_listlike(
@assert multiline @assert multiline
kid = kids[open_idx + 1] kid = kids[open_idx + 1]
idx_after_leading_nl = open_idx + 2 idx_after_leading_nl = open_idx + 2
if kind(kid) === K"NewlineWs" || if kind(kid) === K"NewlineWs"
kind(first_leaf(kid)) === K"NewlineWs" ||
peek_leafs(kid, KSet"Comment NewlineWs") || peek_leafs(kid, KSet"Whitespace Comment NewlineWs")
# Newline or newlinde hidden in first item # Newline or newlinde hidden in first item
any_kid_changed && push!(kids′, kid) any_kid_changed && push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
@ -2405,17 +2253,6 @@ function indent_listlike(
any_kid_changed = true any_kid_changed = true
push!(kids′, kid) push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
elseif kind(first_leaf(kid)) === K"Whitespace"
grandkid = first_leaf(kid)
grandkid = Node(JuliaSyntax.SyntaxHead(K"NewlineWs", JuliaSyntax.TRIVIA_FLAG), span(grandkid) + 1)
replace_bytes!(ctx, "\n", 0)
kid = replace_first_leaf(kid, grandkid)
if kids′ === kids
kids′ = kids[1:(open_idx - 1)]
end
any_kid_changed = true
push!(kids′, kid)
accept_node!(ctx, kid)
elseif kind(kid) === K"parameters" elseif kind(kid) === K"parameters"
# For parameters we want the newline after the semi-colon # For parameters we want the newline after the semi-colon
grandkids = verified_kids(kid) grandkids = verified_kids(kid)
@ -2465,6 +2302,7 @@ function indent_listlike(
accept_node!(ctx, kid) accept_node!(ctx, kid)
end end
else else
@assert kind(first_leaf(kid)) !== K"Whitespace"
nlws = Node(JuliaSyntax.SyntaxHead(K"NewlineWs", JuliaSyntax.TRIVIA_FLAG), 1) nlws = Node(JuliaSyntax.SyntaxHead(K"NewlineWs", JuliaSyntax.TRIVIA_FLAG), 1)
replace_bytes!(ctx, "\n", 0) replace_bytes!(ctx, "\n", 0)
if kids′ === kids if kids′ === kids
@ -2498,8 +2336,7 @@ function indent_listlike(
else else
kid = kids[close_idx - 1] kid = kids[close_idx - 1]
end end
if (kind(kid) === K"NewlineWs" && has_tag(kid, TAG_PRE_DEDENT)) || if kind(kid) === K"NewlineWs" && has_tag(kid, TAG_PRE_DEDENT)
(kind(last_leaf(kid)) === K"NewlineWs" && has_tag(last_leaf(kid), TAG_PRE_DEDENT))
# Newline or newlinde hidden in first item with tag # Newline or newlinde hidden in first item with tag
any_kid_changed && push!(kids′, kid) any_kid_changed && push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
@ -2513,19 +2350,8 @@ function indent_listlike(
any_kid_changed = true any_kid_changed = true
push!(kids′, kid) push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
elseif kind(last_leaf(kid)) === K"NewlineWs"
# Hidden newline without tag
grandkid = last_leaf(kid)
@assert !has_tag(grandkid, TAG_PRE_DEDENT)
grandkid = add_tag(grandkid, TAG_PRE_DEDENT)
kid = replace_last_leaf(kid, grandkid)
if kids′ === kids
kids′ = kids[1:(close_idx - 2)]
end
any_kid_changed = true
push!(kids′, kid)
accept_node!(ctx, kid)
else else
@assert kind(last_leaf(kid)) !== K"NewlineWs"
# Need to insert a newline. Note that we tag the new newline directly since it # Need to insert a newline. Note that we tag the new newline directly since it
# is the responsibility of this function (otherwise there would just be an extra # is the responsibility of this function (otherwise there would just be an extra
# repetitive call to add it anyway). # repetitive call to add it anyway).
@ -2540,26 +2366,8 @@ function indent_listlike(
any_kid_changed = true any_kid_changed = true
push!(kids′, kid) push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
elseif kind(last_leaf(kid)) === K"Whitespace"
# TODO: I believe this is only reachable because whitespace isn't trimmed from
# vcat nodes like function calls and tuples etc.
@assert kind(node) in KSet"vcat typed_vcat"
# Since the whitespace is inside the kid node we assume that a newlinews would
# have ended up inside too. We also remove all existing whitespace since it
# would otherwise have to be cleaned up by the trailing whitespace pass.
accept_node!(ctx, kid)
seek(ctx.fmt_io, position(ctx.fmt_io) - span(last_leaf(kid)))
replace_bytes!(ctx, "\n", span(last_leaf(kid)))
nlws = Node(JuliaSyntax.SyntaxHead(K"NewlineWs", JuliaSyntax.TRIVIA_FLAG), 1)
nlws = add_tag(nlws, TAG_PRE_DEDENT)
accept_node!(ctx, nlws)
kid′ = replace_last_leaf(kid, nlws)
if kids′ === kids
kids′ = kids[1:(open_idx - 1)]
end
any_kid_changed = true
push!(kids′, kid′)
else else
@assert kind(last_leaf(kid)) !== K"Whitespace"
# Note that this is a trailing newline and should be put after this item # Note that this is a trailing newline and should be put after this item
if kids′ === kids if kids′ === kids
kids′ = kids[1:(open_idx - 1)] kids′ = kids[1:(open_idx - 1)]
@ -2587,11 +2395,7 @@ function indent_listlike(
any_kid_changed && push!(kids′, kid) any_kid_changed && push!(kids′, kid)
accept_node!(ctx, kid) accept_node!(ctx, kid)
# Keep any remaining kids # Keep any remaining kids
for i in (close_idx + 1):length(kids) @assert close_idx == lastindex(kids)
kid = kids[i]
any_kid_changed && push!(kids′, kid)
accept_node!(ctx, kid)
end
# Reset stream # Reset stream
seek(ctx.fmt_io, pos) seek(ctx.fmt_io, pos)
# Make a new node and return # Make a new node and return
@ -2815,6 +2619,7 @@ function indent_assignment(ctx::Context, node::Node)
@assert kind(node) === kind(kids[eqidx]) @assert kind(node) === kind(kids[eqidx])
@assert length(kids) > eqidx @assert length(kids) > eqidx
rhsidx = findnext(!JuliaSyntax.is_whitespace, kids, eqidx + 1)::Int rhsidx = findnext(!JuliaSyntax.is_whitespace, kids, eqidx + 1)::Int
@assert rhsidx == lastindex(kids)
r = (eqidx + 1):(rhsidx - 1) r = (eqidx + 1):(rhsidx - 1)
length(r) == 0 && return nothing length(r) == 0 && return nothing
rhs = kids[rhsidx] rhs = kids[rhsidx]
@ -2869,9 +2674,6 @@ function indent_assignment(ctx::Context, node::Node)
end end
if changed if changed
@assert kids !== kids′ @assert kids !== kids′
for i in (rhsidx + 1):length(kids)
push!(kids′, kids[i])
end
return make_node(node, kids′) return make_node(node, kids′)
else else
return nothing return nothing
@ -3001,24 +2803,11 @@ function indent_module(ctx::Context, node::Node; do_indent::Bool = true)
kids[mod_idx] = add_tag(mod_node, TAG_INDENT) kids[mod_idx] = add_tag(mod_node, TAG_INDENT)
any_kid_changed = true any_kid_changed = true
end end
# Next we expect whitespace + identifier, but can also be expression with whitespace # Next we expect whitespace + module identifier
# hidden inside... modname_idx = findnext(x -> !JuliaSyntax.is_whitespace(x), kids, mod_idx + 1)::Int
space_idx = 2
space_node = kids[space_idx]
if kind(space_node) === K"Whitespace"
# Now we need an identifier, var" or parens...
id_idx = 3
id_node = kids[id_idx]
@assert kind(id_node) in KSet"Identifier var parens"
block_idx = 4
else
# This can be reached if the module name is interpolated or parenthesized, for
# example.
@assert kind(first_leaf(space_node)) in KSet"Whitespace ("
@assert !JuliaSyntax.is_whitespace(space_node)
block_idx = 3
end
# Next node is the module body block. # Next node is the module body block.
block_idx = findnext(x -> kind(x) === K"block", kids, modname_idx + 1)::Int
@assert block_idx == modname_idx + 1
let p = position(ctx.fmt_io) let p = position(ctx.fmt_io)
for i in 1:(block_idx - 1) for i in 1:(block_idx - 1)
accept_node!(ctx, kids[i]) accept_node!(ctx, kids[i])
@ -3085,17 +2874,12 @@ function indent_local_global(ctx::Context, node::Node)
kids = verified_kids(node) kids = verified_kids(node)
kw = findfirst(x -> is_leaf(x) && kind(x) in KSet"local global", kids)::Int kw = findfirst(x -> is_leaf(x) && kind(x) in KSet"local global", kids)::Int
nonws = findnext(!JuliaSyntax.is_whitespace, kids, kw + 1)::Int nonws = findnext(!JuliaSyntax.is_whitespace, kids, kw + 1)::Int
@assert kind(first_leaf(kids[nonws])) !== K"NewlineWs"
changed = false changed = false
for i in (kw + 1):nonws for i in (kw + 1):(nonws - 1)
if kind(kids[i]) === K"NewlineWs" && !has_tag(kids[i], TAG_LINE_CONT) if kind(kids[i]) === K"NewlineWs" && !has_tag(kids[i], TAG_LINE_CONT)
kids[i] = add_tag(kids[i], TAG_LINE_CONT) kids[i] = add_tag(kids[i], TAG_LINE_CONT)
changed = true changed = true
elseif i == nonws && kind(first_leaf(kids[i])) === K"NewlineWs" &&
!has_tag(first_leaf(kids[i]), TAG_LINE_CONT)
kid′ = replace_first_leaf(kids[i], add_tag(first_leaf(kids[i]), TAG_LINE_CONT))
@assert kid′ !== nothing
kids[i] = kid′
changed = true
end end
end end
return changed ? make_node(node, kids) : nothing return changed ? make_node(node, kids) : nothing
@ -3503,17 +3287,11 @@ function return_node(ctx::Context, ret::Node)
push!(kids, ws) push!(kids, ws)
push!(kids, ret) push!(kids, ret)
else else
@assert kind(first_leaf(ret)) !== K"NewlineWs" @assert !(kind(first_leaf(ret)) in KSet"NewlineWs Whitespace")
has_ws = kind(first_leaf(ret)) === K"Whitespace"
if has_ws
replace_bytes!(ctx, "return", 0)
push!(kids, ret)
else
replace_bytes!(ctx, "return ", 0) replace_bytes!(ctx, "return ", 0)
push!(kids, ws) push!(kids, ws)
push!(kids, ret) push!(kids, ret)
end end
end
return Node(JuliaSyntax.SyntaxHead(K"return", 0), kids) return Node(JuliaSyntax.SyntaxHead(K"return", 0), kids)
end end
@ -3549,7 +3327,7 @@ function has_return(node::Node)
# that is the job of a linter or something I guess. # that is the job of a linter or something I guess.
return findfirst(x -> kind(x) === K"return", kids) !== nothing return findfirst(x -> kind(x) === K"return", kids) !== nothing
else else
error("unreachable") unreachable()
end end
end end
@ -3587,11 +3365,9 @@ function explicit_return_block(ctx, node)
if kind(rexpr) === K"call" if kind(rexpr) === K"call"
call_kids = verified_kids(rexpr) call_kids = verified_kids(rexpr)
fname_idx = findfirst(!JuliaSyntax.is_whitespace, call_kids)::Int fname_idx = findfirst(!JuliaSyntax.is_whitespace, call_kids)::Int
@assert fname_idx == firstindex(call_kids)
local fname local fname
let p = position(ctx.fmt_io) let p = position(ctx.fmt_io)
for i in 1:(fname_idx - 1)
accept_node!(ctx, call_kids[i])
end
fname = String(read_bytes(ctx, call_kids[fname_idx])) fname = String(read_bytes(ctx, call_kids[fname_idx]))
seek(ctx.fmt_io, p) seek(ctx.fmt_io, p)
end end
@ -3620,11 +3396,8 @@ function explicit_return_block(ctx, node)
seek(ctx.fmt_io, position(ctx.fmt_io) - spn) seek(ctx.fmt_io, position(ctx.fmt_io) - spn)
rexpr_idx -= 1 rexpr_idx -= 1
@assert kind(first_leaf(rexpr)) !== K"Whitespace" @assert kind(first_leaf(rexpr)) !== K"Whitespace"
elseif kind(first_leaf(rexpr)) === K"Whitespace"
# The first child in the return node is whitespace
spn = span(first_leaf(rexpr))
rexpr = replace_first_leaf(rexpr, nullnode)
end end
@assert kind(first_leaf(rexpr)) !== K"Whitespace"
nl = "\n" * " "^(4 * ctx.indent_level) nl = "\n" * " "^(4 * ctx.indent_level)
nlnode = Node(JuliaSyntax.SyntaxHead(K"NewlineWs", JuliaSyntax.TRIVIA_FLAG), sizeof(nl)) nlnode = Node(JuliaSyntax.SyntaxHead(K"NewlineWs", JuliaSyntax.TRIVIA_FLAG), sizeof(nl))
insert!(kids′, rexpr_idx, nlnode) insert!(kids′, rexpr_idx, nlnode)

44
test/runtests.jl
Unescape View File

@ -59,6 +59,30 @@ end
@test KSet"Whitespace ; Whitespace" == (K"Whitespace", K";", K"Whitespace") @test KSet"Whitespace ; Whitespace" == (K"Whitespace", K";", K"Whitespace")
end end
@testset "syntax tree normalization" begin
function rparse(str)
node = Runic.Node(JuliaSyntax.parseall(JuliaSyntax.GreenNode, str))
Runic.normalize_tree!(node)
return node
end
function check_no_leading_trailing_ws(node)
Runic.is_leaf(node) && return
kids = Runic.verified_kids(node)
if length(kids) > 0
@test !JuliaSyntax.is_whitespace(kids[1])
@test !JuliaSyntax.is_whitespace(kids[end])
end
for kid in kids
check_no_leading_trailing_ws(kid)
end
end
# Random things found in the ecosystem
str = "[\n # c\n 0.5 1.0\n 1.0 2.0\n # c\n]"
check_no_leading_trailing_ws(rparse(str))
str = "[0.5 1.0; 1.0 2.0;;; 4.5 6.0; 6.0 8.0]"
check_no_leading_trailing_ws(rparse(str))
end
@testset "Trailing whitespace" begin @testset "Trailing whitespace" begin
io = IOBuffer() io = IOBuffer()
println(io, "a = 1 ") # Trailing space println(io, "a = 1 ") # Trailing space
@ -516,8 +540,8 @@ end
# After `local`, `global`, and `const` a newline can be used instead of a space # After `local`, `global`, and `const` a newline can be used instead of a space
@test format_string("$(word)$(sp)\n$(sp)$(rhs)") == "$(word)\n $(rhs)" @test format_string("$(word)$(sp)\n$(sp)$(rhs)") == "$(word)\n $(rhs)"
end end
@test_broken format_string("global\n\nx = 1") == "global\n\n x = 1" # TODO: Fix double peek @test format_string("global\n\nx = 1") == "global\n\n x = 1"
@test_broken format_string("lobal\n\nx = 1") == "local\n\n x = 1" # TODO: Fix double peek @test format_string("local\n\nx = 1") == "local\n\n x = 1"
@test format_string("const$(sp)x = 1") == "const x = 1" @test format_string("const$(sp)x = 1") == "const x = 1"
# After `where` a newline can be used instead of a space # After `where` a newline can be used instead of a space
@test format_string("A$(sp)where$(sp)\n{A}") == "A where\n{A}" @test format_string("A$(sp)where$(sp)\n{A}") == "A where\n{A}"
@ -935,6 +959,7 @@ end
@test format_string("$(verb) $(sp)A$(sp),$(sp)B") == "$(verb) A, B" @test format_string("$(verb) $(sp)A$(sp),$(sp)B") == "$(verb) A, B"
@test format_string("$(verb) A$(sp),\nB") == "$(verb) A,\n B" @test format_string("$(verb) A$(sp),\nB") == "$(verb) A,\n B"
@test format_string("$(verb) \nA$(sp),\nB") == "$(verb)\n A,\n B" @test format_string("$(verb) \nA$(sp),\nB") == "$(verb)\n A,\n B"
@test format_string("$(verb) $(sp)A,$(sp)\n\n$(sp)B") == "$(verb) A,\n\n B"
# Colon lists # Colon lists
for a in ("a", "@a", "*") for a in ("a", "@a", "*")
@test format_string("$(verb) $(sp)A: $(sp)$(a)") == "$(verb) A: $(a)" @test format_string("$(verb) $(sp)A: $(sp)$(a)") == "$(verb) A: $(a)"
@ -1229,6 +1254,8 @@ end
# Trailing semicolon before ws+comment # Trailing semicolon before ws+comment
f; # comment f; # comment
f;; # comment f;; # comment
# Trailing semicolon with whitespace on both sides
g ; # comment
""" """
bodyfmt = """ bodyfmt = """
# Semicolons on their own lines # Semicolons on their own lines
@ -1252,6 +1279,8 @@ end
# Trailing semicolon before ws+comment # Trailing semicolon before ws+comment
f # comment f # comment
f # comment f # comment
# Trailing semicolon with whitespace on both sides
g # comment
""" """
for prefix in ( for prefix in (
"begin", "quote", "for i in I", "let", "let x = 1", "while cond", "begin", "quote", "for i in I", "let", "let x = 1", "while cond",
@ -1453,6 +1482,17 @@ end
"1 + 1\n$off #src\n1+1\n$on #src\n1 + 1" "1 + 1\n$off #src\n1+1\n$on #src\n1 + 1"
@test format_string("1+1\n#src $off\n1+1\n#src $on\n1+1") == @test format_string("1+1\n#src $off\n1+1\n#src $on\n1+1") ==
"1 + 1\n#src $off\n1+1\n#src $on\n1 + 1" "1 + 1\n#src $off\n1+1\n#src $on\n1 + 1"
# Toggles inside literal array expression (fixed by normalization, PR #101)
let str = """
[
$off
1.10 1.20 1.30
-2.10 -1.20 +1.30
f( a+b )
$on
]"""
@test format_string(str) == str
end
end end
end end

Write Preview
Loading…
Cancel
Save