Browse Source

Wrap JuliaSyntax.GreenNode in Runic.Node

Using JuliaSyntax.GreenNode directly have worked for a long time, but at
this point it seems that it is easier to re-package the tree in a custom
type. This will be used to attach metadata to nodes, for example.

Also include the following renames:
 - s/verified_children/verified_kids/
 - s/children/kids/
 - s/children′/kids′/
 - s/children′′/kids′′/
 - s/child/kid/
 - s/child′/kid′/
 - s/child′′/kid′′/
pull/19/head
Fredrik Ekre 2 years ago
parent
commit
944076139c
No known key found for this signature in database
GPG Key ID: DE82E6D5E364C0A2
  1. 171
      src/Runic.jl
  2. 122
      src/chisels.jl
  3. 34
      src/debug.jl
  4. 368
      src/runestone.jl
  5. 14
      test/runtests.jl

171
src/Runic.jl

@ -14,7 +14,56 @@ using JuliaSyntax: @@ -14,7 +14,56 @@ using JuliaSyntax:
end
end
# JuliaSyntax extensions and other utilities
# Debug and assert utilities
include("debug.jl")
########
# Node #
########
# This is essentially just a re-packed `JuliaSyntax.GreenNode`.
struct Node
# The next three fields directly match JuliaSyntax.GreenNode. We can not store a
# GreenNode directly because the type of the children vector should be `Vector{Node}`
# and not `Vector{GreenNode}`.
head::JuliaSyntax.SyntaxHead
span::UInt32
kids::Union{Tuple{}, Vector{Node}}
end
# Re-package a GreenNode as a Node
function Node(node::JuliaSyntax.GreenNode)
return Node(
JuliaSyntax.head(node), JuliaSyntax.span(node),
map(Node, JuliaSyntax.children(node)),
)
end
# Defining these allow using many duck-typed methods in JuliaSyntax directly without having
# to re-package a Node as a GreenNode.
JuliaSyntax.head(node::Node) = head(node)
JuliaSyntax.span(node::Node) = span(node)
# Matching JuliaSyntax.(head|span|flags|kind)
head(node::Node) = node.head
span(node::Node) = node.span
flags(node::Node) = JuliaSyntax.flags(node)
kind(node::Node) = JuliaSyntax.kind(node)
# Inverse of JuliaSyntax.haschildren
function is_leaf(node::Node)
return node.kids === ()
end
# This function must only be be called after verifying that the node is not a leaf. We can
# then type-assert the return value to narrow it down from `Union{Tuple{}, Vector{Node}}` to
# `Vector{Node}`.
function verified_kids(node::Node)
@assert !is_leaf(node)
return node.kids::Vector{Node}
end
# Node utilities and JuliaSyntax extensions
include("chisels.jl")
# Return the result of expr if it doesn't evaluate to `nothing`
@ -24,14 +73,18 @@ macro return_something(expr) @@ -24,14 +73,18 @@ macro return_something(expr)
end)
end
#######################################################
# Main drivers for traversing and formatting the tree #
#######################################################
mutable struct Context
# Input
@const src_str::String
@const src_tree::JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}
@const src_tree::Node
@const src_io::IOBuffer
# Output
@const fmt_io::IOBuffer
fmt_tree::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing}
fmt_tree::Union{Node, Nothing}
# User settings
quiet::Bool
verbose::Bool
@ -40,10 +93,10 @@ mutable struct Context @@ -40,10 +93,10 @@ mutable struct Context
check::Bool
diff::Bool
# Current state
# node::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing}
prev_sibling::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing}
next_sibling::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing}
# parent::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing}
# node::Union{Node, Nothing}
prev_sibling::Union{Node, Nothing}
next_sibling::Union{Node, Nothing}
# parent::Union{Node, Nothing}
end
function Context(
@ -51,7 +104,9 @@ function Context( @@ -51,7 +104,9 @@ function Context(
diff::Bool = false, check::Bool = false, quiet::Bool = false,
)
src_io = IOBuffer(src_str)
src_tree = JuliaSyntax.parseall(JuliaSyntax.GreenNode, src_str; ignore_warnings = true)
src_tree = Node(
JuliaSyntax.parseall(JuliaSyntax.GreenNode, src_str; ignore_warnings = true),
)
fmt_io = IOBuffer()
fmt_tree = nothing
# Debug mode enforces verbose and assert
@ -71,17 +126,17 @@ end @@ -71,17 +126,17 @@ end
# Read the bytes of the current node from the output io
function read_bytes(ctx, node)
pos = position(ctx.fmt_io)
bytes = read(ctx.fmt_io, JuliaSyntax.span(node))
@assert length(bytes) == JuliaSyntax.span(node)
bytes = read(ctx.fmt_io, span(node))
@assert length(bytes) == span(node)
seek(ctx.fmt_io, pos)
@assert position(ctx.fmt_io) == pos
return bytes
end
function accept_node!(ctx::Context, node::JuliaSyntax.GreenNode)
function accept_node!(ctx::Context, node::Node)
# Accept the string representation of the current node by advancing the
# output IO to the start of the next node
pos = position(ctx.fmt_io) + JuliaSyntax.span(node)
pos = position(ctx.fmt_io) + span(node)
seek(ctx.fmt_io, pos)
return
end
@ -93,9 +148,9 @@ end @@ -93,9 +148,9 @@ end
struct NullNode end
const nullnode = NullNode()
function format_node_with_children!(ctx::Context, node::JuliaSyntax.GreenNode)
# If the node doesn't have children there is nothing to do here
if !JuliaSyntax.haschildren(node)
function format_node_with_kids!(ctx::Context, node::Node)
# If the node doesn't have kids there is nothing to do here
if is_leaf(node)
return nothing
end
@ -105,63 +160,63 @@ function format_node_with_children!(ctx::Context, node::JuliaSyntax.GreenNode) @@ -105,63 +160,63 @@ function format_node_with_children!(ctx::Context, node::JuliaSyntax.GreenNode)
ctx.prev_sibling = nothing
ctx.next_sibling = nothing
# The new node parts. `children′` aliases `children` and only copied below if any of the
# The new node parts. `kids′` aliases `kids` and only copied below if any of the
# nodes change ("copy-on-write").
children = verified_children(node)
children′ = children
any_child_changed = false
# Loop over all the children
for (i, child) in pairs(children)
# Set the siblings: previous from children′, next from children
ctx.prev_sibling = get(children, i - 1, nothing)
ctx.next_sibling = get(children, i + 1, nothing)
child′ = child
this_child_changed = false
kids = verified_kids(node)
kids′ = kids
any_kid_changed = false
# Loop over all the kids
for (i, kid) in pairs(kids)
# Set the siblings: previous from kids′, next from kids
ctx.prev_sibling = get(kids, i - 1, nothing)
ctx.next_sibling = get(kids, i + 1, nothing)
kid′ = kid
this_kid_changed = false
itr = 0
# Loop until this node reaches a steady state and is accepted
while true
# Keep track of the stream position and reset it below if the node is changed
fmt_pos = position(ctx.fmt_io)
# Format the child
child′′ = format_node!(ctx, child′)
if child′′ === nullnode
# Format the kid
kid′′ = format_node!(ctx, kid′)
if kid′′ === nullnode
# This node should be deleted from the tree
# TODO: When this is fixed the sibling setting above needs to be modified to
# handle this too
this_child_changed = true
error("TODO: handle removed children")
elseif child′′ === nothing
this_kid_changed = true
error("TODO: handle removed kids")
elseif kid′′ === nothing
# The node was accepted, continue to next sibling
@assert position(ctx.fmt_io) == fmt_pos + JuliaSyntax.span(child′)
@assert position(ctx.fmt_io) == fmt_pos + span(kid′)
break
else
# The node should be replaced with the new one. Reset the stream and try
# again until it is accepted.
@assert child′′ isa JuliaSyntax.GreenNode
this_child_changed = true
@assert kid′′ isa Node
this_kid_changed = true
seek(ctx.fmt_io, fmt_pos)
child′ = child′′
kid′ = kid′′
end
if (itr += 1) == 1000
error("infinite loop?")
end
end
any_child_changed |= this_child_changed
if any_child_changed
# De-alias the children if not already done
if children′ === children
children = eltype(children)[children[j] for j in 1:(i - 1)]
any_kid_changed |= this_kid_changed
if any_kid_changed
# De-alias the kids if not already done
if kids′ === kids
kids = eltype(kids)[kids[j] for j in 1:(i - 1)]
end
push!(children′, child′)
push!(kids′, kid′)
end
end
# Reset the siblings
ctx.prev_sibling = prev_sibling
ctx.next_sibling = next_sibling
# Return a new node if any of the children changed
if any_child_changed
return make_node(node, children)
# Return a new node if any of the kids changed
if any_kid_changed
return make_node(node, kids)
else
return nothing
end
@ -175,8 +230,8 @@ Format a node. Return values: @@ -175,8 +230,8 @@ Format a node. Return values:
- `nullnode::NullNode`: The node should be deleted from the tree
- `node::JuliaSyntax.GreenNode`: The node should be replaced with the new node
"""
function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyntax.GreenNode, Nothing, NullNode}
node_kind = JuliaSyntax.kind(node)
function format_node!(ctx::Context, node::Node)::Union{Node, Nothing, NullNode}
node_kind = kind(node)
# Go through the runestone and apply transformations.
@return_something trim_trailing_whitespace(ctx, node)
@ -226,7 +281,7 @@ function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyn @@ -226,7 +281,7 @@ function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyn
node_kind === K"vect"
)
@assert !JuliaSyntax.is_trivia(node)
node′ = format_node_with_children!(ctx, node)
node′ = format_node_with_kids!(ctx, node)
@assert node′ !== nullnode
return node′
@ -265,16 +320,16 @@ function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyn @@ -265,16 +320,16 @@ function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyn
node_kind === K"where" ||
node_kind === K"while"
)
node′ = format_node_with_children!(ctx, node)
node′ = format_node_with_kids!(ctx, node)
@assert node′ !== nullnode
return node′
# Nodes that should recurse if they have children (all??)
elseif JuliaSyntax.haschildren(node) && (
# Nodes that should recurse if they have kids (all??)
elseif !is_leaf(node) && (
JuliaSyntax.is_operator(node) ||
node_kind === K"else" # try-(catch|finally)-else
)
node′ = format_node_with_children!(ctx, node)
node′ = format_node_with_kids!(ctx, node)
@assert node′ !== nullnode
return node′
@ -374,8 +429,8 @@ function format_tree!(ctx::Context) @@ -374,8 +429,8 @@ function format_tree!(ctx::Context)
@assert src_pos == 0
fmt_pos = position(ctx.fmt_io)
@assert fmt_pos == 0
nb = write(ctx.fmt_io, read(ctx.src_io, JuliaSyntax.span(root)))
@assert nb == JuliaSyntax.span(root)
nb = write(ctx.fmt_io, read(ctx.src_io, span(root)))
@assert nb == span(root)
# Reset IOs so that the offsets are correct
seek(ctx.src_io, src_pos)
seek(ctx.fmt_io, fmt_pos)
@ -391,10 +446,10 @@ function format_tree!(ctx::Context) @@ -391,10 +446,10 @@ function format_tree!(ctx::Context)
error("root node deleted")
elseif root′′ === nothing
# root′ = root′′
@assert position(ctx.fmt_io) == fmt_pos + JuliaSyntax.span(root′)
@assert position(ctx.fmt_io) == fmt_pos + span(root′)
break
else
@assert root′′ isa JuliaSyntax.GreenNode
@assert root′′ isa Node
# The node was changed, reset the output stream and try again
seek(ctx.fmt_io, fmt_pos)
root′ = root′′
@ -405,7 +460,7 @@ function format_tree!(ctx::Context) @@ -405,7 +460,7 @@ function format_tree!(ctx::Context)
end
end
# Truncate the output at the root span
truncate(ctx.fmt_io, JuliaSyntax.span(root′))
truncate(ctx.fmt_io, span(root′))
# Set the final tree
ctx.fmt_tree = root′
return nothing

122
src/chisels.jl

@ -1,132 +1,88 @@ @@ -1,132 +1,88 @@
# SPDX-License-Identifier: MIT
##############
# Debug info #
##############
########################################################
# Node utilities extensions and JuliaSyntax extensions #
########################################################
# @lock is defined but not exported in older Julia versions
if VERSION < v"1.7.0"
using Base: @lock
# Create a new node with the same head but new kids
function make_node(node::Node, kids′::Vector{Node})
span′ = mapreduce(span, +, kids′; init = 0)
return Node(head(node), span′, kids′)
end
# Code derived from ToggleableAsserts.jl kept in a separate file
include("ToggleableAsserts.jl")
abstract type RunicException <: Exception end
struct AssertionError <: RunicException
msg::String
end
function Base.showerror(io::IO, err::AssertionError)
print(
io,
"Runic.AssertionError: `", err.msg, "`. This is unexpected, " *
"please file an issue with a reproducible example at " *
"https://github.com/fredrikekre/Runic.jl/issues/new.",
)
end
function macroexpand_assert(expr)
msg = string(expr)
return :($(esc(expr)) || throw(AssertionError($msg)))
end
##########################
# JuliaSyntax extensions #
##########################
# Create a new node with the same head but new children
function make_node(node::JuliaSyntax.GreenNode, children′::AbstractVector{<:JuliaSyntax.GreenNode})
span′ = mapreduce(JuliaSyntax.span, +, children′; init = 0)
return JuliaSyntax.GreenNode(JuliaSyntax.head(node), span′, children′)
end
function is_leaf(node::JuliaSyntax.GreenNode)
return !JuliaSyntax.haschildren(node)
end
function first_leaf(node::JuliaSyntax.GreenNode)
function first_leaf(node::Node)
if is_leaf(node)
return node
else
return first_leaf(first(verified_children(node)))
return first_leaf(first(verified_kids(node)))
end
end
# Return number of non-whitespace children
function n_children(node::JuliaSyntax.GreenNode)
return is_leaf(node) ? 0 : count(!JuliaSyntax.is_whitespace, verified_children(node))
end
# This function exist so that we can type-assert the return value to narrow it down from
# `Union{Tuple{}, Vector{JuliaSyntax.GreenNode}}` to `Vector{JuliaSyntax.GreenNode}`. Must
# only be called after verifying that the node has children.
function verified_children(node::JuliaSyntax.GreenNode)
@assert JuliaSyntax.haschildren(node)
return JuliaSyntax.children(node)::AbstractVector
# Return number of non-whitespace kids, basically the length the equivalent
# (expr::Expr).args
function meta_nargs(node::Node)
return is_leaf(node) ? 0 : count(!JuliaSyntax.is_whitespace, verified_kids(node))
end
function replace_first_leaf(node::JuliaSyntax.GreenNode, child′::JuliaSyntax.GreenNode)
function replace_first_leaf(node::Node, kid′::Node)
if is_leaf(node)
return child′
return kid′
else
children = copy(verified_children(node))
children[1] = replace_first_leaf(children[1], child′)
@assert length(children) > 0
return make_node(node, children)
kids′ = copy(verified_kids(node))
kids′[1] = replace_first_leaf(kids′[1], kid′)
@assert length(kids′) > 0
return make_node(node, kids′)
end
end
function last_leaf(node::JuliaSyntax.GreenNode)
function last_leaf(node::Node)
if is_leaf(node)
return node
else
return last_leaf(last(verified_children(node)))
return last_leaf(last(verified_kids(node)))
end
end
function is_assignment(node::JuliaSyntax.GreenNode)
function is_assignment(node::Node)
return JuliaSyntax.is_prec_assignment(node)
return !is_leaf(node) && JuliaSyntax.is_prec_assignment(node)
# return !is_leaf(node) && JuliaSyntax.is_prec_assignment(node)
end
# Just like `JuliaSyntax.is_infix_op_call`, but also check that the node is K"call"
function is_infix_op_call(node::JuliaSyntax.GreenNode)
return JuliaSyntax.kind(node) === K"call" &&
JuliaSyntax.is_infix_op_call(node)
function is_infix_op_call(node::Node)
return kind(node) === K"call" && JuliaSyntax.is_infix_op_call(node)
end
function infix_op_call_op(node::JuliaSyntax.GreenNode)
# Extract the operator of an infix op call node
function infix_op_call_op(node::Node)
@assert is_infix_op_call(node)
children = verified_children(node)
first_operand_index = findfirst(!JuliaSyntax.is_whitespace, children)
op_index = findnext(JuliaSyntax.is_operator, children, first_operand_index + 1)
return children[op_index]
kids = verified_kids(node)
first_operand_index = findfirst(!JuliaSyntax.is_whitespace, kids)
op_index = findnext(JuliaSyntax.is_operator, kids, first_operand_index + 1)
return kids[op_index]
end
# Comparison leaf or a dotted comparison leaf (.<)
function is_comparison_leaf(node::JuliaSyntax.GreenNode)
function is_comparison_leaf(node::Node)
if is_leaf(node) && JuliaSyntax.is_prec_comparison(node)
return true
elseif !is_leaf(node) && JuliaSyntax.kind(node) === K"." &&
n_children(node) == 2 && is_comparison_leaf(verified_children(node)[2])
elseif !is_leaf(node) && kind(node) === K"." &&
meta_nargs(node) == 2 && is_comparison_leaf(verified_kids(node)[2])
return true
else
return false
end
end
function is_operator_leaf(node::JuliaSyntax.GreenNode)
function is_operator_leaf(node::Node)
return is_leaf(node) && JuliaSyntax.is_operator(node)
end
function first_non_whitespace_child(node::JuliaSyntax.GreenNode)
function first_non_whitespace_kid(node::Node)
@assert !is_leaf(node)
children = verified_children(node)
idx = findfirst(!JuliaSyntax.is_whitespace, children)::Int
return children[idx]
kids = verified_kids(node)
idx = findfirst(!JuliaSyntax.is_whitespace, kids)::Int
return kids[idx]
end
##########################

34
src/debug.jl

@ -0,0 +1,34 @@ @@ -0,0 +1,34 @@
# SPDX-License-Identifier: MIT
##############
# Debug info #
##############
# @lock is defined but not exported in older Julia versions
if VERSION < v"1.7.0"
using Base: @lock
end
# Code derived from ToggleableAsserts.jl kept in a separate file
include("ToggleableAsserts.jl")
abstract type RunicException <: Exception end
struct AssertionError <: RunicException
msg::String
end
function Base.showerror(io::IO, err::AssertionError)
print(
io,
"Runic.AssertionError: `", err.msg, "`. This is unexpected, " *
"please file an issue with a reproducible example at " *
"https://github.com/fredrikekre/Runic.jl/issues/new.",
)
end
function macroexpand_assert(expr)
msg = string(expr)
return :($(esc(expr)) || throw(AssertionError($msg)))
end

368
src/runestone.jl

@ -1,10 +1,14 @@ @@ -1,10 +1,14 @@
# SPDX-License-Identifier: MIT
function dumpnode(node)
println("node: {kind: $(kind(node)), span: $(span(node)), flags: $(flags(node)), nkids: $(length(verified_kids(node)))}")
end
# This is the runestone where all the formatting transformations are implemented.
function trim_trailing_whitespace(ctx::Context, node::JuliaSyntax.GreenNode)
JuliaSyntax.kind(node) === K"NewlineWs" || return nothing
@assert !JuliaSyntax.haschildren(node)
function trim_trailing_whitespace(ctx::Context, node::Node)
kind(node) === K"NewlineWs" || return nothing
@assert is_leaf(node)
str = String(read_bytes(ctx, node))
str′ = replace(str, r"\h*(\r\n|\r|\n)" => '\n')
# If the next sibling is also a NewlineWs we can trim trailing
@ -18,44 +22,44 @@ function trim_trailing_whitespace(ctx::Context, node::JuliaSyntax.GreenNode) @@ -18,44 +22,44 @@ function trim_trailing_whitespace(ctx::Context, node::JuliaSyntax.GreenNode)
return nothing
end
# Write new bytes and reset the stream
nb = replace_bytes!(ctx, str′, JuliaSyntax.span(node))
@assert nb != JuliaSyntax.span(node)
nb = replace_bytes!(ctx, str′, span(node))
@assert nb != span(node)
# Create new node and return it
node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ())
node′ = Node(head(node), nb, ())
return node′
end
function format_hex_literals(ctx::Context, node::JuliaSyntax.GreenNode)
JuliaSyntax.kind(node) === K"HexInt" || return nothing
@assert JuliaSyntax.flags(node) == 0
@assert !JuliaSyntax.haschildren(node)
span = JuliaSyntax.span(node)
@assert span > 2 # 0x prefix + something more
function format_hex_literals(ctx::Context, node::Node)
kind(node) === K"HexInt" || return nothing
@assert flags(node) == 0
@assert is_leaf(node)
spn = span(node)
@assert spn > 2 # 0x prefix + something more
# Target spans(0x + maximum chars for formatted UInt8, UInt16, UInt32, UInt64, UInt128)
target_spans = 2 .+ (2, 4, 8, 16, 32)
if span >= 34 || span in target_spans
if spn >= 34 || spn in target_spans
# Do nothing: correctly formatted or a BigInt hex literal
return nothing
end
# Insert leading zeros
i = findfirst(x -> x > span, target_spans)::Int
i = findfirst(x -> x > spn, target_spans)::Int
bytes = read_bytes(ctx, node)
while length(bytes) < target_spans[i]
insert!(bytes, 3, '0')
end
nb = replace_bytes!(ctx, bytes, span)
nb = replace_bytes!(ctx, bytes, spn)
@assert nb == length(bytes) == target_spans[i]
# Create new node and return it
node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ())
node′ = Node(head(node), nb, ())
return node′
end
function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode)
JuliaSyntax.kind(node) === K"OctInt" || return nothing
@assert JuliaSyntax.flags(node) == 0
@assert !JuliaSyntax.haschildren(node)
span = JuliaSyntax.span(node)
@assert span > 2 # 0o prefix + something more
function format_oct_literals(ctx::Context, node::Node)
kind(node) === K"OctInt" || return nothing
@assert flags(node) == 0
@assert is_leaf(node)
spn = span(node)
@assert spn > 2 # 0o prefix + something more
# Padding depends on the value of the literal...
str = String(read_bytes(ctx, node))
n = tryparse(UInt128, str)
@ -69,10 +73,10 @@ function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode) @@ -69,10 +73,10 @@ function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode)
n <= typemax(UInt32) ? 13 : n <= typemax(UInt64) ? 24 :
n <= typemax(UInt128) ? 45 : error("unreachable")
target_spans = (5, 8, 13, 24, 45)
i = findfirst(x -> x >= span, target_spans)::Int
i = findfirst(x -> x >= spn, target_spans)::Int
target_span_from_source = target_spans[i]
target_span = max(target_span_from_value, target_span_from_source)
if span == target_span
if spn == target_span
# Do nothing: correctly formatted oct literal
return nothing
end
@ -81,17 +85,17 @@ function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode) @@ -81,17 +85,17 @@ function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode)
while length(bytes) < target_span
insert!(bytes, 3, '0')
end
nb = replace_bytes!(ctx, bytes, span)
nb = replace_bytes!(ctx, bytes, spn)
@assert nb == length(bytes) == target_span
# Create new node and return it
node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ())
node′ = Node(head(node), nb, ())
return node′
end
function format_float_literals(ctx::Context, node::JuliaSyntax.GreenNode)
JuliaSyntax.kind(node) in KSet"Float Float32" || return nothing
@assert JuliaSyntax.flags(node) == 0
@assert !JuliaSyntax.haschildren(node)
function format_float_literals(ctx::Context, node::Node)
kind(node) in KSet"Float Float32" || return nothing
@assert flags(node) == 0
@assert is_leaf(node)
str = String(read_bytes(ctx, node))
# Check and shortcut the happy path first
r = r"""
@ -138,129 +142,125 @@ function format_float_literals(ctx::Context, node::JuliaSyntax.GreenNode) @@ -138,129 +142,125 @@ function format_float_literals(ctx::Context, node::JuliaSyntax.GreenNode)
write(io, exp_part)
end
bytes = take!(io)
nb = replace_bytes!(ctx, bytes, JuliaSyntax.span(node))
nb = replace_bytes!(ctx, bytes, span(node))
@assert nb == length(bytes)
# Create new node and return it
node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ())
node′ = Node(head(node), nb, ())
return node′
end
# Insert space around `x`, where `x` can be operators, assignments, etc. with the pattern:
# `<something><space><x><space><something>`, for example the spaces around `+` and `=` in
# `a = x + y`.
function spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) where F
function spaces_around_x(ctx::Context, node::Node, is_x::F) where F
# TODO: So much boilerplate here...
@assert JuliaSyntax.haschildren(node)
@assert !is_leaf(node)
children = verified_children(node)
children′ = children
kids = verified_kids(node)
kids′ = kids
any_changes = false
pos = position(ctx.fmt_io)
ws = JuliaSyntax.GreenNode(
JuliaSyntax.SyntaxHead(K"Whitespace", JuliaSyntax.TRIVIA_FLAG), 1, (),
)
ws = Node(JuliaSyntax.SyntaxHead(K"Whitespace", JuliaSyntax.TRIVIA_FLAG), 1, ())
# Toggle for whether we are currently looking for whitespace or not
looking_for_whitespace = false
looking_for_x = false
for (i, child) in pairs(children)
if JuliaSyntax.kind(child) === K"NewlineWs" ||
(i == 1 && JuliaSyntax.kind(child) === K"Whitespace")
for (i, kid) in pairs(kids)
if kind(kid) === K"NewlineWs" ||
(i == 1 && kind(kid) === K"Whitespace")
# NewlineWs are accepted as is by this pass.
# Whitespace is accepted as is if this is the first child even if the span is
# Whitespace is accepted as is if this is the first kid even if the span is
# larger than we expect since we don't look backwards. It should be cleaned up
# by some other pass.
accept_node!(ctx, child)
any_changes && push!(children′, child)
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
looking_for_whitespace = false
elseif looking_for_whitespace
if JuliaSyntax.kind(child) === K"Whitespace" && JuliaSyntax.span(child) == 1
if kind(kid) === K"Whitespace" && span(kid) == 1
# All good, just advance the IO
accept_node!(ctx, child)
any_changes && push!(children′, child)
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
looking_for_whitespace = false
elseif JuliaSyntax.kind(child) === K"Whitespace"
elseif kind(kid) === K"Whitespace"
# Whitespace node but replace since not single space
any_changes = true
if children′ === children
children′ = children[1:i - 1]
if kids′ === kids
kids′ = kids[1:i - 1]
end
push!(children, ws)
replace_bytes!(ctx, " ", JuliaSyntax.span(child))
push!(kids, ws)
replace_bytes!(ctx, " ", span(kid))
accept_node!(ctx, ws)
looking_for_whitespace = false
elseif JuliaSyntax.haschildren(child) &&
JuliaSyntax.kind(first_leaf(child)) === K"Whitespace"
# Whitespace found at the beginning of next child.
child_ws = first_leaf(child)
looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace"
@assert !is_x(child)::Bool
elseif !is_leaf(kid) && kind(first_leaf(kid)) === K"Whitespace"
# Whitespace found at the beginning of next kid.
kid_ws = first_leaf(kid)
looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace"
@assert !is_x(kid)::Bool
looking_for_x = true
if JuliaSyntax.span(child_ws) == 1
if span(kid_ws) == 1
# Accept the node
accept_node!(ctx, child)
any_changes && push!(children′, child)
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
else
# Replace the whitespace node of the child
child′ = replace_first_leaf(child, ws)
@assert JuliaSyntax.span(child′) == JuliaSyntax.span(child) - JuliaSyntax.span(child_ws) + 1
bytes_to_skip = JuliaSyntax.span(child) - JuliaSyntax.span(child′)
# Replace the whitespace node of the kid
kid′ = replace_first_leaf(kid, ws)
@assert span(kid′) == span(kid) - span(kid_ws) + 1
bytes_to_skip = span(kid) - span(kid′)
@assert bytes_to_skip > 0
replace_bytes!(ctx, "", bytes_to_skip)
accept_node!(ctx, child′)
accept_node!(ctx, kid′)
any_changes = true
if children′ === children
children′ = children[1:i - 1]
if kids′ === kids
kids′ = kids[1:i - 1]
end
push!(children′, child′)
push!(kids′, kid′)
end
elseif JuliaSyntax.haschildren(child) &&
JuliaSyntax.kind(first_leaf(child)) === K"NewlineWs"
elseif !is_leaf(kid) && kind(first_leaf(kid)) === K"NewlineWs"
# NewlineWs have to be accepted as is
# @info " ... childs first leaf is NewlineWs I'll take it"
accept_node!(ctx, child)
any_changes && push!(children′, child)
looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace"
@assert !is_x(child)::Bool
# @info " ... kids first leaf is NewlineWs I'll take it"
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace"
@assert !is_x(kid)::Bool
looking_for_x = true
else
# @info " ... no whitespace, inserting" JuliaSyntax.kind(child)
# @info " ... no whitespace, inserting" kind(kid)
# Not a whitespace node, insert one
any_changes = true
if children′ === children
children′ = children[1:i - 1]
if kids′ === kids
kids′ = kids[1:i - 1]
end
push!(children, ws)
push!(kids, ws)
replace_bytes!(ctx, " ", 0)
accept_node!(ctx, ws)
# Write and accept the node
push!(children′, child)
accept_node!(ctx, child)
looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace"
push!(kids′, kid)
accept_node!(ctx, kid)
looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace"
if looking_for_x
@assert is_x(child)::Bool
@assert is_x(kid)::Bool
end
# Flip the switch, unless child is a comment
looking_for_x = JuliaSyntax.kind(child) === K"Comment" ? looking_for_x : !looking_for_x
# Flip the switch, unless kid is a comment
looking_for_x = kind(kid) === K"Comment" ? looking_for_x : !looking_for_x
end
else # !expect_ws
if looking_for_x
@assert is_x(child)::Bool
@assert is_x(kid)::Bool
end
@assert JuliaSyntax.kind(child) !== K"Whitespace" # This would be weird, I think?
any_changes && push!(children′, child)
accept_node!(ctx, child)
looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace"
# Flip the switch, unless child is a comment
looking_for_x = JuliaSyntax.kind(child) === K"Comment" ? looking_for_x : !looking_for_x
@assert kind(kid) !== K"Whitespace" # This would be weird, I think?
any_changes && push!(kids′, kid)
accept_node!(ctx, kid)
looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace"
# Flip the switch, unless kid is a comment
looking_for_x = kind(kid) === K"Comment" ? looking_for_x : !looking_for_x
end
end
# Reset stream
seek(ctx.fmt_io, pos)
if any_changes
# Create new node and return it
return make_node(node, children)
return make_node(node, kids)
else
return nothing
end
@ -268,39 +268,39 @@ end @@ -268,39 +268,39 @@ end
# This pass handles spaces around infix operator calls, comparison chains, and
# <: and >: operators.
function spaces_around_operators(ctx::Context, node::JuliaSyntax.GreenNode)
function spaces_around_operators(ctx::Context, node::Node)
if !(
(is_infix_op_call(node) && !(JuliaSyntax.kind(infix_op_call_op(node)) in KSet": ^")) ||
(JuliaSyntax.kind(node) in KSet"<: >:" && n_children(node) == 3) ||
(JuliaSyntax.kind(node) === K"comparison" && !JuliaSyntax.is_trivia(node))
(is_infix_op_call(node) && !(kind(infix_op_call_op(node)) in KSet": ^")) ||
(kind(node) in KSet"<: >:" && meta_nargs(node) == 3) ||
(kind(node) === K"comparison" && !JuliaSyntax.is_trivia(node))
)
return nothing
end
@assert JuliaSyntax.kind(node) in KSet"call comparison <: >:"
@assert kind(node) in KSet"call comparison <: >:"
is_x = x -> is_operator_leaf(x) || is_comparison_leaf(x)
return spaces_around_x(ctx, node, is_x)
end
function spaces_around_assignments(ctx::Context, node::JuliaSyntax.GreenNode)
function spaces_around_assignments(ctx::Context, node::Node)
if !(is_assignment(node) && !is_leaf(node) )
return nothing
end
# for-loop nodes are of kind K"=" even when `in` or `∈` is used so we need to
# include these kinds in the predicate too.
is_x = x -> is_assignment(x) || JuliaSyntax.kind(x) in KSet"in ∈"
is_x = x -> is_assignment(x) || kind(x) in KSet"in ∈"
return spaces_around_x(ctx, node, is_x)
end
# Opposite of `spaces_around_x`: remove spaces around `x`
function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) where F
@assert JuliaSyntax.haschildren(node)
function no_spaces_around_x(ctx::Context, node::Node, is_x::F) where F
@assert !is_leaf(node)
# TODO: Can't handle NewlineWs here right now
if any(JuliaSyntax.kind(c) === K"NewlineWs" for c in JuliaSyntax.children(node))
if any(kind(c) === K"NewlineWs" for c in verified_kids(node))
return nothing
end
children = verified_children(node)
children′ = children
kids = verified_kids(node)
kids′ = kids
any_changes = false
pos = position(ctx.fmt_io)
@ -308,28 +308,28 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) @@ -308,28 +308,28 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F)
# K"::", K"<:", and K">:" are special cases here since they can be used without an LHS
# in e.g. `f(::Int) = ...` and `Vector{<:Real}`.
if JuliaSyntax.kind(node) in KSet":: <: >:"
looking_for_x = is_x(first_non_whitespace_child(node))::Bool
if kind(node) in KSet":: <: >:"
looking_for_x = is_x(first_non_whitespace_kid(node))::Bool
end
for (i, child) in pairs(children)
if (i == 1 || i == length(children)) && JuliaSyntax.kind(child) === K"Whitespace"
accept_node!(ctx, child)
any_changes && push!(children′, child)
elseif JuliaSyntax.kind(child) === K"Whitespace"
# Ignore it but need to copy children and re-write bytes
for (i, kid) in pairs(kids)
if (i == 1 || i == length(kids)) && kind(kid) === K"Whitespace"
accept_node!(ctx, kid)
any_changes && push!(kids′, kid)
elseif kind(kid) === K"Whitespace"
# Ignore it but need to copy kids and re-write bytes
any_changes = true
if children′ === children
children′ = children[1:i - 1]
if kids′ === kids
kids′ = kids[1:i - 1]
end
replace_bytes!(ctx, "", JuliaSyntax.span(child))
replace_bytes!(ctx, "", span(kid))
else
@assert JuliaSyntax.kind(child) !== K"Whitespace"
@assert kind(kid) !== K"Whitespace"
if looking_for_x
@assert is_x(child)::Bool
@assert is_x(kid)::Bool
end
any_changes && push!(children′, child)
accept_node!(ctx, child)
any_changes && push!(kids′, kid)
accept_node!(ctx, kid)
looking_for_x = !looking_for_x
end
end
@ -337,8 +337,8 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) @@ -337,8 +337,8 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F)
seek(ctx.fmt_io, pos)
if any_changes
# Create new node and return it
node′ = make_node(node, children)
@assert JuliaSyntax.span(node′) < JuliaSyntax.span(node)
node′ = make_node(node, kids)
@assert span(node′) < span(node)
return node′
else
return nothing
@ -346,127 +346,127 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) @@ -346,127 +346,127 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F)
end
# no spaces around `:`, `^`, and `::`
function no_spaces_around_colon_etc(ctx::Context, node::JuliaSyntax.GreenNode)
function no_spaces_around_colon_etc(ctx::Context, node::Node)
if !(
(is_infix_op_call(node) && JuliaSyntax.kind(infix_op_call_op(node)) in KSet": ^") ||
(JuliaSyntax.kind(node) === K"::" && !is_leaf(node)) ||
(JuliaSyntax.kind(node) in KSet"<: >:" && n_children(node) == 2)
(is_infix_op_call(node) && kind(infix_op_call_op(node)) in KSet": ^") ||
(kind(node) === K"::" && !is_leaf(node)) ||
(kind(node) in KSet"<: >:" && meta_nargs(node) == 2)
)
return nothing
end
@assert JuliaSyntax.kind(node) in KSet"call :: <: >:"
is_x = x -> is_leaf(x) && JuliaSyntax.kind(x) in KSet": ^ :: <: >:"
@assert kind(node) in KSet"call :: <: >:"
is_x = x -> is_leaf(x) && kind(x) in KSet": ^ :: <: >:"
return no_spaces_around_x(ctx, node, is_x)
end
# Replace the K"=" operator with `in`
function replace_with_in(ctx::Context, node::JuliaSyntax.GreenNode)
@assert JuliaSyntax.kind(node) === K"=" && !is_leaf(node) && n_children(node) == 3
children = verified_children(node)
vars_index = findfirst(!JuliaSyntax.is_whitespace, children)
function replace_with_in(ctx::Context, node::Node)
@assert kind(node) === K"=" && !is_leaf(node) && meta_nargs(node) == 3
kids = verified_kids(node)
vars_index = findfirst(!JuliaSyntax.is_whitespace, kids)
# TODO: Need to insert whitespaces around `in` when replacing e.g. `i=I` with `iinI`.
# However, at the moment it looks like the whitespace around operator pass does it's
# thing first? I don't really know how though, because the for loop pass should be
# happening before...
in_index = findnext(!JuliaSyntax.is_whitespace, children, vars_index + 1)
in_node = children[in_index]
if JuliaSyntax.kind(in_node) === K"in"
in_index = findnext(!JuliaSyntax.is_whitespace, kids, vars_index + 1)
in_node = kids[in_index]
if kind(in_node) === K"in"
@assert JuliaSyntax.is_trivia(in_node)
@assert is_leaf(in_node)
return nothing
end
@assert JuliaSyntax.kind(in_node) in KSet"∈ ="
@assert kind(in_node) in KSet"∈ ="
@assert JuliaSyntax.is_trivia(in_node)
@assert is_leaf(in_node)
# Accept nodes to advance the stream
for i in 1:(in_index - 1)
accept_node!(ctx, children[i])
accept_node!(ctx, kids[i])
end
# Construct the replacement
nb = replace_bytes!(ctx, "in", JuliaSyntax.span(in_node))
in_node′ = JuliaSyntax.GreenNode(
nb = replace_bytes!(ctx, "in", span(in_node))
in_node′ = Node(
JuliaSyntax.SyntaxHead(K"in", JuliaSyntax.TRIVIA_FLAG), nb, (),
)
accept_node!(ctx, in_node′)
children = copy(children)
children[in_index] = in_node′
# Accept remaining eq_children
for i in (in_index + 1):length(children)
accept_node!(ctx, children[i])
kids = copy(kids)
kids[in_index] = in_node′
# Accept remaining kids
for i in (in_index + 1):length(kids)
accept_node!(ctx, kids[i])
end
return make_node(node, children)
return make_node(node, kids)
end
function replace_with_in_cartesian(ctx::Context, node::JuliaSyntax.GreenNode)
@assert JuliaSyntax.kind(node) === K"cartesian_iterator" && !is_leaf(node)
children = verified_children(node)
children′ = children
for (i, child) in pairs(children)
if JuliaSyntax.kind(child) === K"="
child′ = replace_with_in(ctx, child)
if child′ !== nothing
if children′ === children
children = copy(children)
end
children[i] = child′
function replace_with_in_cartesian(ctx::Context, node::Node)
@assert kind(node) === K"cartesian_iterator" && !is_leaf(node)
kids = verified_kids(node)
kids′ = kids
for (i, kid) in pairs(kids)
if kind(kid) === K"="
kid′ = replace_with_in(ctx, kid)
if kid′ !== nothing
if kids′ === kids
kids = copy(kids)
end
kids[i] = kid′
else
children[i] = child
accept_node!(ctx, child)
kids[i] = kid
accept_node!(ctx, kid)
end
else
children[i] = child
accept_node!(ctx, child)
kids[i] = kid
accept_node!(ctx, kid)
end
end
if children === children
if kids === kids
return nothing
end
return make_node(node, children)
return make_node(node, kids)
end
# replace `=` and `∈` with `in` in for-loops
function for_loop_use_in(ctx::Context, node::JuliaSyntax.GreenNode)
function for_loop_use_in(ctx::Context, node::Node)
if !(
(JuliaSyntax.kind(node) === K"for" && !is_leaf(node) && n_children(node) == 4) ||
(JuliaSyntax.kind(node) === K"generator" && n_children(node) == 3) # TODO: Unsure about 3.
(kind(node) === K"for" && !is_leaf(node) && meta_nargs(node) == 4) ||
(kind(node) === K"generator" && meta_nargs(node) == 3) # TODO: Unsure about 3.
)
return nothing
end
pos = position(ctx.fmt_io)
children = verified_children(node)
for_index = findfirst(c -> JuliaSyntax.kind(c) === K"for" && is_leaf(c), children)::Int
for_node = children[for_index]
@assert JuliaSyntax.kind(for_node) === K"for" && JuliaSyntax.span(for_node) == 3 &&
kids = verified_kids(node)
for_index = findfirst(c -> kind(c) === K"for" && is_leaf(c), kids)::Int
for_node = kids[for_index]
@assert kind(for_node) === K"for" && span(for_node) == 3 &&
is_leaf(for_node) && JuliaSyntax.is_trivia(for_node)
for i in 1:for_index
accept_node!(ctx, children[i])
accept_node!(ctx, kids[i])
end
# The for loop specification node can be either K"=" or K"cartesian_iterator"
for_spec_index = for_index + 1
for_spec_node = children[for_spec_index]
@assert JuliaSyntax.kind(for_spec_node) in KSet"= cartesian_iterator"
if JuliaSyntax.kind(for_spec_node) === K"="
for_spec_node = kids[for_spec_index]
@assert kind(for_spec_node) in KSet"= cartesian_iterator"
if kind(for_spec_node) === K"="
for_spec_node′ = replace_with_in(ctx, for_spec_node)
else
@assert JuliaSyntax.kind(for_spec_node) === K"cartesian_iterator"
@assert kind(for_spec_node) === K"cartesian_iterator"
for_spec_node′ = replace_with_in_cartesian(ctx, for_spec_node)
end
if for_spec_node′ === nothing
seek(ctx.fmt_io, pos)
return nothing
end
@assert position(ctx.fmt_io) == pos + mapreduce(JuliaSyntax.span, +, @view(children[1:for_index])) + JuliaSyntax.span(for_spec_node′)
@assert position(ctx.fmt_io) == pos + mapreduce(span, +, @view(kids[1:for_index])) + span(for_spec_node′)
# Insert the new for spec node
children = copy(children)
children[for_spec_index] = for_spec_node′
kids = copy(kids)
kids[for_spec_index] = for_spec_node′
# At this point the eq node is done, just accept any remaining nodes
# TODO: Don't need to do this...
for i in (for_spec_index + 1):length(children)
accept_node!(ctx, children[i])
for i in (for_spec_index + 1):length(kids)
accept_node!(ctx, kids[i])
end
# Construct the full node and return
node′ = make_node(node, children)
@assert position(ctx.fmt_io) == pos + JuliaSyntax.span(node′)
node′ = make_node(node, kids)
@assert position(ctx.fmt_io) == pos + span(node′)
seek(ctx.fmt_io, pos) # reset
return node′
end

14
test/runtests.jl

@ -8,9 +8,17 @@ using JuliaSyntax: @@ -8,9 +8,17 @@ using JuliaSyntax:
JuliaSyntax
@testset "Chisels" begin
# Type stability of verified_children
node = JuliaSyntax.parseall(JuliaSyntax.GreenNode, "a = 1 + b\n")
@test typeof(@inferred Runic.verified_children(node)) <: Vector{<:JuliaSyntax.GreenNode}
# Type stability of verified_kids
node = Runic.Node(JuliaSyntax.parseall(JuliaSyntax.GreenNode, "a = 1 + b\n"))
@test typeof(@inferred Runic.verified_kids(node)) === Vector{Runic.Node}
# JuliaSyntax duck-typing
for n in (node, Runic.verified_kids(node)...,)
@test Runic.head(n) === JuliaSyntax.head(n) === n.head
@test Runic.kind(n) === JuliaSyntax.kind(n) === n.head.kind
@test Runic.flags(n) === JuliaSyntax.flags(n) === n.head.flags
@test Runic.span(n) === JuliaSyntax.span(n) === n.span
end
# replace_bytes!: insert larger
io = IOBuffer(); write(io, "abc"); seek(io, 1)

Loading…
Cancel
Save