r
/
Literate.jl
mirror of https://github.com/fredrikekre/Literate.jl
1
0
Fork 0
Code Releases Activity
Browse Source

fix stuff

pull/220/head
Simon Christ 2 years ago
parent
17ba06b808
commit
8f0468303f
1 changed files with 96 additions and 140 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 236
      src/Literate.jl

236
src/Literate.jl
Unescape View File

@ -16,7 +16,7 @@ struct DefaultFlavor <: AbstractFlavor end
struct DocumenterFlavor <: AbstractFlavor end struct DocumenterFlavor <: AbstractFlavor end
struct CommonMarkFlavor <: AbstractFlavor end struct CommonMarkFlavor <: AbstractFlavor end
struct FranklinFlavor <: AbstractFlavor end struct FranklinFlavor <: AbstractFlavor end
struct CarpentriesFlavor <: AbstractFlavor end struct CarpentriesFlavor <: AbstractFlavor end
# # Some simple rules: # # Some simple rules:
# #
@ -46,7 +46,7 @@ CodeChunk() = CodeChunk(String[], false)
ismdline(line) = (occursin(r"^\h*#$", line) || occursin(r"^\h*# .*$", line)) && !occursin(r"^\h*##", line) ismdline(line) = (occursin(r"^\h*#$", line) || occursin(r"^\h*# .*$", line)) && !occursin(r"^\h*##", line)
function parse(content; allow_continued = true) function parse(content; allow_continued=true)
lines = collect(eachline(IOBuffer(content))) lines = collect(eachline(IOBuffer(content)))
chunks = Chunk[] chunks = Chunk[]
@ -124,10 +124,10 @@ function parse(content; allow_continued = true)
end end
function replace_default(content, sym; function replace_default(content, sym;
config::Dict, config::Dict,
branch = "gh-pages", branch="gh-pages",
commit = "master" commit="master"
) )
repls = Pair{Any,Any}[] repls = Pair{Any,Any}[]
# add some shameless advertisement # add some shameless advertisement
@ -227,7 +227,7 @@ filename(str) = first(splitext(last(splitdir(str))))
isdocumenter(cfg) = cfg["flavor"]::AbstractFlavor isa DocumenterFlavor isdocumenter(cfg) = cfg["flavor"]::AbstractFlavor isa DocumenterFlavor
_DEFAULT_IMAGE_FORMATS = [(MIME("image/svg+xml"), ".svg"), (MIME("image/png"), ".png"), _DEFAULT_IMAGE_FORMATS = [(MIME("image/svg+xml"), ".svg"), (MIME("image/png"), ".png"),
(MIME("image/jpeg"), ".jpeg")] (MIME("image/jpeg"), ".jpeg")]
# Cache of inputfile => head branch # Cache of inputfile => head branch
const HEAD_BRANCH_CACHE = Dict{String,String}() const HEAD_BRANCH_CACHE = Dict{String,String}()
@ -245,7 +245,7 @@ function edit_commit(inputfile, user_config)
readchomp( readchomp(
pipeline( pipeline(
setenv(`$(git) rev-parse --show-toplevel`; dir=dirname(inputfile)); setenv(`$(git) rev-parse --show-toplevel`; dir=dirname(inputfile));
stderr=devnull, stderr=devnull
) )
) )
catch catch
@ -291,15 +291,15 @@ function create_configuration(inputfile; user_config, user_kwargs, type=nothing)
if (d = get(user_config, "documenter", nothing); d !== nothing) if (d = get(user_config, "documenter", nothing); d !== nothing)
if type === :md if type === :md
Base.depwarn("The documenter=$(d) keyword to Literate.markdown is deprecated." * Base.depwarn("The documenter=$(d) keyword to Literate.markdown is deprecated." *
" Pass `flavor = Literate.$(d ? "DocumenterFlavor" : "CommonMarkFlavor")()`" * " Pass `flavor = Literate.$(d ? "DocumenterFlavor" : "CommonMarkFlavor")()`" *
" instead.", Symbol("Literate.markdown")) " instead.", Symbol("Literate.markdown"))
user_config["flavor"] = d ? DocumenterFlavor() : CommonMarkFlavor() user_config["flavor"] = d ? DocumenterFlavor() : CommonMarkFlavor()
elseif type === :nb elseif type === :nb
Base.depwarn("The documenter=$(d) keyword to Literate.notebook is deprecated." * Base.depwarn("The documenter=$(d) keyword to Literate.notebook is deprecated." *
" It is not used anymore for notebook output.", Symbol("Literate.notebook")) " It is not used anymore for notebook output.", Symbol("Literate.notebook"))
elseif type === :jl elseif type === :jl
Base.depwarn("The documenter=$(d) keyword to Literate.script is deprecated." * Base.depwarn("The documenter=$(d) keyword to Literate.script is deprecated." *
" It is not used anymore for script output.", Symbol("Literate.script")) " It is not used anymore for script output.", Symbol("Literate.script"))
end end
end end
@ -431,7 +431,7 @@ Available options:
`$(_DEFAULT_IMAGE_FORMATS)`. Results which are `showable` with a MIME type are saved with `$(_DEFAULT_IMAGE_FORMATS)`. Results which are `showable` with a MIME type are saved with
the first match, with the corresponding extension. the first match, with the corresponding extension.
""" """
const DEFAULT_CONFIGURATION=nothing # Dummy const for documentation const DEFAULT_CONFIGURATION = nothing # Dummy const for documentation
function preprocessor(inputfile, outputdir; user_config, user_kwargs, type) function preprocessor(inputfile, outputdir; user_config, user_kwargs, type)
# Create configuration by merging default and userdefined # Create configuration by merging default and userdefined
@ -485,7 +485,7 @@ function preprocessor(inputfile, outputdir; user_config, user_kwargs, type)
content = replace_default(content, type; config=config) content = replace_default(content, type; config=config)
# parse the content into chunks # parse the content into chunks
chunks = parse(content; allow_continued = type !== :nb) chunks = parse(content; allow_continued=type !== :nb)
return chunks, config return chunks, config
end end
@ -572,7 +572,7 @@ function processNonAdmonitions(item, io)
write(io, string(Markdown.MD(item))) write(io, string(Markdown.MD(item)))
end end
function writeContent(mdContent, io) function writeContent(mdContent, io)
for item in mdContent for item in mdContent
if isa(item, Markdown.Admonition) if isa(item, Markdown.Admonition)
CarpentriesAdmonition(item, io) CarpentriesAdmonition(item, io)
@ -592,7 +592,7 @@ function CarpentriesCallout(admonition, io)
write(io, ":::::::: callout", '\n') write(io, ":::::::: callout", '\n')
else else
if line != "" if line != ""
write(io, line, '\n') write(io, line, '\n')
end end
end end
end end
@ -648,13 +648,13 @@ function CarpentriesAdmonition(admonition, io)
category = admonition.category category = admonition.category
admonition = split(string(Markdown.MD(admonition)), '\n') admonition = split(string(Markdown.MD(admonition)), '\n')
if category in ("sc", "mc", "freecode") if category in ("sc", "mc", "freecode")
CarpentriesChallenge(admonition, io) CarpentriesChallenge(admonition, io)
elseif category == "tip" elseif category == "tip"
CarpentriesTestamonial(admonition, io) CarpentriesTestamonial(admonition, io)
elseif category == "warning" elseif category == "warning"
CarpentriesWarning(admonition, io) CarpentriesWarning(admonition, io)
elseif category in ("info", "note") elseif category in ("info", "note")
CarpentriesCallout(admonition, io) CarpentriesCallout(admonition, io)
elseif category == "carp" elseif category == "carp"
CarpentriesYAML(admonition, io) CarpentriesYAML(admonition, io)
end end
@ -676,135 +676,91 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg
# preprocessing and parsing # preprocessing and parsing
chunks, config = chunks, config =
preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:md) preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:md)
flavor=config["flavor"]
# create the markdown file # create the markdown file
sb = sandbox()
iomd = IOBuffer() iomd = IOBuffer()
if flavor isa CarpentriesFlavor
for chunk in chunks
if isa(chunk, MDChunk)
if containsAdmonition(chunk)
str = chunkToMD(chunk)
mdContent = str.content
writeContent(mdContent, iomd)
else write_md_chunks!(iomd, chunks, outputdir, config)
# Handle chunks without admonitions
for line in chunk.lines
write(iomd, line.second, '\n') # Skip indent
end
end
## `else` is copied from vanilla:
else # isa(chunk, CodeChunk)
iocode = IOBuffer()
codefence = config["codefence"]::Pair
write(iocode, codefence.first)
# make sure the code block is finalized if we are printing to ```@example
# (or ````@example, any number of backticks >= 3 works)
if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config)
write(iocode, "; continued = true")
end
write(iocode, '\n')
# filter out trailing #hide unless code is executed by Documenter
execute = config["execute"]::Bool
write_hide = isdocumenter(config) && !execute
write_line(line) = write_hide || !endswith(line, "#hide")
for line in chunk.lines
write_line(line) && write(iocode, line, '\n')
end
if write_hide && REPL.ends_with_semicolon(chunk.lines[end])
write(iocode, "nothing #hide\n")
end
write(iocode, codefence.second, '\n')
any(write_line, chunk.lines) && write(iomd, seekstart(iocode))
if execute
cd(config["literate_outputdir"]) do
execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir;
inputfile=config["literate_inputfile"],
fake_source=config["literate_outputfile"],
flavor=config["flavor"],
image_formats=config["image_formats"],
file_prefix="$(config["name"])-$(chunknum)",
)
end
end
end
end
else
# kinda Vanilla Function
for (chunknum, chunk) in enumerate(chunks)
if isa(chunk, MDChunk)
#______________________________________________________________________________________________________________
# custom post-processing from user
content = config["postprocess"](String(take!(iomd)))
# write to file
outputfile = write_result(content, config)
return outputfile
end
function write_md_chunks!(iomd, chunks, outputdir, config)
flavor = config["flavor"]
sb = sandbox()
for (chunknum, chunk) in enumerate(chunks)
if isa(chunk, MDChunk)
#______________________________________________________________________________________________________________
if flavor isa CarpentriesFlavor
if containsAdmonition(chunk)
writeContent(chunk.lines, iomd)
end
else
if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD.
continue continue
end end
#______________________________________________________________________________________________________________ end
#______________________________________________________________________________________________________________
for line in chunk.lines for line in chunk.lines
write(iomd, line.second, '\n') # skip indent here write(iomd, line.second, '\n') # skip indent here
end end
else # isa(chunk, CodeChunk) else # isa(chunk, CodeChunk)
iocode = IOBuffer() iocode = IOBuffer()
codefence = config["codefence"]::Pair codefence = config["codefence"]::Pair
write(iocode, codefence.first) write(iocode, codefence.first)
# make sure the code block is finalized if we are printing to ```@example # make sure the code block is finalized if we are printing to ```@example
# (or ````@example, any number of backticks >= 3 works) # (or ````@example, any number of backticks >= 3 works)
if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config)
write(iocode, "; continued = true") write(iocode, "; continued = true")
end end
write(iocode, '\n') write(iocode, '\n')
# filter out trailing #hide unless code is executed by Documenter # filter out trailing #hide unless code is executed by Documenter
execute = config["execute"]::Bool execute = config["execute"]::Bool
write_hide = isdocumenter(config) && !execute write_hide = isdocumenter(config) && !execute
write_line(line) = write_hide || !endswith(line, "#hide") write_line(line) = write_hide || !endswith(line, "#hide")
for line in chunk.lines for line in chunk.lines
write_line(line) && write(iocode, line, '\n') write_line(line) && write(iocode, line, '\n')
end end
if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) if write_hide && REPL.ends_with_semicolon(chunk.lines[end])
write(iocode, "nothing #hide\n") write(iocode, "nothing #hide\n")
end end
write(iocode, codefence.second, '\n') write(iocode, codefence.second, '\n')
any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) any(write_line, chunk.lines) && write(iomd, seekstart(iocode))
if execute if execute
cd(config["literate_outputdir"]) do cd(config["literate_outputdir"]) do
execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir;
inputfile=config["literate_inputfile"], inputfile=config["literate_inputfile"],
fake_source=config["literate_outputfile"], fake_source=config["literate_outputfile"],
flavor=config["flavor"], flavor=config["flavor"],
image_formats=config["image_formats"], image_formats=config["image_formats"],
file_prefix="$(config["name"])-$(chunknum)", file_prefix="$(config["name"])-$(chunknum)"
) )
end
end end
end end
end end
write(iomd, '\n') # add a newline between each chunk
end end
write(iomd, '\n') # add a newline between each chunk end
# custom post-processing from user
content = config["postprocess"](String(take!(iomd)))
# write to file
outputfile = write_result(content, config)
return outputfile
end
function execute_markdown!(io::IO, sb::Module, block::String, outputdir; function execute_markdown!(io::IO, sb::Module, block::String, outputdir;
inputfile::String, fake_source::String, inputfile::String, fake_source::String,
flavor::AbstractFlavor, image_formats::Vector, file_prefix::String) flavor::AbstractFlavor, image_formats::Vector, file_prefix::String)
# TODO: Deal with explicit display(...) calls # TODO: Deal with explicit display(...) calls
r, str, _ = execute_block(sb, block; inputfile=inputfile, fake_source=fake_source) r, str, _ = execute_block(sb, block; inputfile=inputfile, fake_source=fake_source)
# issue #101: consecutive codefenced blocks need newline # issue #101: consecutive codefenced blocks need newline
# issue #144: quadruple backticks allow for triple backticks in the output # issue #144: quadruple backticks allow for triple backticks in the output
plain_fence = "\n````\n" => "\n````" plain_fence = "\n````\n" => "\n````"
# Here CarpentiresFlavor fork... # Here CarpentiresFlavor fork...
if r !== nothing && !REPL.ends_with_semicolon(block) if r !== nothing && !REPL.ends_with_semicolon(block)
if (flavor isa FranklinFlavor || flavor isa DocumenterFlavor) && if (flavor isa FranklinFlavor || flavor isa DocumenterFlavor) &&
@ -851,8 +807,8 @@ function parse_nbmeta(line)
# Cf. https://jupytext.readthedocs.io/en/latest/formats.html#the-percent-format # Cf. https://jupytext.readthedocs.io/en/latest/formats.html#the-percent-format
m = match(r"^%% ([^[{]+)?\s*(?:\[(\w+)\])?\s*(\{.*)?$", line) m = match(r"^%% ([^[{]+)?\s*(?:\[(\w+)\])?\s*(\{.*)?$", line)
typ = m.captures[2] typ = m.captures[2]
name = m.captures[1] === nothing ? Dict{String, String}() : Dict("name" => m.captures[1]) name = m.captures[1] === nothing ? Dict{String,String}() : Dict("name" => m.captures[1])
meta = m.captures[3] === nothing ? Dict{String, Any}() : JSON.parse(m.captures[3]) meta = m.captures[3] === nothing ? Dict{String,Any}() : JSON.parse(m.captures[3])
return typ, merge(name, meta) return typ, merge(name, meta)
end end
line_is_nbmeta(line::Pair) = line_is_nbmeta(line.second) line_is_nbmeta(line::Pair) = line_is_nbmeta(line.second)
@ -875,7 +831,7 @@ function notebook(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg
nb = jupyter_notebook(chunks, config) nb = jupyter_notebook(chunks, config)
# write to file # write to file
outputfile = write_result(nb, config; print = (io, c)->JSON.print(io, c, 1)) outputfile = write_result(nb, config; print=(io, c) -> JSON.print(io, c, 1))
return outputfile return outputfile
end end
@ -894,11 +850,11 @@ function jupyter_notebook(chunks, config)
metatype !== nothing && metatype != chunktype && error("specifying a different cell type is not supported") metatype !== nothing && metatype != chunktype && error("specifying a different cell type is not supported")
popfirst!(chunk.lines) popfirst!(chunk.lines)
else else
metadata = Dict{String, Any}() metadata = Dict{String,Any}()
end end
lines = isa(chunk, MDChunk) ? lines = isa(chunk, MDChunk) ?
String[x.second for x in chunk.lines] : # skip indent String[x.second for x in chunk.lines] : # skip indent
chunk.lines chunk.lines
@views map!(x -> x * '\n', lines[1:end-1], lines[1:end-1]) @views map!(x -> x * '\n', lines[1:end-1], lines[1:end-1])
cell["cell_type"] = chunktype cell["cell_type"] = chunktype
cell["metadata"] = metadata cell["metadata"] = metadata
@ -915,15 +871,15 @@ function jupyter_notebook(chunks, config)
metadata = Dict() metadata = Dict()
kernelspec = Dict() kernelspec = Dict()
kernelspec["language"] = "julia" kernelspec["language"] = "julia"
kernelspec["name"] = "julia-$(VERSION.major).$(VERSION.minor)" kernelspec["name"] = "julia-$(VERSION.major).$(VERSION.minor)"
kernelspec["display_name"] = "Julia $(string(VERSION))" kernelspec["display_name"] = "Julia $(string(VERSION))"
metadata["kernelspec"] = kernelspec metadata["kernelspec"] = kernelspec
language_info = Dict() language_info = Dict()
language_info["file_extension"] = ".jl" language_info["file_extension"] = ".jl"
language_info["mimetype"] = "application/julia" language_info["mimetype"] = "application/julia"
language_info["name"]= "julia" language_info["name"] = "julia"
language_info["version"] = string(VERSION) language_info["version"] = string(VERSION)
metadata["language_info"] = language_info metadata["language_info"] = language_info
@ -937,7 +893,7 @@ function jupyter_notebook(chunks, config)
try try
cd(config["literate_outputdir"]) do cd(config["literate_outputdir"]) do
nb = execute_notebook(nb; inputfile=config["literate_inputfile"], nb = execute_notebook(nb; inputfile=config["literate_inputfile"],
fake_source=config["literate_outputfile"]) fake_source=config["literate_outputfile"])
end end
catch err catch err
@error "error when executing notebook based on input file: " * @error "error when executing notebook based on input file: " *
@ -963,7 +919,7 @@ function execute_notebook(nb; inputfile::String, fake_source::String)
stream = Dict{String,Any}() stream = Dict{String,Any}()
stream["output_type"] = "stream" stream["output_type"] = "stream"
stream["name"] = "stdout" stream["name"] = "stdout"
stream["text"] = collect(Any, eachline(IOBuffer(String(str)), keep = true)) stream["text"] = collect(Any, eachline(IOBuffer(String(str)), keep=true))
push!(cell["outputs"], stream) push!(cell["outputs"], stream)
end end
@ -973,7 +929,7 @@ function execute_notebook(nb; inputfile::String, fake_source::String)
function split_mime(dict) function split_mime(dict)
for mime in ("image/svg+xml", "text/html") for mime in ("image/svg+xml", "text/html")
if haskey(dict, mime) if haskey(dict, mime)
dict[mime] = collect(Any, eachline(IOBuffer(dict[mime]), keep = true)) dict[mime] = collect(Any, eachline(IOBuffer(dict[mime]), keep=true))
end end
end end
return dict return dict
@ -1053,7 +1009,7 @@ function execute_block(sb::Module, block::String; inputfile::String, fake_source
# - c.output: combined stdout and stderr # - c.output: combined stdout and stderr
# `rethrow = Union{}` means that we try-catch all the exceptions thrown in the do-block # `rethrow = Union{}` means that we try-catch all the exceptions thrown in the do-block
# and return them via the return value (they get handled below). # and return them via the return value (they get handled below).
c = IOCapture.capture(rethrow = Union{}) do c = IOCapture.capture(rethrow=Union{}) do
include_string(sb, block, fake_source) include_string(sb, block, fake_source)
end end
popdisplay(disp) # IOCapture.capture has a try-catch so should always end up here popdisplay(disp) # IOCapture.capture has a try-catch so should always end up here
@ -1070,4 +1026,4 @@ function execute_block(sb::Module, block::String; inputfile::String, fake_source
return c.value, c.output, disp.data return c.value, c.output, disp.data
end end
end # module end # module

Write Preview
Loading…
Cancel
Save