From b3e19a1d73a2a67622dae40cb65d2ebc86dbe132 Mon Sep 17 00:00:00 2001 From: Myrkwid Date: Wed, 12 Apr 2023 13:25:33 +0200 Subject: [PATCH 01/24] Added flavor tag. --- src/Literate.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Literate.jl b/src/Literate.jl index f511898..6c63cec 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -16,6 +16,7 @@ struct DefaultFlavor <: AbstractFlavor end struct DocumenterFlavor <: AbstractFlavor end struct CommonMarkFlavor <: AbstractFlavor end struct FranklinFlavor <: AbstractFlavor end +struct Carpentries <: AbstractFlavor end # # Some simple rules: # From 5aaa53b44a75e83bf6f6d16d78abbb20d6922b60 Mon Sep 17 00:00:00 2001 From: Myrkwid Date: Wed, 12 Apr 2023 13:33:17 +0200 Subject: [PATCH 02/24] Added Markdown as an used package. --- src/Literate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Literate.jl b/src/Literate.jl index 6c63cec..8a3167b 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -6,7 +6,7 @@ https://fredrikekre.github.io/Literate.jl/ for documentation. """ module Literate -import JSON, REPL, IOCapture +import JSON, REPL, IOCapture, Markdown include("IJulia.jl") import .IJulia From abf6164de2b4a302a4d0f4fb62be2e6799bba360 Mon Sep 17 00:00:00 2001 From: Myrkwid Date: Tue, 25 Jul 2023 19:31:28 +0200 Subject: [PATCH 03/24] Fuck, I forgot to commit after starting new... --- src/Literate.jl | 226 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 191 insertions(+), 35 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 8a3167b..9b11609 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -16,7 +16,7 @@ struct DefaultFlavor <: AbstractFlavor end struct DocumenterFlavor <: AbstractFlavor end struct CommonMarkFlavor <: AbstractFlavor end struct FranklinFlavor <: AbstractFlavor end -struct Carpentries <: AbstractFlavor end +struct CarpentriesFlavor <: AbstractFlavor end # # Some simple rules: # @@ -499,6 +499,104 @@ function write_result(content, config; print=print) return outputfile end +# Define general functions needed for admonitions formating. + +function containsAdmonition(chunk) + if startswith(strip(line.first * line.second), "!!!") + return true + end + end + return false +end + +function chunkToMD(chunk) + buffer = IOBuffer() + for line in chunk.lines + write(buffer, line.first * line.second, '\n') + end + seek(buffer, 0) + return Markdown.parse(read(buffer, String)) +end + +function processNonAdmonitions(item, io) + # Handle non-admonition elements + return string(Markdown.MD(item)) +end + +function writeContent(mdContent, io) + for item in mdContent + if isa(item, Markdown.Admonition) + result = CarpentriesAdmonition(item, io) + else + result = processNonAdmonitions(item, io) + end + write(io, result, '\n') + end +end + + +#Functions needed for addition transformation into Carpentries style. Markdown style into pandoc fenced divs + +function CarpentriesCallout(admonition, io) + for line in admonition + if startswith(strip(line.first * line.second), "!!!") + write(io, ":::::::: callout", '\n') + else + write(io, line, '\n') + end + end + write(io, "::::::::", '\n') +end + +function CarpentriesTestamonial(admonition, io) + for line in admonition + if startswith(strip(line.first * line.second), "!!!") + write(io, ":::::::: testamonial", '\n') + else + write(io, line, '\n') + end + end + write(io, "::::::::", '\n') +end + +function CarpentriesChallenge(admonition, io) + for line in admonition + if startswith(strip(line.first * line.second), r"!!! [smf][cr]") + write(io, ":::::::: challenge", '\n') + elseif startswith(strip(line.first * line.second), "!!! solution") + write(io, ":::::::: solution", '\n') + else + write(io, line, '\n') + end + end + write(io, "::::::::", '\n', "::::::::", '\n') +end + +function CarpentriesWarning(admonition, io) + for line in admonition + if startswith(strip(line.first * line.second), "!!!") + write(io, ":::::::: warning", '\n') + else + write(io, line, '\n') + end + end + write(io, "::::::::", '\n') +end + +function CarpentriesAdmonition(admonition, io) + category = admonition.category + + if category == "sc" || "mc" || "freecode" + return CarpentriesChallenge(admonition, io) + elseif category == "tip" + return CarpentriesTestamonial(admonition, io) + elseif category == "warning" + return CarpentriesWarning(admonition, io) + elseif category == "info" || "note" + return CarpentriesCallout(admonition, io) + end +end + """ Literate.script(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwargs...) @@ -550,48 +648,102 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg # preprocessing and parsing chunks, config = preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:md) - + flavor=config["flavor"] # create the markdown file sb = sandbox() iomd = IOBuffer() - for (chunknum, chunk) in enumerate(chunks) - if isa(chunk, MDChunk) - for line in chunk.lines - write(iomd, line.second, '\n') # skip indent here - end - else # isa(chunk, CodeChunk) - iocode = IOBuffer() - codefence = config["codefence"]::Pair - write(iocode, codefence.first) - # make sure the code block is finalized if we are printing to ```@example - # (or ````@example, any number of backticks >= 3 works) - if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) - write(iocode, "; continued = true") - end - write(iocode, '\n') - # filter out trailing #hide unless code is executed by Documenter - execute = config["execute"]::Bool - write_hide = isdocumenter(config) && !execute - write_line(line) = write_hide || !endswith(line, "#hide") - for line in chunk.lines - write_line(line) && write(iocode, line, '\n') - end - if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) - write(iocode, "nothing #hide\n") - end - write(iocode, codefence.second, '\n') - any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) - if execute - cd(config["literate_outputdir"]) do - execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; + + for chunk in chunks + io = IOBuffer() + if flavor isa CarpentriesFlavor + if isa(chunk, MDChunk) + if containsAdmonition(chunk) + + str = chunkToMD(chunk) + mdContent = str.content + + writeContent(mdContent, io) + + else + # Handle chunks without admonitions + for line in chunk.lines + write(io, line.second, '\n') # Skip indent + end + end + ## else is copied from vanilla: + else # isa(chunk, CodeChunk) + iocode = IOBuffer() + codefence = config["codefence"]::Pair + write(iocode, codefence.first) + # make sure the code block is finalized if we are printing to ```@example + # (or ````@example, any number of backticks >= 3 works) + if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) + write(iocode, "; continued = true") + end + write(iocode, '\n') + # filter out trailing #hide unless code is executed by Documenter + execute = config["execute"]::Bool + write_hide = isdocumenter(config) && !execute + write_line(line) = write_hide || !endswith(line, "#hide") + for line in chunk.lines + write_line(line) && write(iocode, line, '\n') + end + if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) + write(iocode, "nothing #hide\n") + end + write(iocode, codefence.second, '\n') + any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) + if execute + cd(config["literate_outputdir"]) do + execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; + inputfile=config["literate_inputfile"], + fake_source=config["literate_outputfile"], + flavor=config["flavor"], + image_formats=config["image_formats"], + file_prefix="$(config["name"])-$(chunknum)", + ) + end + end + else + # Vanilla Function + for (chunknum, chunk) in enumerate(chunks) + if isa(chunk, MDChunk) + for line in chunk.lines + write(iomd, line.second, '\n') # skip indent here + end + else # isa(chunk, CodeChunk) + iocode = IOBuffer() + codefence = config["codefence"]::Pair + write(iocode, codefence.first) + # make sure the code block is finalized if we are printing to ```@example + # (or ````@example, any number of backticks >= 3 works) + if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) + write(iocode, "; continued = true") + end + write(iocode, '\n') + # filter out trailing #hide unless code is executed by Documenter + execute = config["execute"]::Bool + write_hide = isdocumenter(config) && !execute + write_line(line) = write_hide || !endswith(line, "#hide") + for line in chunk.lines + write_line(line) && write(iocode, line, '\n') + end + if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) + write(iocode, "nothing #hide\n") + end + write(iocode, codefence.second, '\n') + any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) + if execute + cd(config["literate_outputdir"]) do + execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; inputfile=config["literate_inputfile"], fake_source=config["literate_outputfile"], flavor=config["flavor"], image_formats=config["image_formats"], file_prefix="$(config["name"])-$(chunknum)", - ) + ) + end end - end end write(iomd, '\n') # add a newline between each chunk end @@ -602,7 +754,10 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg # write to file outputfile = write_result(content, config) return outputfile -end + +end + + function execute_markdown!(io::IO, sb::Module, block::String, outputdir; inputfile::String, fake_source::String, @@ -612,6 +767,7 @@ function execute_markdown!(io::IO, sb::Module, block::String, outputdir; # issue #101: consecutive codefenced blocks need newline # issue #144: quadruple backticks allow for triple backticks in the output plain_fence = "\n````\n" => "\n````" + # Here CarpentiresFlavor fork... if r !== nothing && !REPL.ends_with_semicolon(block) if (flavor isa FranklinFlavor || flavor isa DocumenterFlavor) && Base.invokelatest(showable, MIME("text/html"), r) From eb6968de99ddcc720718dfd41c3e477c56d37261 Mon Sep 17 00:00:00 2001 From: Myrkwid Date: Fri, 28 Jul 2023 10:09:54 +0200 Subject: [PATCH 04/24] Changed order of functions and a regex. --- src/Literate.jl | 97 +++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 9b11609..d59916b 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -16,7 +16,7 @@ struct DefaultFlavor <: AbstractFlavor end struct DocumenterFlavor <: AbstractFlavor end struct CommonMarkFlavor <: AbstractFlavor end struct FranklinFlavor <: AbstractFlavor end -struct CarpentriesFlavor <: AbstractFlavor end +struct CarpentriesFlavor <: AbstractFlavor end # # Some simple rules: # @@ -499,6 +499,45 @@ function write_result(content, config; print=print) return outputfile end +""" + Literate.script(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwargs...) + +Generate a plain script file from `inputfile` and write the result to `outputdir`. + +See the manual section on [Configuration](@ref) for documentation +of possible configuration with `config` and other keyword arguments. +""" +function script(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwargs...) + # preprocessing and parsing + chunks, config = + preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:jl) + + # create the script file + ioscript = IOBuffer() + for chunk in chunks + if isa(chunk, CodeChunk) + for line in chunk.lines + write(ioscript, line, '\n') + end + write(ioscript, '\n') # add a newline between each chunk + elseif isa(chunk, MDChunk) && config["keep_comments"]::Bool + for line in chunk.lines + write(ioscript, rstrip(line.first * "# " * line.second) * '\n') + end + write(ioscript, '\n') # add a newline between each chunk + end + end + + # custom post-processing from user + content = config["postprocess"](String(take!(ioscript))) + + # write to file + outputfile = write_result(content, config) + return outputfile +end + + +#_______________________________________________________________________________________ # Define general functions needed for admonitions formating. function containsAdmonition(chunk) @@ -561,7 +600,7 @@ end function CarpentriesChallenge(admonition, io) for line in admonition - if startswith(strip(line.first * line.second), r"!!! [smf][cr]") + if startswith(strip(line.first * line.second), r"\S+\s[smf][cr]") write(io, ":::::::: challenge", '\n') elseif startswith(strip(line.first * line.second), "!!! solution") write(io, ":::::::: solution", '\n') @@ -583,6 +622,16 @@ function CarpentriesWarning(admonition, io) write(io, "::::::::", '\n') end +function CarpentriesYAML(admonition, io) + for line in admonition + if startswith(strip(line.first * line.second), "!!!") + write(io, "", '\n') + else + write(io, line, '\n') + end + end +end + function CarpentriesAdmonition(admonition, io) category = admonition.category @@ -594,45 +643,12 @@ function CarpentriesAdmonition(admonition, io) return CarpentriesWarning(admonition, io) elseif category == "info" || "note" return CarpentriesCallout(admonition, io) + elseif category == "carp" + return CarpentriesYAML(admonition, io) end end +#_______________________________________________________________________________________ -""" - Literate.script(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwargs...) - -Generate a plain script file from `inputfile` and write the result to `outputdir`. - -See the manual section on [Configuration](@ref) for documentation -of possible configuration with `config` and other keyword arguments. -""" -function script(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwargs...) - # preprocessing and parsing - chunks, config = - preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:jl) - - # create the script file - ioscript = IOBuffer() - for chunk in chunks - if isa(chunk, CodeChunk) - for line in chunk.lines - write(ioscript, line, '\n') - end - write(ioscript, '\n') # add a newline between each chunk - elseif isa(chunk, MDChunk) && config["keep_comments"]::Bool - for line in chunk.lines - write(ioscript, rstrip(line.first * "# " * line.second) * '\n') - end - write(ioscript, '\n') # add a newline between each chunk - end - end - - # custom post-processing from user - content = config["postprocess"](String(take!(ioscript))) - - # write to file - outputfile = write_result(content, config) - return outputfile -end """ @@ -703,7 +719,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg file_prefix="$(config["name"])-$(chunknum)", ) end - end + end else # Vanilla Function for (chunknum, chunk) in enumerate(chunks) @@ -744,6 +760,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg ) end end + end end write(iomd, '\n') # add a newline between each chunk end @@ -1032,4 +1049,4 @@ function execute_block(sb::Module, block::String; inputfile::String, fake_source return c.value, c.output, disp.data end -end # module +end # module \ No newline at end of file From e1c9f0d6e22f8f15aa6f67d3963d09466f34dd60 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 10:49:03 +0200 Subject: [PATCH 05/24] Added function to remove YAML from non Carp MD. --- src/Literate.jl | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/Literate.jl b/src/Literate.jl index d59916b..7da27e3 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -548,6 +548,14 @@ function containsAdmonition(chunk) return false end +function containsYAML(chunk) + if startswith(strip(line.first * line.second), "!!! carp") + return true + end +end +return false +end + function chunkToMD(chunk) buffer = IOBuffer() for line in chunk.lines @@ -724,6 +732,23 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg # Vanilla Function for (chunknum, chunk) in enumerate(chunks) if isa(chunk, MDChunk) + + #______________________________________________________________________________________________________________ + if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. + str = chunkToMD(chunk) + mdContent = str.content + + for item in mdContent + if isa(item, Markdown.Admonition) + result = "" + else + result=string(Markdown.MD(item)) + end + write(io, result, '\n') + end + end + #______________________________________________________________________________________________________________ + for line in chunk.lines write(iomd, line.second, '\n') # skip indent here end From 2d0d6f5ce8db13650bf638d823d3d430d626dc1a Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 11:00:27 +0200 Subject: [PATCH 06/24] Fixed loops that were in the wrong order. --- src/Literate.jl | 126 ++++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 62 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 7da27e3..ae8137a 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -549,11 +549,11 @@ function containsAdmonition(chunk) end function containsYAML(chunk) - if startswith(strip(line.first * line.second), "!!! carp") - return true - end -end -return false + if startswith(strip(line.first * line.second), "!!! carp") + return true + end + #??? + return false end function chunkToMD(chunk) @@ -677,9 +677,10 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg sb = sandbox() iomd = IOBuffer() - for chunk in chunks - io = IOBuffer() - if flavor isa CarpentriesFlavor + + if flavor isa CarpentriesFlavor + for chunk in chunks + io = IOBuffer() if isa(chunk, MDChunk) if containsAdmonition(chunk) @@ -728,67 +729,68 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg ) end end - else - # Vanilla Function - for (chunknum, chunk) in enumerate(chunks) - if isa(chunk, MDChunk) - - #______________________________________________________________________________________________________________ - if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. - str = chunkToMD(chunk) - mdContent = str.content - - for item in mdContent - if isa(item, Markdown.Admonition) - result = "" - else - result=string(Markdown.MD(item)) - end - write(io, result, '\n') + end + else + # Vanilla Function + for (chunknum, chunk) in enumerate(chunks) + if isa(chunk, MDChunk) + + #______________________________________________________________________________________________________________ + if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. + str = chunkToMD(chunk) + mdContent = str.content + + for item in mdContent + if isa(item, Markdown.Admonition) + result = "" + else + result=string(Markdown.MD(item)) end + write(io, result, '\n') end - #______________________________________________________________________________________________________________ + end + #______________________________________________________________________________________________________________ - for line in chunk.lines - write(iomd, line.second, '\n') # skip indent here - end - else # isa(chunk, CodeChunk) - iocode = IOBuffer() - codefence = config["codefence"]::Pair - write(iocode, codefence.first) - # make sure the code block is finalized if we are printing to ```@example - # (or ````@example, any number of backticks >= 3 works) - if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) - write(iocode, "; continued = true") - end - write(iocode, '\n') - # filter out trailing #hide unless code is executed by Documenter - execute = config["execute"]::Bool - write_hide = isdocumenter(config) && !execute - write_line(line) = write_hide || !endswith(line, "#hide") - for line in chunk.lines - write_line(line) && write(iocode, line, '\n') - end - if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) - write(iocode, "nothing #hide\n") - end - write(iocode, codefence.second, '\n') - any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) - if execute - cd(config["literate_outputdir"]) do - execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; - inputfile=config["literate_inputfile"], - fake_source=config["literate_outputfile"], - flavor=config["flavor"], - image_formats=config["image_formats"], - file_prefix="$(config["name"])-$(chunknum)", - ) - end + for line in chunk.lines + write(iomd, line.second, '\n') # skip indent here + end + else # isa(chunk, CodeChunk) + iocode = IOBuffer() + codefence = config["codefence"]::Pair + write(iocode, codefence.first) + # make sure the code block is finalized if we are printing to ```@example + # (or ````@example, any number of backticks >= 3 works) + if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) + write(iocode, "; continued = true") + end + write(iocode, '\n') + # filter out trailing #hide unless code is executed by Documenter + execute = config["execute"]::Bool + write_hide = isdocumenter(config) && !execute + write_line(line) = write_hide || !endswith(line, "#hide") + for line in chunk.lines + write_line(line) && write(iocode, line, '\n') + end + if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) + write(iocode, "nothing #hide\n") + end + write(iocode, codefence.second, '\n') + any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) + if execute + cd(config["literate_outputdir"]) do + execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; + inputfile=config["literate_inputfile"], + fake_source=config["literate_outputfile"], + flavor=config["flavor"], + image_formats=config["image_formats"], + file_prefix="$(config["name"])-$(chunknum)", + ) end end end - write(iomd, '\n') # add a newline between each chunk end + write(iomd, '\n') # add a newline between each chunk + # custom post-processing from user content = config["postprocess"](String(take!(iomd))) From c41bb32a378f838e9d1fe8aa3807a6f9680fc24f Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 11:06:59 +0200 Subject: [PATCH 07/24] Changed IO to IOmd which was already defined. --- src/Literate.jl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index ae8137a..1321c87 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -680,22 +680,21 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg if flavor isa CarpentriesFlavor for chunk in chunks - io = IOBuffer() if isa(chunk, MDChunk) if containsAdmonition(chunk) str = chunkToMD(chunk) mdContent = str.content - writeContent(mdContent, io) + writeContent(mdContent, iomd) else # Handle chunks without admonitions for line in chunk.lines - write(io, line.second, '\n') # Skip indent + write(iomd, line.second, '\n') # Skip indent end end - ## else is copied from vanilla: + ## `else` is copied from vanilla: else # isa(chunk, CodeChunk) iocode = IOBuffer() codefence = config["codefence"]::Pair @@ -746,7 +745,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg else result=string(Markdown.MD(item)) end - write(io, result, '\n') + write(iomd, result, '\n') end end #______________________________________________________________________________________________________________ @@ -789,7 +788,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg end end end - write(iomd, '\n') # add a newline between each chunk + write(iomd, '\n') # add a newline between each chunk # custom post-processing from user From 01b6da57adcaa914484d875d2cc420e2ce0b2dd1 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 11:22:16 +0200 Subject: [PATCH 08/24] Forgot the for loop around the function. --- src/Literate.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Literate.jl b/src/Literate.jl index 1321c87..079918b 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -541,6 +541,7 @@ end # Define general functions needed for admonitions formating. function containsAdmonition(chunk) + for line in chunk.lines if startswith(strip(line.first * line.second), "!!!") return true end @@ -549,10 +550,11 @@ function containsAdmonition(chunk) end function containsYAML(chunk) + for line in chunk.lines if startswith(strip(line.first * line.second), "!!! carp") return true end - #??? + end return false end From 68123b7291480c56d3880e021d61ebabb8100ab2 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 11:23:50 +0200 Subject: [PATCH 09/24] Removed probably unnecessary passing of IO. --- src/Literate.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 079918b..cb0df56 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -567,7 +567,7 @@ function chunkToMD(chunk) return Markdown.parse(read(buffer, String)) end -function processNonAdmonitions(item, io) +function processNonAdmonitions(item) # Handle non-admonition elements return string(Markdown.MD(item)) end @@ -577,7 +577,7 @@ function writeContent(mdContent, io) if isa(item, Markdown.Admonition) result = CarpentriesAdmonition(item, io) else - result = processNonAdmonitions(item, io) + result = processNonAdmonitions(item) end write(io, result, '\n') end From db12c3b1fa8b2000f9e283f05421bc7fe9d944d7 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 11:29:30 +0200 Subject: [PATCH 10/24] Fixed missing end of a loop. --- src/Literate.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index cb0df56..263f726 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -728,7 +728,8 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg image_formats=config["image_formats"], file_prefix="$(config["name"])-$(chunknum)", ) - end + end + end end end else @@ -786,6 +787,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg image_formats=config["image_formats"], file_prefix="$(config["name"])-$(chunknum)", ) + end end end end @@ -800,7 +802,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg outputfile = write_result(content, config) return outputfile -end +end From 26255dc01904cb6e53ef8bc1d951f714de125259 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 11:43:02 +0200 Subject: [PATCH 11/24] Changed Yaml removal. --- src/Literate.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Literate.jl b/src/Literate.jl index 263f726..193cd89 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -744,7 +744,9 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg for item in mdContent if isa(item, Markdown.Admonition) - result = "" + if startswith(strip(line.first * line.second), "!!! carp") + result = "" + end else result=string(Markdown.MD(item)) end From cc6ad1faefecadc12e79ad3667a06253ab8cbf34 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 11:55:43 +0200 Subject: [PATCH 12/24] Fixed pasting of code to the wrong space. --- src/Literate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Literate.jl b/src/Literate.jl index 193cd89..5f44a51 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -733,7 +733,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg end end else - # Vanilla Function + # kinda Vanilla Function for (chunknum, chunk) in enumerate(chunks) if isa(chunk, MDChunk) From 13eb8f9bac1b416897a91ab33a6726a083bf6f71 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 14:11:48 +0200 Subject: [PATCH 13/24] fix stuff --- Project.toml | 1 + src/Literate.jl | 53 +++++++++++++++++++++++++------------------------ 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Project.toml b/Project.toml index 3347508..cd71aa1 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "2.14.0" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [compat] diff --git a/src/Literate.jl b/src/Literate.jl index 5f44a51..94a87e6 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -561,25 +561,25 @@ end function chunkToMD(chunk) buffer = IOBuffer() for line in chunk.lines - write(buffer, line.first * line.second, '\n') + write(buffer, line.first * line.second, "\n\n") end - seek(buffer, 0) - return Markdown.parse(read(buffer, String)) + str = String(take!(buffer)) + return Markdown.parse(str) end -function processNonAdmonitions(item) +function processNonAdmonitions(item, io) # Handle non-admonition elements - return string(Markdown.MD(item)) + write(io, string(Markdown.MD(item))) end function writeContent(mdContent, io) for item in mdContent if isa(item, Markdown.Admonition) - result = CarpentriesAdmonition(item, io) + CarpentriesAdmonition(item, io) else - result = processNonAdmonitions(item) + processNonAdmonitions(item, io) end - write(io, result, '\n') + # write(io, '\n') end end @@ -588,13 +588,15 @@ end function CarpentriesCallout(admonition, io) for line in admonition - if startswith(strip(line.first * line.second), "!!!") + if startswith(line, "!!!") write(io, ":::::::: callout", '\n') else - write(io, line, '\n') + if line != "" + write(io, line, '\n') + end end end - write(io, "::::::::", '\n') + write(io, "::::::::", "\n\n") end function CarpentriesTestamonial(admonition, io) @@ -605,7 +607,7 @@ function CarpentriesTestamonial(admonition, io) write(io, line, '\n') end end - write(io, "::::::::", '\n') + write(io, "::::::::", "\n\n") end function CarpentriesChallenge(admonition, io) @@ -618,7 +620,7 @@ function CarpentriesChallenge(admonition, io) write(io, line, '\n') end end - write(io, "::::::::", '\n', "::::::::", '\n') + write(io, "::::::::", "\n\n", "::::::::", "\n\n") end function CarpentriesWarning(admonition, io) @@ -629,13 +631,13 @@ function CarpentriesWarning(admonition, io) write(io, line, '\n') end end - write(io, "::::::::", '\n') + write(io, "::::::::", "\n\n") end function CarpentriesYAML(admonition, io) for line in admonition - if startswith(strip(line.first * line.second), "!!!") - write(io, "", '\n') + if startswith(line, "!!!") + continue else write(io, line, '\n') end @@ -644,17 +646,17 @@ end function CarpentriesAdmonition(admonition, io) category = admonition.category - - if category == "sc" || "mc" || "freecode" - return CarpentriesChallenge(admonition, io) + admonition = split(string(Markdown.MD(admonition)), '\n') + if category in ("sc", "mc", "freecode") + CarpentriesChallenge(admonition, io) elseif category == "tip" - return CarpentriesTestamonial(admonition, io) + CarpentriesTestamonial(admonition, io) elseif category == "warning" - return CarpentriesWarning(admonition, io) - elseif category == "info" || "note" - return CarpentriesCallout(admonition, io) + CarpentriesWarning(admonition, io) + elseif category in ("info", "note") + CarpentriesCallout(admonition, io) elseif category == "carp" - return CarpentriesYAML(admonition, io) + CarpentriesYAML(admonition, io) end end #_______________________________________________________________________________________ @@ -687,7 +689,6 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg str = chunkToMD(chunk) mdContent = str.content - writeContent(mdContent, iomd) else @@ -745,7 +746,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg for item in mdContent if isa(item, Markdown.Admonition) if startswith(strip(line.first * line.second), "!!! carp") - result = "" + continue end else result=string(Markdown.MD(item)) From 17ba06b808453165e28a62bbd9b59ca9618cfde4 Mon Sep 17 00:00:00 2001 From: Jonas Kroll Date: Fri, 28 Jul 2023 14:19:54 +0200 Subject: [PATCH 14/24] fix YAML removement --- src/Literate.jl | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 94a87e6..c33c9a0 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -740,19 +740,7 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg #______________________________________________________________________________________________________________ if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. - str = chunkToMD(chunk) - mdContent = str.content - - for item in mdContent - if isa(item, Markdown.Admonition) - if startswith(strip(line.first * line.second), "!!! carp") - continue - end - else - result=string(Markdown.MD(item)) - end - write(iomd, result, '\n') - end + continue end #______________________________________________________________________________________________________________ From 8f0468303f9cd6c457662823ac7b4dc4f298b4df Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Thu, 3 Aug 2023 18:41:20 +0200 Subject: [PATCH 15/24] fix stuff --- src/Literate.jl | 236 ++++++++++++++++++++---------------------------- 1 file changed, 96 insertions(+), 140 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index c33c9a0..8284d09 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -16,7 +16,7 @@ struct DefaultFlavor <: AbstractFlavor end struct DocumenterFlavor <: AbstractFlavor end struct CommonMarkFlavor <: AbstractFlavor end struct FranklinFlavor <: AbstractFlavor end -struct CarpentriesFlavor <: AbstractFlavor end +struct CarpentriesFlavor <: AbstractFlavor end # # Some simple rules: # @@ -46,7 +46,7 @@ CodeChunk() = CodeChunk(String[], false) ismdline(line) = (occursin(r"^\h*#$", line) || occursin(r"^\h*# .*$", line)) && !occursin(r"^\h*##", line) -function parse(content; allow_continued = true) +function parse(content; allow_continued=true) lines = collect(eachline(IOBuffer(content))) chunks = Chunk[] @@ -124,10 +124,10 @@ function parse(content; allow_continued = true) end function replace_default(content, sym; - config::Dict, - branch = "gh-pages", - commit = "master" - ) + config::Dict, + branch="gh-pages", + commit="master" +) repls = Pair{Any,Any}[] # add some shameless advertisement @@ -227,7 +227,7 @@ filename(str) = first(splitext(last(splitdir(str)))) isdocumenter(cfg) = cfg["flavor"]::AbstractFlavor isa DocumenterFlavor _DEFAULT_IMAGE_FORMATS = [(MIME("image/svg+xml"), ".svg"), (MIME("image/png"), ".png"), - (MIME("image/jpeg"), ".jpeg")] + (MIME("image/jpeg"), ".jpeg")] # Cache of inputfile => head branch const HEAD_BRANCH_CACHE = Dict{String,String}() @@ -245,7 +245,7 @@ function edit_commit(inputfile, user_config) readchomp( pipeline( setenv(`$(git) rev-parse --show-toplevel`; dir=dirname(inputfile)); - stderr=devnull, + stderr=devnull ) ) catch @@ -291,15 +291,15 @@ function create_configuration(inputfile; user_config, user_kwargs, type=nothing) if (d = get(user_config, "documenter", nothing); d !== nothing) if type === :md Base.depwarn("The documenter=$(d) keyword to Literate.markdown is deprecated." * - " Pass `flavor = Literate.$(d ? "DocumenterFlavor" : "CommonMarkFlavor")()`" * - " instead.", Symbol("Literate.markdown")) + " Pass `flavor = Literate.$(d ? "DocumenterFlavor" : "CommonMarkFlavor")()`" * + " instead.", Symbol("Literate.markdown")) user_config["flavor"] = d ? DocumenterFlavor() : CommonMarkFlavor() elseif type === :nb Base.depwarn("The documenter=$(d) keyword to Literate.notebook is deprecated." * - " It is not used anymore for notebook output.", Symbol("Literate.notebook")) + " It is not used anymore for notebook output.", Symbol("Literate.notebook")) elseif type === :jl Base.depwarn("The documenter=$(d) keyword to Literate.script is deprecated." * - " It is not used anymore for script output.", Symbol("Literate.script")) + " It is not used anymore for script output.", Symbol("Literate.script")) end end @@ -431,7 +431,7 @@ Available options: `$(_DEFAULT_IMAGE_FORMATS)`. Results which are `showable` with a MIME type are saved with the first match, with the corresponding extension. """ -const DEFAULT_CONFIGURATION=nothing # Dummy const for documentation +const DEFAULT_CONFIGURATION = nothing # Dummy const for documentation function preprocessor(inputfile, outputdir; user_config, user_kwargs, type) # Create configuration by merging default and userdefined @@ -485,7 +485,7 @@ function preprocessor(inputfile, outputdir; user_config, user_kwargs, type) content = replace_default(content, type; config=config) # parse the content into chunks - chunks = parse(content; allow_continued = type !== :nb) + chunks = parse(content; allow_continued=type !== :nb) return chunks, config end @@ -572,7 +572,7 @@ function processNonAdmonitions(item, io) write(io, string(Markdown.MD(item))) end -function writeContent(mdContent, io) +function writeContent(mdContent, io) for item in mdContent if isa(item, Markdown.Admonition) CarpentriesAdmonition(item, io) @@ -592,7 +592,7 @@ function CarpentriesCallout(admonition, io) write(io, ":::::::: callout", '\n') else if line != "" - write(io, line, '\n') + write(io, line, '\n') end end end @@ -648,13 +648,13 @@ function CarpentriesAdmonition(admonition, io) category = admonition.category admonition = split(string(Markdown.MD(admonition)), '\n') if category in ("sc", "mc", "freecode") - CarpentriesChallenge(admonition, io) + CarpentriesChallenge(admonition, io) elseif category == "tip" - CarpentriesTestamonial(admonition, io) + CarpentriesTestamonial(admonition, io) elseif category == "warning" - CarpentriesWarning(admonition, io) + CarpentriesWarning(admonition, io) elseif category in ("info", "note") - CarpentriesCallout(admonition, io) + CarpentriesCallout(admonition, io) elseif category == "carp" CarpentriesYAML(admonition, io) end @@ -676,135 +676,91 @@ function markdown(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg # preprocessing and parsing chunks, config = preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:md) - flavor=config["flavor"] # create the markdown file - sb = sandbox() iomd = IOBuffer() - - - if flavor isa CarpentriesFlavor - for chunk in chunks - if isa(chunk, MDChunk) - if containsAdmonition(chunk) - str = chunkToMD(chunk) - mdContent = str.content - writeContent(mdContent, iomd) - else - # Handle chunks without admonitions - for line in chunk.lines - write(iomd, line.second, '\n') # Skip indent - end - end - ## `else` is copied from vanilla: - else # isa(chunk, CodeChunk) - iocode = IOBuffer() - codefence = config["codefence"]::Pair - write(iocode, codefence.first) - # make sure the code block is finalized if we are printing to ```@example - # (or ````@example, any number of backticks >= 3 works) - if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) - write(iocode, "; continued = true") - end - write(iocode, '\n') - # filter out trailing #hide unless code is executed by Documenter - execute = config["execute"]::Bool - write_hide = isdocumenter(config) && !execute - write_line(line) = write_hide || !endswith(line, "#hide") - for line in chunk.lines - write_line(line) && write(iocode, line, '\n') - end - if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) - write(iocode, "nothing #hide\n") - end - write(iocode, codefence.second, '\n') - any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) - if execute - cd(config["literate_outputdir"]) do - execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; - inputfile=config["literate_inputfile"], - fake_source=config["literate_outputfile"], - flavor=config["flavor"], - image_formats=config["image_formats"], - file_prefix="$(config["name"])-$(chunknum)", - ) - end - end - end - end - else - # kinda Vanilla Function - for (chunknum, chunk) in enumerate(chunks) - if isa(chunk, MDChunk) + write_md_chunks!(iomd, chunks, outputdir, config) - #______________________________________________________________________________________________________________ + + # custom post-processing from user + content = config["postprocess"](String(take!(iomd))) + + # write to file + outputfile = write_result(content, config) + return outputfile + +end + +function write_md_chunks!(iomd, chunks, outputdir, config) + flavor = config["flavor"] + sb = sandbox() + for (chunknum, chunk) in enumerate(chunks) + if isa(chunk, MDChunk) + + #______________________________________________________________________________________________________________ + if flavor isa CarpentriesFlavor + if containsAdmonition(chunk) + writeContent(chunk.lines, iomd) + end + else if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. continue end - #______________________________________________________________________________________________________________ + end + #______________________________________________________________________________________________________________ - for line in chunk.lines - write(iomd, line.second, '\n') # skip indent here - end - else # isa(chunk, CodeChunk) - iocode = IOBuffer() - codefence = config["codefence"]::Pair - write(iocode, codefence.first) - # make sure the code block is finalized if we are printing to ```@example - # (or ````@example, any number of backticks >= 3 works) - if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) - write(iocode, "; continued = true") - end - write(iocode, '\n') - # filter out trailing #hide unless code is executed by Documenter - execute = config["execute"]::Bool - write_hide = isdocumenter(config) && !execute - write_line(line) = write_hide || !endswith(line, "#hide") - for line in chunk.lines - write_line(line) && write(iocode, line, '\n') - end - if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) - write(iocode, "nothing #hide\n") - end - write(iocode, codefence.second, '\n') - any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) - if execute - cd(config["literate_outputdir"]) do + for line in chunk.lines + write(iomd, line.second, '\n') # skip indent here + end + else # isa(chunk, CodeChunk) + iocode = IOBuffer() + codefence = config["codefence"]::Pair + write(iocode, codefence.first) + # make sure the code block is finalized if we are printing to ```@example + # (or ````@example, any number of backticks >= 3 works) + if chunk.continued && occursin(r"^`{3,}@example", codefence.first) && isdocumenter(config) + write(iocode, "; continued = true") + end + write(iocode, '\n') + # filter out trailing #hide unless code is executed by Documenter + execute = config["execute"]::Bool + write_hide = isdocumenter(config) && !execute + write_line(line) = write_hide || !endswith(line, "#hide") + for line in chunk.lines + write_line(line) && write(iocode, line, '\n') + end + if write_hide && REPL.ends_with_semicolon(chunk.lines[end]) + write(iocode, "nothing #hide\n") + end + write(iocode, codefence.second, '\n') + any(write_line, chunk.lines) && write(iomd, seekstart(iocode)) + if execute + cd(config["literate_outputdir"]) do execute_markdown!(iomd, sb, join(chunk.lines, '\n'), outputdir; - inputfile=config["literate_inputfile"], - fake_source=config["literate_outputfile"], - flavor=config["flavor"], - image_formats=config["image_formats"], - file_prefix="$(config["name"])-$(chunknum)", + inputfile=config["literate_inputfile"], + fake_source=config["literate_outputfile"], + flavor=config["flavor"], + image_formats=config["image_formats"], + file_prefix="$(config["name"])-$(chunknum)" ) - end end end end + write(iomd, '\n') # add a newline between each chunk end - write(iomd, '\n') # add a newline between each chunk - - - # custom post-processing from user - content = config["postprocess"](String(take!(iomd))) - - # write to file - outputfile = write_result(content, config) - return outputfile - -end +end function execute_markdown!(io::IO, sb::Module, block::String, outputdir; - inputfile::String, fake_source::String, - flavor::AbstractFlavor, image_formats::Vector, file_prefix::String) + inputfile::String, fake_source::String, + flavor::AbstractFlavor, image_formats::Vector, file_prefix::String) # TODO: Deal with explicit display(...) calls r, str, _ = execute_block(sb, block; inputfile=inputfile, fake_source=fake_source) # issue #101: consecutive codefenced blocks need newline # issue #144: quadruple backticks allow for triple backticks in the output - plain_fence = "\n````\n" => "\n````" + plain_fence = "\n````\n" => "\n````" # Here CarpentiresFlavor fork... if r !== nothing && !REPL.ends_with_semicolon(block) if (flavor isa FranklinFlavor || flavor isa DocumenterFlavor) && @@ -851,8 +807,8 @@ function parse_nbmeta(line) # Cf. https://jupytext.readthedocs.io/en/latest/formats.html#the-percent-format m = match(r"^%% ([^[{]+)?\s*(?:\[(\w+)\])?\s*(\{.*)?$", line) typ = m.captures[2] - name = m.captures[1] === nothing ? Dict{String, String}() : Dict("name" => m.captures[1]) - meta = m.captures[3] === nothing ? Dict{String, Any}() : JSON.parse(m.captures[3]) + name = m.captures[1] === nothing ? Dict{String,String}() : Dict("name" => m.captures[1]) + meta = m.captures[3] === nothing ? Dict{String,Any}() : JSON.parse(m.captures[3]) return typ, merge(name, meta) end line_is_nbmeta(line::Pair) = line_is_nbmeta(line.second) @@ -875,7 +831,7 @@ function notebook(inputfile, outputdir=pwd(); config::AbstractDict=Dict(), kwarg nb = jupyter_notebook(chunks, config) # write to file - outputfile = write_result(nb, config; print = (io, c)->JSON.print(io, c, 1)) + outputfile = write_result(nb, config; print=(io, c) -> JSON.print(io, c, 1)) return outputfile end @@ -894,11 +850,11 @@ function jupyter_notebook(chunks, config) metatype !== nothing && metatype != chunktype && error("specifying a different cell type is not supported") popfirst!(chunk.lines) else - metadata = Dict{String, Any}() + metadata = Dict{String,Any}() end lines = isa(chunk, MDChunk) ? - String[x.second for x in chunk.lines] : # skip indent - chunk.lines + String[x.second for x in chunk.lines] : # skip indent + chunk.lines @views map!(x -> x * '\n', lines[1:end-1], lines[1:end-1]) cell["cell_type"] = chunktype cell["metadata"] = metadata @@ -915,15 +871,15 @@ function jupyter_notebook(chunks, config) metadata = Dict() kernelspec = Dict() - kernelspec["language"] = "julia" - kernelspec["name"] = "julia-$(VERSION.major).$(VERSION.minor)" + kernelspec["language"] = "julia" + kernelspec["name"] = "julia-$(VERSION.major).$(VERSION.minor)" kernelspec["display_name"] = "Julia $(string(VERSION))" metadata["kernelspec"] = kernelspec language_info = Dict() language_info["file_extension"] = ".jl" language_info["mimetype"] = "application/julia" - language_info["name"]= "julia" + language_info["name"] = "julia" language_info["version"] = string(VERSION) metadata["language_info"] = language_info @@ -937,7 +893,7 @@ function jupyter_notebook(chunks, config) try cd(config["literate_outputdir"]) do nb = execute_notebook(nb; inputfile=config["literate_inputfile"], - fake_source=config["literate_outputfile"]) + fake_source=config["literate_outputfile"]) end catch err @error "error when executing notebook based on input file: " * @@ -963,7 +919,7 @@ function execute_notebook(nb; inputfile::String, fake_source::String) stream = Dict{String,Any}() stream["output_type"] = "stream" stream["name"] = "stdout" - stream["text"] = collect(Any, eachline(IOBuffer(String(str)), keep = true)) + stream["text"] = collect(Any, eachline(IOBuffer(String(str)), keep=true)) push!(cell["outputs"], stream) end @@ -973,7 +929,7 @@ function execute_notebook(nb; inputfile::String, fake_source::String) function split_mime(dict) for mime in ("image/svg+xml", "text/html") if haskey(dict, mime) - dict[mime] = collect(Any, eachline(IOBuffer(dict[mime]), keep = true)) + dict[mime] = collect(Any, eachline(IOBuffer(dict[mime]), keep=true)) end end return dict @@ -1053,7 +1009,7 @@ function execute_block(sb::Module, block::String; inputfile::String, fake_source # - c.output: combined stdout and stderr # `rethrow = Union{}` means that we try-catch all the exceptions thrown in the do-block # and return them via the return value (they get handled below). - c = IOCapture.capture(rethrow = Union{}) do + c = IOCapture.capture(rethrow=Union{}) do include_string(sb, block, fake_source) end popdisplay(disp) # IOCapture.capture has a try-catch so should always end up here @@ -1070,4 +1026,4 @@ function execute_block(sb::Module, block::String; inputfile::String, fake_source return c.value, c.output, disp.data end -end # module \ No newline at end of file +end # module From d68cd5340a509db44a52baae59abcff02f428b90 Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Fri, 4 Aug 2023 10:47:16 +0200 Subject: [PATCH 16/24] fix admonition handling --- src/Literate.jl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 8284d09..6ba2999 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -573,7 +573,7 @@ function processNonAdmonitions(item, io) end function writeContent(mdContent, io) - for item in mdContent + for item in mdContent.content if isa(item, Markdown.Admonition) CarpentriesAdmonition(item, io) else @@ -601,7 +601,7 @@ end function CarpentriesTestamonial(admonition, io) for line in admonition - if startswith(strip(line.first * line.second), "!!!") + if startswith(strip(line), "!!!") write(io, ":::::::: testamonial", '\n') else write(io, line, '\n') @@ -612,9 +612,9 @@ end function CarpentriesChallenge(admonition, io) for line in admonition - if startswith(strip(line.first * line.second), r"\S+\s[smf][cr]") + if startswith(strip(line), r"\S+\s[smf][cr]") write(io, ":::::::: challenge", '\n') - elseif startswith(strip(line.first * line.second), "!!! solution") + elseif startswith(strip(line), "!!! solution") write(io, ":::::::: solution", '\n') else write(io, line, '\n') @@ -625,7 +625,7 @@ end function CarpentriesWarning(admonition, io) for line in admonition - if startswith(strip(line.first * line.second), "!!!") + if startswith(strip(line), "!!!") write(io, ":::::::: warning", '\n') else write(io, line, '\n') @@ -701,13 +701,12 @@ function write_md_chunks!(iomd, chunks, outputdir, config) #______________________________________________________________________________________________________________ if flavor isa CarpentriesFlavor if containsAdmonition(chunk) - writeContent(chunk.lines, iomd) + writeContent(chunkToMD(chunk), iomd) end - else + end if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. continue end - end #______________________________________________________________________________________________________________ for line in chunk.lines From 6b53729da2375957b5d2f69ddbaa710e05935eb2 Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Fri, 4 Aug 2023 14:43:49 +0200 Subject: [PATCH 17/24] support nested blocks via recursion --- src/Literate.jl | 104 ++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 66 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 6ba2999..58e49b2 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -572,91 +572,60 @@ function processNonAdmonitions(item, io) write(io, string(Markdown.MD(item))) end -function writeContent(mdContent, io) - for item in mdContent.content +function rewriteContent!(mdContent) + for (i, item) in enumerate(mdContent.content) if isa(item, Markdown.Admonition) - CarpentriesAdmonition(item, io) - else - processNonAdmonitions(item, io) + mdContent.content[i] = if any(x->x isa Markdown.Admonition, item.content) + CarpentriesAdmonition(rewriteContent!(item)) + else + CarpentriesAdmonition(item) + end end - # write(io, '\n') end + mdContent end #Functions needed for addition transformation into Carpentries style. Markdown style into pandoc fenced divs -function CarpentriesCallout(admonition, io) - for line in admonition - if startswith(line, "!!!") - write(io, ":::::::: callout", '\n') - else - if line != "" - write(io, line, '\n') - end - end - end - write(io, "::::::::", "\n\n") +function CarpentriesCallout(admonition) + vcat(Markdown.Paragraph(":::::::: callout\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) end -function CarpentriesTestamonial(admonition, io) - for line in admonition - if startswith(strip(line), "!!!") - write(io, ":::::::: testamonial", '\n') - else - write(io, line, '\n') - end - end - write(io, "::::::::", "\n\n") +function CarpentriesTestamonial(admonition) + vcat(Markdown.Paragraph(":::::::: testamonial\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) end -function CarpentriesChallenge(admonition, io) - for line in admonition - if startswith(strip(line), r"\S+\s[smf][cr]") - write(io, ":::::::: challenge", '\n') - elseif startswith(strip(line), "!!! solution") - write(io, ":::::::: solution", '\n') - else - write(io, line, '\n') - end - end - write(io, "::::::::", "\n\n", "::::::::", "\n\n") +function CarpentriesSolution(admonition) + vcat(Markdown.Paragraph(":::::::: solution\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) end -function CarpentriesWarning(admonition, io) - for line in admonition - if startswith(strip(line), "!!!") - write(io, ":::::::: warning", '\n') - else - write(io, line, '\n') - end - end - write(io, "::::::::", "\n\n") +function CarpentriesChallenge(admonition) + vcat(Markdown.Paragraph(":::::::: challenge\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) end -function CarpentriesYAML(admonition, io) - for line in admonition - if startswith(line, "!!!") - continue - else - write(io, line, '\n') - end - end +function CarpentriesWarning(admonition) + vcat(Markdown.Paragraph(":::::::: warning\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) end -function CarpentriesAdmonition(admonition, io) +function CarpentriesYAML(admonition) + admonition.content +end + +function CarpentriesAdmonition(admonition) category = admonition.category - admonition = split(string(Markdown.MD(admonition)), '\n') - if category in ("sc", "mc", "freecode") - CarpentriesChallenge(admonition, io) + return if category in ("challenge", "sc", "mc", "freecode") + CarpentriesChallenge(admonition) + elseif category == "solution" + CarpentriesSolution(admonition) elseif category == "tip" - CarpentriesTestamonial(admonition, io) + CarpentriesTestamonial(admonition) elseif category == "warning" - CarpentriesWarning(admonition, io) + CarpentriesWarning(admonition) elseif category in ("info", "note") - CarpentriesCallout(admonition, io) + CarpentriesCallout(admonition) elseif category == "carp" - CarpentriesYAML(admonition, io) + CarpentriesYAML(admonition) end end #_______________________________________________________________________________________ @@ -701,12 +670,15 @@ function write_md_chunks!(iomd, chunks, outputdir, config) #______________________________________________________________________________________________________________ if flavor isa CarpentriesFlavor if containsAdmonition(chunk) - writeContent(chunkToMD(chunk), iomd) - end - end - if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. + md_chunk = chunkToMD(chunk) + rewriteContent!(md_chunk) + write(iomd, string(Markdown.MD(md_chunk))) continue end + end + if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. + continue + end #______________________________________________________________________________________________________________ for line in chunk.lines From 93c19204d60457579dc9674cdc39221a10f89c6c Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Fri, 4 Aug 2023 14:52:06 +0200 Subject: [PATCH 18/24] delete unused function --- src/Literate.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 58e49b2..28940a0 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -567,11 +567,6 @@ function chunkToMD(chunk) return Markdown.parse(str) end -function processNonAdmonitions(item, io) - # Handle non-admonition elements - write(io, string(Markdown.MD(item))) -end - function rewriteContent!(mdContent) for (i, item) in enumerate(mdContent.content) if isa(item, Markdown.Admonition) From ef753678844fb7bba75000eac2368b82f169aeb9 Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Fri, 4 Aug 2023 16:41:04 +0200 Subject: [PATCH 19/24] fix testimonials --- src/Literate.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 28940a0..c91eee5 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -587,8 +587,8 @@ function CarpentriesCallout(admonition) vcat(Markdown.Paragraph(":::::::: callout\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) end -function CarpentriesTestamonial(admonition) - vcat(Markdown.Paragraph(":::::::: testamonial\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) +function CarpentriesTestimonial(admonition) + vcat(Markdown.Paragraph(":::::::: testimonial\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) end function CarpentriesSolution(admonition) @@ -614,7 +614,7 @@ function CarpentriesAdmonition(admonition) elseif category == "solution" CarpentriesSolution(admonition) elseif category == "tip" - CarpentriesTestamonial(admonition) + CarpentriesTestimonial(admonition) elseif category == "warning" CarpentriesWarning(admonition) elseif category in ("info", "note") From 658515dcd5a7e53a8b619cc02fc7702078b7628a Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Mon, 7 Aug 2023 17:30:10 +0200 Subject: [PATCH 20/24] more general conversion approach --- src/Literate.jl | 65 ++++++++++--------------------------------------- 1 file changed, 13 insertions(+), 52 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index c91eee5..13ffcce 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -549,15 +549,6 @@ function containsAdmonition(chunk) return false end -function containsYAML(chunk) - for line in chunk.lines - if startswith(strip(line.first * line.second), "!!! carp") - return true - end - end - return false -end - function chunkToMD(chunk) buffer = IOBuffer() for line in chunk.lines @@ -570,10 +561,10 @@ end function rewriteContent!(mdContent) for (i, item) in enumerate(mdContent.content) if isa(item, Markdown.Admonition) - mdContent.content[i] = if any(x->x isa Markdown.Admonition, item.content) - CarpentriesAdmonition(rewriteContent!(item)) + mdContent.content[i] = if any(x -> x isa Markdown.Admonition, item.content) + admonition_to_fenced_div(rewriteContent!(item)) else - CarpentriesAdmonition(item) + admonition_to_fenced_div(item) end end end @@ -583,46 +574,19 @@ end #Functions needed for addition transformation into Carpentries style. Markdown style into pandoc fenced divs -function CarpentriesCallout(admonition) - vcat(Markdown.Paragraph(":::::::: callout\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) -end - -function CarpentriesTestimonial(admonition) - vcat(Markdown.Paragraph(":::::::: testimonial\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) -end - -function CarpentriesSolution(admonition) - vcat(Markdown.Paragraph(":::::::: solution\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) -end - -function CarpentriesChallenge(admonition) - vcat(Markdown.Paragraph(":::::::: challenge\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) -end - -function CarpentriesWarning(admonition) - vcat(Markdown.Paragraph(":::::::: warning\n"), admonition.content, Markdown.Paragraph("::::::::\n\n")) -end - -function CarpentriesYAML(admonition) - admonition.content -end - -function CarpentriesAdmonition(admonition) +function admonition_to_fenced_div(admonition) category = admonition.category - return if category in ("challenge", "sc", "mc", "freecode") - CarpentriesChallenge(admonition) - elseif category == "solution" - CarpentriesSolution(admonition) - elseif category == "tip" - CarpentriesTestimonial(admonition) - elseif category == "warning" - CarpentriesWarning(admonition) - elseif category in ("info", "note") - CarpentriesCallout(admonition) - elseif category == "carp" - CarpentriesYAML(admonition) + if category == "yaml" + single_paragraph = [Markdown.HorizontalRule(), Markdown.Paragraph( + reduce(vcat, map(Base.Fix2(getproperty, :content), Iterators.filter(Base.Fix2(!isa, Markdown.HorizontalRule), admonition.content))) .* '\n' + ), Markdown.HorizontalRule()] + return single_paragraph end + title = admonition.title + vcat(Markdown.Paragraph(":::::::: $category \n"), + isempty(title) ? [] : Markdown.Header{2}(title), admonition.content, Markdown.Paragraph("::::::::\n\n")) end + #_______________________________________________________________________________________ @@ -671,9 +635,6 @@ function write_md_chunks!(iomd, chunks, outputdir, config) continue end end - if containsYAML(chunk) # This part is the only change. It (should) delete the YAML Admo for non Carpentries MD. - continue - end #______________________________________________________________________________________________________________ for line in chunk.lines From 27d9a58e0e9d5ac86f86f4dca0e32e07f548a596 Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Mon, 7 Aug 2023 17:36:53 +0200 Subject: [PATCH 21/24] fix yaml handling --- src/Literate.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 13ffcce..b97f11c 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -577,9 +577,10 @@ end function admonition_to_fenced_div(admonition) category = admonition.category if category == "yaml" - single_paragraph = [Markdown.HorizontalRule(), Markdown.Paragraph( - reduce(vcat, map(Base.Fix2(getproperty, :content), Iterators.filter(Base.Fix2(!isa, Markdown.HorizontalRule), admonition.content))) .* '\n' - ), Markdown.HorizontalRule()] + single_paragraph = Markdown.Paragraph(vcat("---\n", + reduce(vcat, map(Base.Fix2(getproperty, :content), Iterators.filter(Base.Fix2(!isa, Markdown.HorizontalRule), admonition.content))) .* '\n', + "---\n" + )) return single_paragraph end title = admonition.title From a7723529f35fb8cdef1833148feb3724e3084278 Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Tue, 8 Aug 2023 14:25:53 +0200 Subject: [PATCH 22/24] switch to CommonMark parser --- Project.toml | 2 +- src/Literate.jl | 48 ++++++++++++++++++++++-------------------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/Project.toml b/Project.toml index cd71aa1..47a89a3 100644 --- a/Project.toml +++ b/Project.toml @@ -4,9 +4,9 @@ version = "2.14.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +CommonMark = "a80b9123-70ca-4bc0-993e-6e3bcb318db6" IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [compat] diff --git a/src/Literate.jl b/src/Literate.jl index b97f11c..2b0dc5b 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -6,7 +6,7 @@ https://fredrikekre.github.io/Literate.jl/ for documentation. """ module Literate -import JSON, REPL, IOCapture, Markdown +import JSON, REPL, IOCapture, CommonMark include("IJulia.jl") import .IJulia @@ -550,44 +550,39 @@ function containsAdmonition(chunk) end function chunkToMD(chunk) + parser = CommonMark.Parser() + CommonMark.enable!(parser, CommonMark.AdmonitionRule()) buffer = IOBuffer() for line in chunk.lines - write(buffer, line.first * line.second, "\n\n") + write(buffer, line.first * line.second, '\n') end str = String(take!(buffer)) - return Markdown.parse(str) + return parser(str) end function rewriteContent!(mdContent) - for (i, item) in enumerate(mdContent.content) - if isa(item, Markdown.Admonition) - mdContent.content[i] = if any(x -> x isa Markdown.Admonition, item.content) - admonition_to_fenced_div(rewriteContent!(item)) + for (node, entering) in mdContent + if isa(node.t, CommonMark.Admonition) + admonition = node.t + node.t = CommonMark.Paragraph() + if admonition.category == "yaml" + node.first_child.t = CommonMark.Text() + node.first_child.nxt.t = CommonMark.Paragraph() + CommonMark.insert_after(node.first_child.nxt.last_child, CommonMark.text("\n---\n")) + CommonMark.insert_before(node.first_child.nxt, CommonMark.text("---\n")) else - admonition_to_fenced_div(item) + CommonMark.insert_before(node, CommonMark.text(""":::::: $(admonition.category) + + ## $(admonition.title) + + """)) + CommonMark.insert_after(node, CommonMark.text("::::::\n\n")) end end end mdContent end - -#Functions needed for addition transformation into Carpentries style. Markdown style into pandoc fenced divs - -function admonition_to_fenced_div(admonition) - category = admonition.category - if category == "yaml" - single_paragraph = Markdown.Paragraph(vcat("---\n", - reduce(vcat, map(Base.Fix2(getproperty, :content), Iterators.filter(Base.Fix2(!isa, Markdown.HorizontalRule), admonition.content))) .* '\n', - "---\n" - )) - return single_paragraph - end - title = admonition.title - vcat(Markdown.Paragraph(":::::::: $category \n"), - isempty(title) ? [] : Markdown.Header{2}(title), admonition.content, Markdown.Paragraph("::::::::\n\n")) -end - #_______________________________________________________________________________________ @@ -631,8 +626,9 @@ function write_md_chunks!(iomd, chunks, outputdir, config) if flavor isa CarpentriesFlavor if containsAdmonition(chunk) md_chunk = chunkToMD(chunk) + # CommonMark.term(stdout, md_chunk) rewriteContent!(md_chunk) - write(iomd, string(Markdown.MD(md_chunk))) + CommonMark.markdown(iomd, md_chunk) continue end end From b1e267b2a3d07f560d9380ba70d595f0dd7813cb Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Tue, 8 Aug 2023 14:27:45 +0200 Subject: [PATCH 23/24] clean up --- src/Literate.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 2b0dc5b..f250a3d 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -622,17 +622,14 @@ function write_md_chunks!(iomd, chunks, outputdir, config) for (chunknum, chunk) in enumerate(chunks) if isa(chunk, MDChunk) - #______________________________________________________________________________________________________________ if flavor isa CarpentriesFlavor if containsAdmonition(chunk) md_chunk = chunkToMD(chunk) - # CommonMark.term(stdout, md_chunk) rewriteContent!(md_chunk) CommonMark.markdown(iomd, md_chunk) continue end end - #______________________________________________________________________________________________________________ for line in chunk.lines write(iomd, line.second, '\n') # skip indent here From 42548626b366a67f2d7a16034f61cef9896f144e Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Tue, 8 Aug 2023 16:18:58 +0200 Subject: [PATCH 24/24] handle output --- src/Literate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Literate.jl b/src/Literate.jl index f250a3d..0be0be4 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -681,7 +681,7 @@ function execute_markdown!(io::IO, sb::Module, block::String, outputdir; r, str, _ = execute_block(sb, block; inputfile=inputfile, fake_source=fake_source) # issue #101: consecutive codefenced blocks need newline # issue #144: quadruple backticks allow for triple backticks in the output - plain_fence = "\n````\n" => "\n````" + plain_fence = "\n````$(flavor == CarpentriesFlavor() ? "output" : "")\n" => "\n````" # Here CarpentiresFlavor fork... if r !== nothing && !REPL.ends_with_semicolon(block) if (flavor isa FranklinFlavor || flavor isa DocumenterFlavor) &&