diff --git a/docs/src/fileformat.md b/docs/src/fileformat.md index b25fc40..8e7a49e 100644 --- a/docs/src/fileformat.md +++ b/docs/src/fileformat.md @@ -11,8 +11,9 @@ The basic syntax is simple: - lines starting with `# ` are treated as markdown, - all other lines are treated as julia code. -!!! note - If you want regular julia comments in the source file use `## ` instead of `# `. +Leading whitespace is allowed before `#`, but it will be removed when generating the +output. Since `#`-lines is treated as markdown we can not use that for regular julia +comments, for this you can instead use `##`, which will render as `#` in the output. Lets look at a simple example: ```julia diff --git a/src/Literate.jl b/src/Literate.jl index 131e569..eff3d82 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -1,7 +1,7 @@ __precompile__() module Literate -import Compat: replace, popfirst!, @error, @info +import Compat: replace, popfirst!, @error, @info, occursin import JSON @@ -25,7 +25,7 @@ import .Documenter # Parser abstract type Chunk end struct MDChunk <: Chunk - lines::Vector{String} + lines::Vector{Pair{String,String}} # indent and content end MDChunk() = MDChunk(String[]) mutable struct CodeChunk <: Chunk @@ -34,7 +34,7 @@ mutable struct CodeChunk <: Chunk end CodeChunk() = CodeChunk(String[], false) -ismdline(line) = (line == "#" || startswith(line, "# ")) && !startswith(line, "##") +ismdline(line) = (occursin(r"^\h*#$", line) || occursin(r"^\h*# .*$", line)) && !occursin(r"^\h*##", line) function parse(content; allow_continued = true) lines = collect(eachline(IOBuffer(content))) @@ -44,22 +44,25 @@ function parse(content; allow_continued = true) for line in lines line = rstrip(line) - if startswith(line, "#-") # new chunk + # print("line = `$line`: ") + if occursin(r"^\h*#-", line) # new chunk # assume same as last chunk, will be cleaned up otherwise push!(chunks, typeof(chunks[end])()) elseif ismdline(line) # markdown if !(chunks[end] isa MDChunk) push!(chunks, MDChunk()) end - # remove "# " and "#\n" - line = replace(replace(line, r"^# " => ""), r"^#$" => "") - push!(chunks[end].lines, line) + # capture what is before and after # (need to store the indent) + m = match(r"^(\h*)#( (.*))?$", line) + indent = convert(String, m.captures[1]) + linecontent = m.captures[3] === nothing ? "" : convert(String, m.captures[3]) + push!(chunks[end].lines, indent => linecontent) else # code if !(chunks[end] isa CodeChunk) push!(chunks, CodeChunk()) end # remove "## " and "##\n" - line = replace(replace(line, r"^## " => "# "), r"^##$" => "#") + line = replace(replace(line, r"^(\h*)#(# .*)$" => s"\1\2"), r"^(\h*#)#$" => "\1") push!(chunks[end].lines, line) end end @@ -70,10 +73,10 @@ function parse(content; allow_continued = true) filter!(x -> !all(y -> isempty(y), x.lines), chunks) ## remove leading/trailing empty lines for chunk in chunks - while isempty(chunk.lines[1]) + while chunk.lines[1] == "" || chunk.lines[1] == ("" => "") popfirst!(chunk.lines) end - while isempty(chunk.lines[end]) + while chunk.lines[end] == "" || chunk.lines[end] == ("" => "") pop!(chunk.lines) end end @@ -102,7 +105,7 @@ function parse(content; allow_continued = true) append!(merged_chunks[end].lines, chunk.lines) else # need to put back "#" for line in chunk.lines - push!(merged_chunks[end].lines, rstrip("# " * line)) + push!(merged_chunks[end].lines, rstrip(line.first * "# " * line.second)) end end else @@ -253,7 +256,7 @@ function script(inputfile, outputdir; preprocess = identity, postprocess = ident write(ioscript, '\n') # add a newline between each chunk elseif isa(chunk, MDChunk) && keep_comments for line in chunk.lines - write(ioscript, rstrip("# " * line * '\n')) + write(ioscript, rstrip(line.first * "# " * line.second * '\n')) end write(ioscript, '\n') # add a newline between each chunk end @@ -336,7 +339,7 @@ function markdown(inputfile, outputdir; preprocess = identity, postprocess = ide for chunk in chunks if isa(chunk, MDChunk) for line in chunk.lines - write(iomd, line, '\n') + write(iomd, line.second, '\n') # skip indent here end else # isa(chunk, CodeChunk) write(iomd, codefence.first) @@ -419,8 +422,9 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide if isa(chunk, MDChunk) cell["cell_type"] = "markdown" cell["metadata"] = Dict() - @views map!(x -> x * '\n', chunk.lines[1:end-1], chunk.lines[1:end-1]) - cell["source"] = chunk.lines + lines = String[x.second for x in chunk.lines] # skip indent + @views map!(x -> x * '\n', lines[1:end-1], lines[1:end-1]) + cell["source"] = lines cell["outputs"] = [] else # isa(chunk, CodeChunk) cell["cell_type"] = "code" diff --git a/test/runtests.jl b/test/runtests.jl index 0fca534..0d12300 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,9 +10,11 @@ function compare_chunks(chunks1, chunks2) for (c1, c2) in zip(chunks1, chunks2) # compare types @test typeof(c1) == typeof(c2) - # test that no chunk start or end with "" - @test !isempty(first(c1.lines)); @test !isempty(last(c1.lines)) - @test !isempty(first(c2.lines)); @test !isempty(last(c2.lines)) + # test that the chunk don't start or end with empty line + @test first(c1.lines) != "" && first(c1.lines) != ("" => "") + @test first(c2.lines) != "" && first(c2.lines) != ("" => "") + @test last(c1.lines) != "" && last(c1.lines) != ("" => "") + @test last(c2.lines) != "" && last(c2.lines) != ("" => "") # compare content for (l1, l2) in zip(c1.lines, c2.lines) @test l1 == l2 @@ -86,39 +88,48 @@ end Line 58 ## Line 59 ##Line 60 + #- + # Line 62 + # # Line 63 + Line 64 + ## Line 65 + Line 66 + Line 67 """ expected_chunks = Chunk[ - MDChunk(["Line 1"]), + MDChunk(["" => "Line 1"]), CodeChunk(["Line 2"], false), - MDChunk(["Line 3", "","Line 5"]), + MDChunk(["" => "Line 3", "" => "","" => "Line 5"]), CodeChunk(["Line 6", "","Line 8"], false), - MDChunk(["Line 9"]), - MDChunk(["Line 11"]), + MDChunk(["" => "Line 9"]), + MDChunk(["" => "Line 11"]), CodeChunk(["Line 12"], false), CodeChunk(["Line 14"], false), - MDChunk(["Line 15"]), - MDChunk(["Line 17"]), + MDChunk(["" => "Line 15"]), + MDChunk(["" => "Line 17"]), CodeChunk(["Line 18"], false), CodeChunk(["Line 20"], false), - MDChunk(["Line 21"]), + MDChunk(["" => "Line 21"]), CodeChunk(["Line 22", " Line 23", "Line 24"], false), CodeChunk(["Line 26", " Line 27"], true), CodeChunk(["Line 29"], false), CodeChunk(["Line 31", " Line 32"], true), - MDChunk(["Line 33"]), + MDChunk(["" => "Line 33"]), CodeChunk(["Line 34"], false), CodeChunk(["Line 36"], true), CodeChunk([" Line 38"], true), CodeChunk(["Line 40"], false), CodeChunk(["Line 42", " Line 43"], true), - MDChunk(["Line 44"]), + MDChunk(["" => "Line 44"]), CodeChunk([" Line 45"], true), - MDChunk(["Line 46"]), + MDChunk(["" => "Line 46"]), CodeChunk(["Line 47"], false), - MDChunk(["Line 48"]), + MDChunk(["" => "Line 48"]), CodeChunk(["#Line 49", "Line 50"], false), - MDChunk(["Line 53"]), - CodeChunk(["# Line 57", "Line 58", "# Line 59", "##Line 60"], false) + MDChunk(["" => "Line 53"]), + CodeChunk(["# Line 57", "Line 58", "# Line 59", "##Line 60"], false), + MDChunk([" " => "Line 62", " " => "# Line 63"]), + CodeChunk(["Line 64", " # Line 65", " Line 66", "Line 67"], false) ] parsed_chunks = Literate.parse(content) compare_chunks(parsed_chunks, expected_chunks) @@ -178,6 +189,12 @@ content = """ # ```math # \\int f(x) dx # ``` + #- + # Indented markdown + for i in 1:10 + # Indented markdown + ## Indented comment + end """ @testset "Literate.script" begin @@ -213,6 +230,11 @@ content = """ # PLACEHOLDER3 # PLACEHOLDER4 + for i in 1:10 + + # Indented comment + end + # This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl """ @@ -325,6 +347,19 @@ end \\int f(x) dx ``` + Indented markdown + + ```@example inputfile; continued = true + for i in 1:10 + ``` + + Indented markdown + + ```@example inputfile + # Indented comment + end + ``` + *This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* """ @@ -475,6 +510,21 @@ end ] """, + """ + "source": [ + "Indented markdown" + ] + """, + + """ + "source": [ + "for i in 1:10\\n", + " # Indented markdown\\n", + " # Indented comment\\n", + "end" + ] + """, + """ "source": [ "*This notebook was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*"