allow leading whitespace before #

docs/src/fileformat.md

@@ -11,8 +11,9 @@ The basic syntax is simple:
 - lines starting with `# ` are treated as markdown,
 - all other lines are treated as julia code.
 
-!!! note
+Leading whitespace is allowed before `#`, but it will be removed when generating the
-    If you want regular julia comments in the source file use `## ` instead of `# `.
+output. Since `#`-lines is treated as markdown we can not use that for regular julia
+comments, for this you can instead use `##`, which will render as `#` in the output.
 
 Lets look at a simple example:
 ```julia

src/Literate.jl

@@ -1,7 +1,7 @@
 __precompile__()
 module Literate
 
-import Compat: replace, popfirst!, @error, @info
+import Compat: replace, popfirst!, @error, @info, occursin
 
 import JSON
 
@@ -25,7 +25,7 @@ import .Documenter
 # Parser
 abstract type Chunk end
 struct MDChunk <: Chunk
-    lines::Vector{String}
+    lines::Vector{Pair{String,String}} # indent and content
 end
 MDChunk() = MDChunk(String[])
 mutable struct CodeChunk <: Chunk
@@ -34,7 +34,7 @@ mutable struct CodeChunk <: Chunk
 end
 CodeChunk() = CodeChunk(String[], false)
 
-ismdline(line) = (line == "#" || startswith(line, "# ")) && !startswith(line, "##")
+ismdline(line) = (occursin(r"^\h*#$", line) || occursin(r"^\h*# .*$", line)) && !occursin(r"^\h*##", line)
 
 function parse(content; allow_continued = true)
     lines = collect(eachline(IOBuffer(content)))
@@ -44,22 +44,25 @@ function parse(content; allow_continued = true)
 
     for line in lines
         line = rstrip(line)
-        if startswith(line, "#-") # new chunk
+        # print("line = `$line`: ")
+        if occursin(r"^\h*#-", line) # new chunk
             # assume same as last chunk, will be cleaned up otherwise
             push!(chunks, typeof(chunks[end])())
         elseif ismdline(line) # markdown
             if !(chunks[end] isa MDChunk)
                 push!(chunks, MDChunk())
             end
-            # remove "# " and "#\n"
+            # capture what is before and after # (need to store the indent)
-            line = replace(replace(line, r"^# " => ""), r"^#$" => "")
+            m = match(r"^(\h*)#( (.*))?$", line)
-            push!(chunks[end].lines, line)
+            indent = convert(String, m.captures[1])
+            linecontent = m.captures[3] === nothing ? "" : convert(String, m.captures[3])
+            push!(chunks[end].lines, indent => linecontent)
         else # code
             if !(chunks[end] isa CodeChunk)
                 push!(chunks, CodeChunk())
             end
             # remove "## " and "##\n"
-            line = replace(replace(line, r"^## " => "# "), r"^##$" => "#")
+            line = replace(replace(line, r"^(\h*)#(# .*)$" => s"\1\2"), r"^(\h*#)#$" => "\1")
             push!(chunks[end].lines, line)
         end
     end
@@ -70,10 +73,10 @@ function parse(content; allow_continued = true)
     filter!(x -> !all(y -> isempty(y), x.lines), chunks)
     ## remove leading/trailing empty lines
     for chunk in chunks
-        while isempty(chunk.lines[1])
+        while chunk.lines[1] == "" || chunk.lines[1] == ("" => "")
             popfirst!(chunk.lines)
         end
-        while isempty(chunk.lines[end])
+        while chunk.lines[end] == "" || chunk.lines[end] == ("" => "")
             pop!(chunk.lines)
         end
     end
@@ -102,7 +105,7 @@ function parse(content; allow_continued = true)
                     append!(merged_chunks[end].lines, chunk.lines)
                 else # need to put back "#"
                     for line in chunk.lines
-                        push!(merged_chunks[end].lines, rstrip("# " * line))
+                        push!(merged_chunks[end].lines, rstrip(line.first * "# " * line.second))
                     end
                 end
             else
@@ -253,7 +256,7 @@ function script(inputfile, outputdir; preprocess = identity, postprocess = ident
             write(ioscript, '\n') # add a newline between each chunk
         elseif isa(chunk, MDChunk) && keep_comments
             for line in chunk.lines
-                write(ioscript, rstrip("# " * line * '\n'))
+                write(ioscript, rstrip(line.first * "# " * line.second * '\n'))
             end
             write(ioscript, '\n') # add a newline between each chunk
         end
@@ -336,7 +339,7 @@ function markdown(inputfile, outputdir; preprocess = identity, postprocess = ide
     for chunk in chunks
         if isa(chunk, MDChunk)
             for line in chunk.lines
-                write(iomd, line, '\n')
+                write(iomd, line.second, '\n') # skip indent here
             end
         else # isa(chunk, CodeChunk)
             write(iomd, codefence.first)
@@ -419,8 +422,9 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide
         if isa(chunk, MDChunk)
             cell["cell_type"] = "markdown"
             cell["metadata"] = Dict()
-            @views map!(x -> x * '\n', chunk.lines[1:end-1], chunk.lines[1:end-1])
+            lines = String[x.second for x in chunk.lines] # skip indent
-            cell["source"] = chunk.lines
+            @views map!(x -> x * '\n', lines[1:end-1], lines[1:end-1])
+            cell["source"] = lines
             cell["outputs"] = []
         else # isa(chunk, CodeChunk)
             cell["cell_type"] = "code"

test/runtests.jl

@@ -10,9 +10,11 @@ function compare_chunks(chunks1, chunks2)
     for (c1, c2) in zip(chunks1, chunks2)
         # compare types
         @test typeof(c1) == typeof(c2)
-        # test that no chunk start or end with ""
+        # test that the chunk don't start or end with empty line
-        @test !isempty(first(c1.lines)); @test !isempty(last(c1.lines))
+        @test first(c1.lines) != "" && first(c1.lines) != ("" => "")
-        @test !isempty(first(c2.lines)); @test !isempty(last(c2.lines))
+        @test first(c2.lines) != "" && first(c2.lines) != ("" => "")
+        @test last(c1.lines) != "" && last(c1.lines) != ("" => "")
+        @test last(c2.lines) != "" && last(c2.lines) != ("" => "")
         # compare content
         for (l1, l2) in zip(c1.lines, c2.lines)
             @test l1 == l2
@@ -86,39 +88,48 @@ end
     Line 58
     ## Line 59
     ##Line 60
+    #-
+        # Line 62
+        # # Line 63
+    Line 64
+        ## Line 65
+        Line 66
+    Line 67
     """
     expected_chunks = Chunk[
-        MDChunk(["Line 1"]),
+        MDChunk(["" => "Line 1"]),
         CodeChunk(["Line 2"], false),
-        MDChunk(["Line 3", "","Line 5"]),
+        MDChunk(["" => "Line 3", "" => "","" => "Line 5"]),
         CodeChunk(["Line 6", "","Line 8"], false),
-        MDChunk(["Line 9"]),
+        MDChunk(["" => "Line 9"]),
-        MDChunk(["Line 11"]),
+        MDChunk(["" => "Line 11"]),
         CodeChunk(["Line 12"], false),
         CodeChunk(["Line 14"], false),
-        MDChunk(["Line 15"]),
+        MDChunk(["" => "Line 15"]),
-        MDChunk(["Line 17"]),
+        MDChunk(["" => "Line 17"]),
         CodeChunk(["Line 18"], false),
         CodeChunk(["Line 20"], false),
-        MDChunk(["Line 21"]),
+        MDChunk(["" => "Line 21"]),
         CodeChunk(["Line 22", "    Line 23", "Line 24"], false),
         CodeChunk(["Line 26", "    Line 27"], true),
         CodeChunk(["Line 29"], false),
         CodeChunk(["Line 31", "    Line 32"], true),
-        MDChunk(["Line 33"]),
+        MDChunk(["" => "Line 33"]),
         CodeChunk(["Line 34"], false),
         CodeChunk(["Line 36"], true),
         CodeChunk(["    Line 38"], true),
         CodeChunk(["Line 40"], false),
         CodeChunk(["Line 42", "    Line 43"], true),
-        MDChunk(["Line 44"]),
+        MDChunk(["" => "Line 44"]),
         CodeChunk(["    Line 45"], true),
-        MDChunk(["Line 46"]),
+        MDChunk(["" => "Line 46"]),
         CodeChunk(["Line 47"], false),
-        MDChunk(["Line 48"]),
+        MDChunk(["" => "Line 48"]),
         CodeChunk(["#Line 49", "Line 50"], false),
-        MDChunk(["Line 53"]),
+        MDChunk(["" => "Line 53"]),
-        CodeChunk(["# Line 57", "Line 58", "# Line 59", "##Line 60"], false)
+        CodeChunk(["# Line 57", "Line 58", "# Line 59", "##Line 60"], false),
+        MDChunk(["    " => "Line 62", "    " => "# Line 63"]),
+        CodeChunk(["Line 64", "    # Line 65", "    Line 66", "Line 67"], false)
         ]
     parsed_chunks = Literate.parse(content)
     compare_chunks(parsed_chunks, expected_chunks)
@@ -178,6 +189,12 @@ content = """
     # ```math
     # \\int f(x) dx
     # ```
+    #-
+        # Indented markdown
+    for i in 1:10
+        # Indented markdown
+        ## Indented comment
+    end
     """
 
 @testset "Literate.script" begin
@@ -213,6 +230,11 @@ content = """
             # PLACEHOLDER3
             # PLACEHOLDER4
 
+            for i in 1:10
+
+                # Indented comment
+            end
+
             # This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl
 
             """
@@ -325,6 +347,19 @@ end
             \\int f(x) dx
             ```
 
+            Indented markdown
+
+            ```@example inputfile; continued = true
+            for i in 1:10
+            ```
+
+            Indented markdown
+
+            ```@example inputfile
+                # Indented comment
+            end
+            ```
+
             *This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*
 
             """
@@ -475,6 +510,21 @@ end
                ]
             """,
 
+            """
+               "source": [
+                "Indented markdown"
+               ]
+            """,
+
+            """
+               "source": [
+                "for i in 1:10\\n",
+                "    # Indented markdown\\n",
+                "    # Indented comment\\n",
+                "end"
+               ]
+            """,
+
             """
                "source": [
                 "*This notebook was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*"