RFC: Add support for Jupyter cell metadata via %% syntax (#43)

Admittedly this is a little strange: it adds support for a special first-line
syntax that only applies to Jupyter notebook outputs. But this %% format is
somewhat standard across notebook-generating tools, and the ability to protect
it with a `#nb` leader makes it possible to specifically target this line to a
notebook output. We aim to use something like this for generation of notebooks
with support for nbgrader's metadata extensions, but it could also be used to
specify cell types for Jupyter notebook presentations \(with presentation
extensions\) or other such fun. Fixes #67.

Co-authored-by: Matt Bauman <mbauman@gmail.com>
Co-authored-by: Fredrik Ekre <ekrefredrik@gmail.com>

docs/src/outputformats.md

@@ -51,6 +51,22 @@ arguments to [`Literate.notebook`](@ref):
 Literate.notebook
 ```
 
+### Notebook metadata
+
+Jupyter notebook cells (both code cells and markdown cells) can contain metadata. This is enabled
+in Literate by the `%%` token, similar to
+[Jupytext](https://jupytext.readthedocs.io/en/latest/formats.html#the-percent-format).
+The format is as follows
+
+```
+%% optional ignored text [type] {optional metadata JSON}
+```
+
+Cell metadata can, for example, be used for
+[nbgrader](https://nbgrader.readthedocs.io/en/stable/contributor_guide/metadata.html)
+and the [reveal.js](https://github.com/hakimel/reveal.js) notebook extension
+[RISE](https://github.com/damianavila/RISE).
+
 
 ## [**4.3.** Script Output](@id Script-Output)
 

src/Literate.jl

@@ -398,6 +398,19 @@ end
 
 const JUPYTER_VERSION = v"4.3.0"
 
+parse_nbmeta(line::Pair) = parse_nbmeta(line.second)
+function parse_nbmeta(line)
+    # Format: %% optional ignored text [type] {optional metadata JSON}
+    # Cf. https://jupytext.readthedocs.io/en/latest/formats.html#the-percent-format
+    m = match(r"^%% ([^[{]+)?\s*(?:\[(\w+)\])?\s*(\{.*)?$", line)
+    typ = m.captures[2]
+    name = m.captures[1] === nothing ? Dict{String, String}() : Dict("name" => m.captures[1])
+    meta = m.captures[3] === nothing ? Dict{String, Any}() : JSON.parse(m.captures[3])
+    return typ, merge(name, meta)
+end
+line_is_nbmeta(line::Pair) = line_is_nbmeta(line.second)
+line_is_nbmeta(line) = startswith(line, "%% ")
+
 """
     Literate.notebook(inputfile, outputdir; kwargs...)
 
@@ -451,20 +464,24 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide
     cells = []
     for chunk in chunks
         cell = Dict()
-        if isa(chunk, MDChunk)
+        chunktype = isa(chunk, MDChunk) ? "markdown" : "code"
-            cell["cell_type"] = "markdown"
+        if !isempty(chunk.lines) && line_is_nbmeta(chunk.lines[1])
-            cell["metadata"] = Dict()
+            metatype, metadata = parse_nbmeta(chunk.lines[1])
-            lines = String[x.second for x in chunk.lines] # skip indent
+            metatype !== nothing && metatype != chunktype && error("specifying a different cell type is not supported")
-            @views map!(x -> x * '\n', lines[1:end-1], lines[1:end-1])
+            popfirst!(chunk.lines)
-            cell["source"] = lines
+        else
-            cell["outputs"] = []
+            metadata = Dict{String, Any}()
-        else # isa(chunk, CodeChunk)
+        end
-            cell["cell_type"] = "code"
+        lines = isa(chunk, MDChunk) ?
-            cell["metadata"] = Dict()
+                    String[x.second for x in chunk.lines] : # skip indent
-            @views map!(x -> x * '\n', chunk.lines[1:end-1], chunk.lines[1:end-1])
+                    chunk.lines
-            cell["source"] = chunk.lines
+        @views map!(x -> x * '\n', lines[1:end-1], lines[1:end-1])
+        cell["cell_type"] = chunktype
+        cell["metadata"] = metadata
+        cell["source"] = lines
+        cell["outputs"] = []
+        if chunktype == "code"
             cell["execution_count"] = nothing
-            cell["outputs"] = []
         end
         push!(cells, cell)
     end

test/runtests.jl

@@ -208,6 +208,18 @@ content = """
     #+
         ## Indented comment
     end
+
+    #nb # A notebook cell with special metadata
+    #nb %% Meta1 {"meta": "data"}
+    #nb 1+1
+    #nb #-
+    #nb # A explicit code notebook cell
+    #nb #-
+    #nb %% [code]
+    #nb 1+2
+    #nb #-
+    #nb # %% [markdown] {"meta": "data"}
+    #nb # # Explicit markdown cell with metadata
     """
 
 @testset "Literate.script" begin
@@ -584,6 +596,12 @@ end
                ]
             """,
 
+            """
+               "metadata": {
+                "meta": "data"
+               }
+            """,
+
             """
                "source": [
                 "*This notebook was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*"
@@ -691,6 +709,7 @@ end
             r = try
                 Literate.notebook(inputfile, outdir)
             catch err
+                @info "^^ the above error log message is expected ^^"
                 err
             end
             @test isa(r, ErrorException)