execute_notebook

REQUIRE

@@ -1,3 +1,3 @@
 julia 0.6
 JSON
-IJulia
+Documenter

src/Examples.jl

@@ -2,7 +2,10 @@ module Examples
 
 import Compat: replace, popfirst!, @error, @info
 
-import JSON, IJulia
+import JSON
+
+include("IJulia.jl")
+import .IJulia
 
 # # Some simple rules:
 #
@@ -210,6 +213,8 @@ function markdown(inputfile, outputdir; preprocess = identity, postprocess = ide
     return outputfile
 end
 
+const JUPYTER_VERSION = v"4.3.0"
+
 """
     Examples.notebook(inputfile, outputdir; kwargs...)
 
@@ -249,8 +254,8 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide
 
     # create the notebook
     nb = Dict()
-    nb["nbformat"] = IJulia.jupyter_vers.major
+    nb["nbformat"] = JUPYTER_VERSION.major
-    nb["nbformat_minor"] = IJulia.jupyter_vers.minor
+    nb["nbformat_minor"] = JUPYTER_VERSION.minor
 
     ## create the notebook cells
     chunks = parse(content)
@@ -260,13 +265,13 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide
         if isa(chunk, MDChunk)
             cell["cell_type"] = "markdown"
             cell["metadata"] = Dict()
-            @views map!(x -> x*'\n', chunk.lines[1:end-1], chunk.lines[1:end-1])
+            @views map!(x -> x * '\n', chunk.lines[1:end-1], chunk.lines[1:end-1])
             cell["source"] = chunk.lines
             cell["outputs"] = []
         else # isa(chunk, CodeChunk)
             cell["cell_type"] = "code"
             cell["metadata"] = Dict()
-            @views map!(x -> x*'\n', chunk.lines[1:end-1], chunk.lines[1:end-1])
+            @views map!(x -> x * '\n', chunk.lines[1:end-1], chunk.lines[1:end-1])
             cell["source"] = chunk.lines
             cell["execution_count"] = nothing
             cell["outputs"] = []
@@ -293,32 +298,81 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide
 
     nb["metadata"] = metadata
 
-    ionb = IOBuffer()
-    JSON.print(ionb, nb, 2)
-
     # custom post-processing from user
-    content = postprocess(String(take!(ionb)))
+    nb = postprocess(nb)
-
-    # write to file
-    isdir(outputdir) || error("not a directory: $(outputdir)")
-    outputfile = joinpath(outputdir, name * ".ipynb")
-
-    @info "writing result to $(outputfile)"
-    write(outputfile, content)
 
     if execute
-        @info "executing notebook $(outputfile)"
+        @info "executing notebook $(name * ".ipynb")"
         try
-            run(`$(IJulia.jupyter)-nbconvert --ExecutePreprocessor.timeout=-1 --to notebook --execute $(abspath(outputfile)) --output $(filename(outputfile)).ipynb`)
+            # run(`jupyter nbconvert --ExecutePreprocessor.timeout=-1 --to notebook --execute $(abspath(outputfile)) --output $(filename(outputfile)).ipynb`)
+            nb = execute_notebook(nb)
         catch err
-            @error "error when executing notebook $(outputfile)"
+            @error "error when executing notebook $(name * ".ipynb")"
             rethrow(err)
         end
-        # clean up
+        # clean up (only needed for jupyter-nbconvert)
-        rm(joinpath(first(splitdir(outputfile)), ".ipynb_checkpoints"), force=true, recursive = true)
+        rm(joinpath(outputdir, ".ipynb_checkpoints"), force=true, recursive = true)
     end
 
+    # write to file
+    isdir(outputdir) || error("not a directory: $(outputdir)")
+    outputfile = joinpath(outputdir, name * ".ipynb")
+
+    @info "writing result to $(outputfile)"
+    ionb = IOBuffer()
+    JSON.print(ionb, nb, 1)
+    write(outputfile, seekstart(ionb))
+
     return outputfile
 end
 
+import Documenter # just copy paste Documenter.Utilities.withoutput instead
+
+function execute_notebook(nb)
+    # sandbox module for the notebook (TODO: Do this in Main?)
+    m = Module(gensym())
+    io = IOBuffer()
+
+    execution_count = 0
+    for cell in nb["cells"]
+        cell["cell_type"] == "code" || continue
+        execution_count += 1
+        cell["execution_count"] = execution_count
+        block = join(cell["source"], '\n')
+        # r is the result
+        # status = (true|false)
+        # _: backtrace
+        # str combined stdout, stderr output
+        r, status, _, str = Documenter.Utilities.withoutput() do
+            include_string(m, block)
+        end
+        status || error("something went wrong when evaluating code")
+
+        # str should go into stream
+        if !isempty(str)
+            stream = Dict{String,Any}()
+            stream["output_type"] = "stream"
+            stream["name"] = "stdout"
+            stream["text"] = collect(Any, eachline(IOBuffer(String(str)), chomp = false)) # 0.7 chomp = false => keep = true
+            push!(cell["outputs"], stream)
+        end
+
+        # check if ; is used to suppress output
+        r = Base.REPL.ends_with_semicolon(block) ? nothing : r
+
+        # r should go into execute_result
+        if r !== nothing
+            execute_result = Dict{String,Any}()
+            execute_result["output_type"] = "execute_result"
+            execute_result["metadata"] = Dict()
+            execute_result["execution_count"] = execution_count
+            execute_result["data"] = IJulia.display_dict(r)
+
+            push!(cell["outputs"], execute_result)
+        end
+
+    end
+    nb
+end
+
 end # module

src/IJulia.jl

@@ -0,0 +1,73 @@
+
+# minimal (modified) subset of IJulia.jl to evaluate notebooks
+module IJulia
+
+const text_plain = MIME("text/plain")
+const image_svg = MIME("image/svg+xml")
+const image_png = MIME("image/png")
+const image_jpeg = MIME("image/jpeg")
+const text_markdown = MIME("text/markdown")
+const text_html = MIME("text/html")
+const text_latex = MIME("text/latex") # Jupyter expects this
+const text_latex2 = MIME("application/x-latex") # but this is more standard?
+const application_vnd_vegalite_v2 = MIME("application/vnd.vegalite.v2+json")
+
+# return a String=>String dictionary of mimetype=>data
+# for passing to Jupyter display_data and execute_result messages.
+function display_dict(x)
+    data = Dict{String,Any}("text/plain" => limitstringmime(text_plain, x))
+    if mimewritable(application_vnd_vegalite_v2, x)
+        data[string(application_vnd_vegalite_v2)] = JSON.parse(limitstringmime(application_vnd_vegalite_v2, x))
+    end
+    if mimewritable(image_svg, x)
+        data[string(image_svg)] = limitstringmime(image_svg, x)
+    end
+    if mimewritable(image_png, x)
+        data[string(image_png)] = limitstringmime(image_png, x)
+    elseif mimewritable(image_jpeg, x) # don't send jpeg if we have png
+        data[string(image_jpeg)] = limitstringmime(image_jpeg, x)
+    end
+    if mimewritable(text_markdown, x)
+        data[string(text_markdown)] = limitstringmime(text_markdown, x)
+    elseif mimewritable(text_html, x)
+        data[string(text_html)] = limitstringmime(text_html, x)
+    elseif mimewritable(text_latex, x)
+        data[string(text_latex)] = limitstringmime(text_latex, x)
+    elseif mimewritable(text_latex2, x)
+        data[string(text_latex)] = limitstringmime(text_latex2, x)
+    end
+    return data
+end
+
+# need special handling for showing a string as a textmime
+# type, since in that case the string is assumed to be
+# raw data unless it is text/plain
+israwtext(::MIME, x::AbstractString) = true
+israwtext(::MIME"text/plain", x::AbstractString) = false
+israwtext(::MIME, x) = false
+
+# convert x to a string of type mime, making sure to use an
+# IOContext that tells the underlying show function to limit output
+function limitstringmime(mime::MIME, x)
+    buf = IOBuffer()
+    if istextmime(mime)
+        if israwtext(mime, x)
+            return String(x)
+        else
+            # show(IOContext(buf, :limit=>true, :color=>true), mime, x)
+            Base.invokelatest(show, IOContext(buf, :limit=>true, :color=>true), mime, x)
+        end
+    else
+        b64 = Base64EncodePipe(buf)
+        if isa(x, Vector{UInt8})
+            write(b64, x) # x assumed to be raw binary data
+        else
+            # show(IOContext(b64, :limit=>true, :color=>true), mime, x)
+            Base.invokelatest(show, IOContext(b64, :limit=>true, :color=>true), mime, x)
+        end
+        close(b64)
+    end
+    return String(take!(buf))
+end
+
+end # module