From 93c845379026426bf625598aea2f41535bbd3798 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Mon, 9 Apr 2018 11:21:02 +0200 Subject: [PATCH] execute_notebook --- REQUIRE | 2 +- src/Examples.jl | 96 ++++++++++++++++++++++++++++++++++++++----------- src/IJulia.jl | 73 +++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 22 deletions(-) create mode 100644 src/IJulia.jl diff --git a/REQUIRE b/REQUIRE index f6031fa..dec20ce 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,3 +1,3 @@ julia 0.6 JSON -IJulia +Documenter diff --git a/src/Examples.jl b/src/Examples.jl index 77d80ef..17334a0 100644 --- a/src/Examples.jl +++ b/src/Examples.jl @@ -2,7 +2,10 @@ module Examples import Compat: replace, popfirst!, @error, @info -import JSON, IJulia +import JSON + +include("IJulia.jl") +import .IJulia # # Some simple rules: # @@ -210,6 +213,8 @@ function markdown(inputfile, outputdir; preprocess = identity, postprocess = ide return outputfile end +const JUPYTER_VERSION = v"4.3.0" + """ Examples.notebook(inputfile, outputdir; kwargs...) @@ -249,8 +254,8 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide # create the notebook nb = Dict() - nb["nbformat"] = IJulia.jupyter_vers.major - nb["nbformat_minor"] = IJulia.jupyter_vers.minor + nb["nbformat"] = JUPYTER_VERSION.major + nb["nbformat_minor"] = JUPYTER_VERSION.minor ## create the notebook cells chunks = parse(content) @@ -260,13 +265,13 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide if isa(chunk, MDChunk) cell["cell_type"] = "markdown" cell["metadata"] = Dict() - @views map!(x -> x*'\n', chunk.lines[1:end-1], chunk.lines[1:end-1]) + @views map!(x -> x * '\n', chunk.lines[1:end-1], chunk.lines[1:end-1]) cell["source"] = chunk.lines cell["outputs"] = [] else # isa(chunk, CodeChunk) cell["cell_type"] = "code" cell["metadata"] = Dict() - @views map!(x -> x*'\n', chunk.lines[1:end-1], chunk.lines[1:end-1]) + @views map!(x -> x * '\n', chunk.lines[1:end-1], chunk.lines[1:end-1]) cell["source"] = chunk.lines cell["execution_count"] = nothing cell["outputs"] = [] @@ -293,32 +298,81 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide nb["metadata"] = metadata - ionb = IOBuffer() - JSON.print(ionb, nb, 2) - # custom post-processing from user - content = postprocess(String(take!(ionb))) - - # write to file - isdir(outputdir) || error("not a directory: $(outputdir)") - outputfile = joinpath(outputdir, name * ".ipynb") - - @info "writing result to $(outputfile)" - write(outputfile, content) + nb = postprocess(nb) if execute - @info "executing notebook $(outputfile)" + @info "executing notebook $(name * ".ipynb")" try - run(`$(IJulia.jupyter)-nbconvert --ExecutePreprocessor.timeout=-1 --to notebook --execute $(abspath(outputfile)) --output $(filename(outputfile)).ipynb`) + # run(`jupyter nbconvert --ExecutePreprocessor.timeout=-1 --to notebook --execute $(abspath(outputfile)) --output $(filename(outputfile)).ipynb`) + nb = execute_notebook(nb) catch err - @error "error when executing notebook $(outputfile)" + @error "error when executing notebook $(name * ".ipynb")" rethrow(err) end - # clean up - rm(joinpath(first(splitdir(outputfile)), ".ipynb_checkpoints"), force=true, recursive = true) + # clean up (only needed for jupyter-nbconvert) + rm(joinpath(outputdir, ".ipynb_checkpoints"), force=true, recursive = true) end + # write to file + isdir(outputdir) || error("not a directory: $(outputdir)") + outputfile = joinpath(outputdir, name * ".ipynb") + + @info "writing result to $(outputfile)" + ionb = IOBuffer() + JSON.print(ionb, nb, 1) + write(outputfile, seekstart(ionb)) + return outputfile end +import Documenter # just copy paste Documenter.Utilities.withoutput instead + +function execute_notebook(nb) + # sandbox module for the notebook (TODO: Do this in Main?) + m = Module(gensym()) + io = IOBuffer() + + execution_count = 0 + for cell in nb["cells"] + cell["cell_type"] == "code" || continue + execution_count += 1 + cell["execution_count"] = execution_count + block = join(cell["source"], '\n') + # r is the result + # status = (true|false) + # _: backtrace + # str combined stdout, stderr output + r, status, _, str = Documenter.Utilities.withoutput() do + include_string(m, block) + end + status || error("something went wrong when evaluating code") + + # str should go into stream + if !isempty(str) + stream = Dict{String,Any}() + stream["output_type"] = "stream" + stream["name"] = "stdout" + stream["text"] = collect(Any, eachline(IOBuffer(String(str)), chomp = false)) # 0.7 chomp = false => keep = true + push!(cell["outputs"], stream) + end + + # check if ; is used to suppress output + r = Base.REPL.ends_with_semicolon(block) ? nothing : r + + # r should go into execute_result + if r !== nothing + execute_result = Dict{String,Any}() + execute_result["output_type"] = "execute_result" + execute_result["metadata"] = Dict() + execute_result["execution_count"] = execution_count + execute_result["data"] = IJulia.display_dict(r) + + push!(cell["outputs"], execute_result) + end + + end + nb +end + end # module diff --git a/src/IJulia.jl b/src/IJulia.jl new file mode 100644 index 0000000..be4af52 --- /dev/null +++ b/src/IJulia.jl @@ -0,0 +1,73 @@ + +# minimal (modified) subset of IJulia.jl to evaluate notebooks +module IJulia + +const text_plain = MIME("text/plain") +const image_svg = MIME("image/svg+xml") +const image_png = MIME("image/png") +const image_jpeg = MIME("image/jpeg") +const text_markdown = MIME("text/markdown") +const text_html = MIME("text/html") +const text_latex = MIME("text/latex") # Jupyter expects this +const text_latex2 = MIME("application/x-latex") # but this is more standard? +const application_vnd_vegalite_v2 = MIME("application/vnd.vegalite.v2+json") + +# return a String=>String dictionary of mimetype=>data +# for passing to Jupyter display_data and execute_result messages. +function display_dict(x) + data = Dict{String,Any}("text/plain" => limitstringmime(text_plain, x)) + if mimewritable(application_vnd_vegalite_v2, x) + data[string(application_vnd_vegalite_v2)] = JSON.parse(limitstringmime(application_vnd_vegalite_v2, x)) + end + if mimewritable(image_svg, x) + data[string(image_svg)] = limitstringmime(image_svg, x) + end + if mimewritable(image_png, x) + data[string(image_png)] = limitstringmime(image_png, x) + elseif mimewritable(image_jpeg, x) # don't send jpeg if we have png + data[string(image_jpeg)] = limitstringmime(image_jpeg, x) + end + if mimewritable(text_markdown, x) + data[string(text_markdown)] = limitstringmime(text_markdown, x) + elseif mimewritable(text_html, x) + data[string(text_html)] = limitstringmime(text_html, x) + elseif mimewritable(text_latex, x) + data[string(text_latex)] = limitstringmime(text_latex, x) + elseif mimewritable(text_latex2, x) + data[string(text_latex)] = limitstringmime(text_latex2, x) + end + return data +end + +# need special handling for showing a string as a textmime +# type, since in that case the string is assumed to be +# raw data unless it is text/plain +israwtext(::MIME, x::AbstractString) = true +israwtext(::MIME"text/plain", x::AbstractString) = false +israwtext(::MIME, x) = false + +# convert x to a string of type mime, making sure to use an +# IOContext that tells the underlying show function to limit output +function limitstringmime(mime::MIME, x) + buf = IOBuffer() + if istextmime(mime) + if israwtext(mime, x) + return String(x) + else + # show(IOContext(buf, :limit=>true, :color=>true), mime, x) + Base.invokelatest(show, IOContext(buf, :limit=>true, :color=>true), mime, x) + end + else + b64 = Base64EncodePipe(buf) + if isa(x, Vector{UInt8}) + write(b64, x) # x assumed to be raw binary data + else + # show(IOContext(b64, :limit=>true, :color=>true), mime, x) + Base.invokelatest(show, IOContext(b64, :limit=>true, :color=>true), mime, x) + end + close(b64) + end + return String(take!(buf)) +end + +end # module