From 3e11724c8faba7f7dce82f923de185ccd2df86b2 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Fri, 21 Aug 2020 11:32:30 +0200 Subject: [PATCH 1/6] WIP: Output Pluto-flavored notebooks. --- src/Literate.jl | 78 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 63bc675..74a2008 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -604,16 +604,17 @@ Generate a notebook from `inputfile` and write the result to `outputdir`. See the manual section on [Configuration](@ref) for documentation of possible configuration with `config` and other keyword arguments. """ -function notebook(inputfile, outputdir=pwd(); config::Dict=Dict(), kwargs...) +function notebook(inputfile, outputdir=pwd(); config::Dict=Dict(), flavor=:jupyter, kwargs...) # preprocessing and parsing chunks, config = preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:nb) # create the notebook - nb = jupyter_notebook(chunks, config) + nb = flavor == :jupyter ? jupyter_notebook(chunks, config) : pluto_notebook(chunks, config) # write to file - outputfile = write_result(nb, config; print = (io, c)->JSON.print(io, c, 1)) + print = flavor == :jupyter ? (io, c)->JSON.print(io, c, 1) : Base.print + outputfile = write_result(nb, config; print = print) return outputfile end @@ -742,6 +743,77 @@ function execute_notebook(nb; inputfile::String="") return nb end +function pluto_notebook(chunks, config) + ionb = IOBuffer() + # Print header + write(ionb, """ + ### A Pluto.jl notebook ### + # v0.11.0 + + using Markdown + + """) + + # Print cells + uuids = Base.UUID[] + for (i, chunk) in enumerate(chunks) + io = IOBuffer() + if isa(chunk, MDChunk) + if length(chunk.lines) == 1 + line = escape_string(chunk.lines[1].second, '"') + write(io, "md\"", line, "\"\n") + else + write(io, "md\"\"\"\n") + for line in chunk.lines + write(io, line.second, '\n') # Skip indent + end + write(io, "\"\"\"\n") + end + content = String(take!(io)) + else # isa(chunk, CodeChunk) + for line in chunk.lines + write(io, line, '\n') + end + content = String(take!(io)) + # Compute number of expressions in the code block and perhaps wrap in begin/end + nexprs, idx = 0, 1 + while true + ex, idx = Meta.parse(content, idx) + ex === nothing && break + nexprs += 1 + end + if nexprs > 1 + io = IOBuffer() + print(io, "begin\n") + foreach(l -> print(io, " ", l, '\n'), eachline(IOBuffer(content))) + print(io, "end\n") + content = String(take!(io)) + end + end + uuid = uuid4(content, i) + push!(uuids, uuid) + print(ionb, "# ╔═╡ ", uuid, '\n') + write(ionb, content, '\n') + end + + # Print cell order + print(ionb, "# ╔═╡ Cell order:\n") + foreach(x -> print(ionb, "# ╠═", x, '\n'), uuids) + + # custom post-processing from user + nb = config["postprocess"](String(take!(ionb))) + return nb +end + +# UUID v4 from cell content and cell number (to keep it somewhat stable) +function uuid4(c, n) + c, n = hash(c), hash(n) + u = (convert(UInt128, c) << 64) ⊻ convert(UInt128, n) + u &= 0xffffffffffff0fff3fffffffffffffff + u |= 0x00000000000040008000000000000000 + return Base.UUID(u) +end + # Create a sandbox module for evaluation function sandbox() m = Module(gensym()) From 872c00d1e019beebc80433797e39c0a69989bfbf Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Fri, 4 Sep 2020 01:30:36 +0200 Subject: [PATCH 2/6] folding cells --- src/Literate.jl | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 74a2008..352345f 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -373,7 +373,7 @@ Available options: """ const DEFAULT_CONFIGURATION=nothing # Dummy const for documentation -function preprocessor(inputfile, outputdir; user_config, user_kwargs, type) +function preprocessor(inputfile, outputdir; user_config, user_kwargs, type, flavor) # Create configuration by merging default and userdefined config = create_configuration(inputfile; user_config=user_config, user_kwargs=user_kwargs, type=type) @@ -393,7 +393,7 @@ function preprocessor(inputfile, outputdir; user_config, user_kwargs, type) # Add some information for passing around Literate methods config["literate_inputfile"] = inputfile config["literate_outputdir"] = outputdir - config["literate_ext"] = type === (:nb) ? ".ipynb" : ".$(type)" + config["literate_ext"] = type === (:nb) ? (flavor === :pluto ? ".jl" : ".ipynb") : ".$(type)" # read content content = read(inputfile, String) @@ -607,7 +607,7 @@ of possible configuration with `config` and other keyword arguments. function notebook(inputfile, outputdir=pwd(); config::Dict=Dict(), flavor=:jupyter, kwargs...) # preprocessing and parsing chunks, config = - preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:nb) + preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:nb, flavor=flavor) # create the notebook nb = flavor == :jupyter ? jupyter_notebook(chunks, config) : pluto_notebook(chunks, config) @@ -756,8 +756,22 @@ function pluto_notebook(chunks, config) # Print cells uuids = Base.UUID[] + folds = Bool[] + default_fold = Dict{String,Bool}("markdown"=>true, "code"=>false) # toggleable ??? for (i, chunk) in enumerate(chunks) io = IOBuffer() + + # Jupyter style metadata # TODO: factor out, identical to jupyter notebook + chunktype = isa(chunk, MDChunk) ? "markdown" : "code" + fold = default_fold[chunktype] + if !isempty(chunk.lines) && line_is_nbmeta(chunk.lines[1]) + @show chunk.lines + metatype, metadata = parse_nbmeta(chunk.lines[1]) + metatype !== nothing && metatype != chunktype && error("specifying a different cell type is not supported") + popfirst!(chunk.lines) + fold = get(metadata, "fold", fold) + end + if isa(chunk, MDChunk) if length(chunk.lines) == 1 line = escape_string(chunk.lines[1].second, '"') @@ -792,13 +806,14 @@ function pluto_notebook(chunks, config) end uuid = uuid4(content, i) push!(uuids, uuid) + push!(folds, fold) print(ionb, "# ╔═╡ ", uuid, '\n') write(ionb, content, '\n') end # Print cell order print(ionb, "# ╔═╡ Cell order:\n") - foreach(x -> print(ionb, "# ╠═", x, '\n'), uuids) + foreach(((x, f),) -> print(ionb, "# $(f ? "╟─" : "╠═")", x, '\n'), zip(uuids, folds)) # custom post-processing from user nb = config["postprocess"](String(take!(ionb))) From 2ec34eeaeb91af48b02851b7b12b6c02efd11cfb Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Mon, 18 Oct 2021 16:10:55 +0200 Subject: [PATCH 3/6] Fixes --- src/Literate.jl | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index 352345f..ae50f8e 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -16,6 +16,8 @@ struct DefaultFlavor <: AbstractFlavor end struct DocumenterFlavor <: AbstractFlavor end struct CommonMarkFlavor <: AbstractFlavor end struct FranklinFlavor <: AbstractFlavor end +struct JupyterFlavor <: AbstractFlavor end +struct PlutoFlavor <: AbstractFlavor end # # Some simple rules: # @@ -251,7 +253,7 @@ function create_configuration(inputfile; user_config, user_kwargs, type=nothing) cfg["name"] = filename(inputfile) cfg["preprocess"] = identity cfg["postprocess"] = identity - cfg["flavor"] = type === (:md) ? DocumenterFlavor() : DefaultFlavor() + cfg["flavor"] = type === (:md) ? DocumenterFlavor() : type === (:nb) ? JupyterFlavor() : DefaultFlavor() cfg["credit"] = true cfg["mdstrings"] = false cfg["keep_comments"] = false @@ -354,8 +356,10 @@ Available options: - `codefence` (default: `````"````@example \$(name)" => "````"````` for `DocumenterFlavor()` and `````"````julia" => "````"````` otherwise): Pair containing opening and closing code fence for wrapping code blocks. -- `flavor` (default: `Literate.DocumenterFlavor()`) Output flavor for markdown, see - [Markdown flavors](@ref). Only applicable for `Literate.markdown`. +- `flavor` (default: `Literate.DocumenterFlavor()` for `Literate.markdown` and + `Literate.JupyterFlavor()` for `Literate.notebook`) Output flavor for markdown and + notebook output, see [Markdown flavors](@ref) and [Notebook flavors](@ref). + Not used for `Literate.script`. - `devurl` (default: `"dev"`): URL for "in-development" docs, see [Documenter docs] (https://juliadocs.github.io/Documenter.jl/). Unused if `repo_root_url`/ `nbviewer_root_url`/`binder_root_url` are set. @@ -373,7 +377,7 @@ Available options: """ const DEFAULT_CONFIGURATION=nothing # Dummy const for documentation -function preprocessor(inputfile, outputdir; user_config, user_kwargs, type, flavor) +function preprocessor(inputfile, outputdir; user_config, user_kwargs, type) # Create configuration by merging default and userdefined config = create_configuration(inputfile; user_config=user_config, user_kwargs=user_kwargs, type=type) @@ -393,7 +397,9 @@ function preprocessor(inputfile, outputdir; user_config, user_kwargs, type, flav # Add some information for passing around Literate methods config["literate_inputfile"] = inputfile config["literate_outputdir"] = outputdir - config["literate_ext"] = type === (:nb) ? (flavor === :pluto ? ".jl" : ".ipynb") : ".$(type)" + config["literate_ext"] = type === (:nb) ? ( + config["flavor"]::AbstractFlavor isa JupyterFlavor ? ".ipynb" : ".jl") : + ".$(type)" # read content content = read(inputfile, String) @@ -604,21 +610,21 @@ Generate a notebook from `inputfile` and write the result to `outputdir`. See the manual section on [Configuration](@ref) for documentation of possible configuration with `config` and other keyword arguments. """ -function notebook(inputfile, outputdir=pwd(); config::Dict=Dict(), flavor=:jupyter, kwargs...) +function notebook(inputfile, outputdir=pwd(); config::Dict=Dict(), flavor=JupyterFlavor(), kwargs...) # preprocessing and parsing chunks, config = - preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:nb, flavor=flavor) + preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:nb) # create the notebook - nb = flavor == :jupyter ? jupyter_notebook(chunks, config) : pluto_notebook(chunks, config) + nb = create_notebook(config["flavor"]::AbstractFlavor, chunks, config) # write to file - print = flavor == :jupyter ? (io, c)->JSON.print(io, c, 1) : Base.print + print = config["flavor"]::AbstractFlavor isa JupyterFlavor ? (io, c) -> JSON.print(io, c, 1) : Base.print outputfile = write_result(nb, config; print = print) return outputfile end -function jupyter_notebook(chunks, config) +function create_notebook(::JupyterFlavor, chunks, config) nb = Dict() nb["nbformat"] = JUPYTER_VERSION.major nb["nbformat_minor"] = JUPYTER_VERSION.minor @@ -743,7 +749,7 @@ function execute_notebook(nb; inputfile::String="") return nb end -function pluto_notebook(chunks, config) +function create_notebook(::PlutoFlavor, chunks, config) ionb = IOBuffer() # Print header write(ionb, """ From 874b758e82c5f4fc0ad45d71b94d1d4f5d197952 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Tue, 19 Oct 2021 01:43:35 +0200 Subject: [PATCH 4/6] fixes --- src/Literate.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Literate.jl b/src/Literate.jl index ae50f8e..aa728fc 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -610,7 +610,7 @@ Generate a notebook from `inputfile` and write the result to `outputdir`. See the manual section on [Configuration](@ref) for documentation of possible configuration with `config` and other keyword arguments. """ -function notebook(inputfile, outputdir=pwd(); config::Dict=Dict(), flavor=JupyterFlavor(), kwargs...) +function notebook(inputfile, outputdir=pwd(); config::Dict=Dict(), kwargs...) # preprocessing and parsing chunks, config = preprocessor(inputfile, outputdir; user_config=config, user_kwargs=kwargs, type=:nb) @@ -754,9 +754,10 @@ function create_notebook(::PlutoFlavor, chunks, config) # Print header write(ionb, """ ### A Pluto.jl notebook ### - # v0.11.0 + # v0.16.0 using Markdown + using InteractiveUtils """) From 442eea1786fc3581b848d494c237722bc89964d8 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Mon, 1 Nov 2021 10:28:08 +0100 Subject: [PATCH 5/6] Readme --- README.md | 13 +++++++------ examples/README.jl | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b90c109..7c7536e 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,13 @@ Literate is a package for [Literate Programming](https://en.wikipedia.org/wiki/L The main purpose is to facilitate writing Julia examples/tutorials that can be included in your package documentation. -Literate can generate markdown pages -(for e.g. [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl)), and -[Jupyter notebooks](http://jupyter.org/), from the same source file. There is also -an option to "clean" the source from all metadata, and produce a pure Julia script. -Using a single source file for multiple purposes reduces maintenance, and makes sure -your different output formats are synced with each other. +Literate can generate multiple outputs from a single source file: Markdown pages +(for e.g. [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl)), +[Jupyter notebooks](http://jupyter.org/), and +[Pluto notebooks](https://github.com/fonsp/Pluto.jl). +There is also an option to "clean" the source from all metadata, and produce a +pure Julia script. Using a single source file for multiple purposes reduces maintenance, +and makes sure your different output formats are synced with each other. This README was generated directly from [this source file](https://github.com/fredrikekre/Literate.jl/blob/master/examples/README.jl) diff --git a/examples/README.jl b/examples/README.jl index d0966c8..384da0d 100644 --- a/examples/README.jl +++ b/examples/README.jl @@ -8,12 +8,13 @@ # The main purpose is to facilitate writing Julia examples/tutorials that can be included in # your package documentation. -# Literate can generate markdown pages -# (for e.g. [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl)), and -# [Jupyter notebooks](http://jupyter.org/), from the same source file. There is also -# an option to "clean" the source from all metadata, and produce a pure Julia script. -# Using a single source file for multiple purposes reduces maintenance, and makes sure -# your different output formats are synced with each other. +# Literate can generate multiple outputs from a single source file: Markdown pages +# (for e.g. [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl)), +# [Jupyter notebooks](http://jupyter.org/), and +# [Pluto notebooks](https://github.com/fonsp/Pluto.jl). +# There is also an option to "clean" the source from all metadata, and produce a +# pure Julia script. Using a single source file for multiple purposes reduces maintenance, +# and makes sure your different output formats are synced with each other. # # This README was generated directly from # [this source file](https://github.com/fredrikekre/Literate.jl/blob/master/examples/README.jl) From 54bc6119c6636b6caef0323e7ff256860675337a Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Mon, 1 Nov 2021 13:32:13 +0100 Subject: [PATCH 6/6] docs --- docs/src/outputformats.md | 17 +++++++++++++---- src/Literate.jl | 5 +++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/src/outputformats.md b/docs/src/outputformats.md index 8716620..8021807 100644 --- a/docs/src/outputformats.md +++ b/docs/src/outputformats.md @@ -80,16 +80,17 @@ Literate.markdown Literate can output markdown in different flavors. The flavor is specified using the `flavor` keyword argument. The following flavors are currently supported: - - `flavor = Literate.DocumenterFlavor()` this is the default flavor and the output is + - `flavor = Literate.DocumenterFlavor()`: this is the default flavor and the output is meant to be used as input to [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl). - - `flavor = Literate.FranklinFlavor()` this outputs markdown meant to be used as input + - `flavor = Literate.FranklinFlavor()`: this outputs markdown meant to be used as input to [Franklin.jl](https://franklinjl.org/). ## [**4.2.** Notebook output](@id Notebook-output) -Notebook output is generated by [`Literate.notebook`](@ref). The (default) notebook output -of the source snippet can be seen here: [notebook.ipynb](generated/notebook.ipynb). +Notebook output is generated by [`Literate.notebook`](@ref). The Jupyter notebook output +of the source snippet can be seen here: [notebook.ipynb](generated/notebook.ipynb), +and the Pluto notebook output can be seen here: [notebook.jl](generated/notebook.jl). We note that lines starting with `# ` are placed in markdown cells, and the code lines have been placed in code cells. By default the notebook @@ -103,6 +104,14 @@ output of [`Literate.notebook`](@ref). Literate.notebook ``` +### Notebook flavors + +Literate can output both Jupyter notebooks and Pluto notebooks. The flavor is specified +using the `flavor` keyword argument: + + - `flavor = Literate.JupyterFlavor()` (default) + - `flavor = Literate.PlutoFlavor()` + ### Notebook metadata Jupyter notebook cells (both code cells and markdown cells) can contain metadata. This is enabled diff --git a/src/Literate.jl b/src/Literate.jl index aa728fc..9327a95 100644 --- a/src/Literate.jl +++ b/src/Literate.jl @@ -350,9 +350,10 @@ Available options: `This file was generated with Literate.jl ...` to the bottom of the page. If you find Literate.jl useful then feel free to keep this. - `keep_comments` (default: `false`): When `true`, keeps markdown lines as comments in the - output script. Only applicable for `Literate.script`. + output script. Only applicable for [`Literate.script`](@ref) - `execute` (default: `true` for notebook, `false` for markdown): Whether to execute and - capture the output. Only applicable for `Literate.notebook` and `Literate.markdown`. + capture the output. Only applicable for [`Literate.notebook`](@ref) and + [`Literate.markdown`](@ref). - `codefence` (default: `````"````@example \$(name)" => "````"````` for `DocumenterFlavor()` and `````"````julia" => "````"````` otherwise): Pair containing opening and closing code fence for wrapping code blocks.