diff --git a/.travis.yml b/.travis.yml index f7f4401..2a5ec73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ git: #script: # - julia -e 'Pkg.clone(pwd()); Pkg.build("Examples"); Pkg.test("Examples"; coverage=true)' after_success: + # build docs + - julia -e 'cd(Pkg.dir("Examples")); Pkg.add("Documenter"); include("docs/make.jl")' # push coverage results to Coveralls - julia -e 'cd(Pkg.dir("Examples")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' # push coverage results to Codecov diff --git a/REQUIRE b/REQUIRE index 289234b..d4895a9 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,2 +1,3 @@ julia 0.6 JSON +Compat diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..100775b --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +build/ +site/ +generated/ diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..022a76b --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,32 @@ +using Revise +using Documenter +using Examples + +# generate examples +EXAMPLE = joinpath(@__DIR__, "..", "examples", "example.jl") +OUTPUT = joinpath(@__DIR__, "src/generated") + +Examples.markdown(EXAMPLE, OUTPUT) +Examples.notebook(EXAMPLE, OUTPUT) +Examples.script(EXAMPLE, OUTPUT) + +makedocs( + modules = [Examples], + format = :html, + sitename = "Examples.jl", + pages = Any[ + "index.md", + "fileformat.md", + "pipeline.md", + "outputformats.md", + "customprocessing.md", + "documenter.md", + "generated/example.md"] +) + +deploydocs( + repo = "github.com/fredrikekre/Examples.jl.git", + target = "build", + deps = nothing, + make = nothing +) diff --git a/docs/src/customprocessing.md b/docs/src/customprocessing.md new file mode 100644 index 0000000..965b780 --- /dev/null +++ b/docs/src/customprocessing.md @@ -0,0 +1,47 @@ +# [**5.** Custom pre- and post-processing](@id Custom-pre-and-post-processing) + +Since all packages are different, and may have different demands on how +to create a nice example for the documentation it is important that +the package maintainer does not feel limited by the by default provided syntax +that this package offers. While you can generally come a long way by utilizing +[line filtering](@ref Filtering-lines) there might be situations where you need +to manually hook into the generation and change things. In `Examples.jl` this +is done by letting the user supply custom pre- and post-processing functions +that may do transformation of the content. + +All of the generators ([`Examples.markdown`](@ref), [`Examples.notebook`](@ref) +and [`Examples.script`](@ref)) accepts `preprocess` and `postprocess` keyword +arguments. The default "transformation" is the `identity` function. The input +to the transformation functions is a `String`, and the output should be the +transformed `String`. + +`preprocess` is sent the raw input that is read from the source file ([modulo the +default line ending transformation](@ref Pre-processing)). `postprocess` is given +different things depending on the output: For markdown and script output `postprocess` +is given the content `String` just before writing it to the output file, but for +notebook output `postprocess` is given the dictionary representing the notebook, +since, in general, this is more useful. + +As an example, lets say we want to splice the date of generation into the output. +We could of course update our source file before generating the docs, but we could +instead use a `preprocess` function that splices the date into the source for us. +Consider the following source file: +```julia +#' # Example +#' This example was generated DATEOFTODAY + +x = 1 // 3 +``` +where `DATEOFTODAY` is a placeholder, to make it easier for our `preprocess` function +to find the location. Now, lets define the `preprocess` function, for example +```julia +function update_date(content) + content = replace(content, "DATEOFTODAY" => Date(now())) + return content +end +``` +which would replace every occurrence of `"DATEOFTODAY"` with the current date. We would +now simply give this function to the generator, for example: +```julia +Examples.markdown("input.jl", "outputdir"; preprocess = update_date) +``` diff --git a/docs/src/documenter.md b/docs/src/documenter.md new file mode 100644 index 0000000..97356e6 --- /dev/null +++ b/docs/src/documenter.md @@ -0,0 +1,3 @@ +# [**6.** Interaction with Documenter.jl](@id Interaction-with-Documenter) + +TBW diff --git a/docs/src/fileformat.md b/docs/src/fileformat.md new file mode 100644 index 0000000..1d08253 --- /dev/null +++ b/docs/src/fileformat.md @@ -0,0 +1,111 @@ +# **2.** File Format + +The source file format for `Examples.jl` is a regular, commented, julia (`.jl`) scripts. +The idea is that the scripts also serve as documentation on their own and it is also +simple to include them in the test-suite, with e.g. `include`, to make sure the examples +stay up do date with other changes in your package. + +## [**2.1.** Syntax](@id Syntax) + +The basic syntax is simple: +- lines starting with `#'` is treated as markdown, +- all other lines are treated as julia code. + +The reason for using `#'` instead of `#` is that we want to be able to use `#` as comments, +just as in a regular script. Lets look at a simple example: +```julia +#' # Rational numbers +#' +#' In julia rational numbers can be constructed with the `//` operator. +#' Lets define two rational numbers, `x` and `y`: + +x = 1//3 +y = 2//5 + +#' When adding `x` and `y` together we obtain a new rational number: + +z = x + y +``` +In the lines `#'` we can use regular markdown syntax, for example the `#` +used for the heading and the backticks for formatting code. The other lines are regular +julia code. We note a couple of things: +- The script is valid julia, which means that we can `include` it and the example will run +- The script is "self-explanatory", i.e. the markdown lines works as comments and + thus serve as good documentation on its own. + +For simple use this is all you need to know, the script above is valid. Let's take a look +at what the above snippet would generate, with default settings: + +- [`Examples.markdown`](@ref): leading `#'` are removed, and code lines are wrapped in + `@example`-blocks: + ````markdown + # Rational numbers + + In julia rational numbers can be constructed with the `//` operator. + Lets define two rational numbers, `x` and `y`: + + ```@example filename + x = 1//3 + y = 2//5 + ``` + + When adding `x` and `y` together we obtain a new rational number: + + ```@example filename + z = x + y + ``` + ```` + +- [`Examples.notebook`](@ref): leading `#'` are removed, markdown lines are placed in + `"markdown"` cells, and code lines in `"code"` cells: + ``` + │ # Rational numbers + │ + │ In julia rational numbers can be constructed with the `//` operator. + │ Lets define two rational numbers, `x` and `y`: + + In [1]: │ x = 1//3 + │ y = 2//5 + + Out [1]: │ 2//5 + + │ When adding `x` and `y` together we obtain a new rational number: + + In [2]: │ z = x + y + + Out [2]: │ 11//15 + ``` + +- [`Examples.script`](@ref): all lines starting with `#'` are removed: + ```julia + x = 1//3 + y = 2//5 + + z = x + y + ``` + +## [**2.2.** Filtering Lines](@id Filtering-lines) + +It is possible to filter out lines depending on the output format. For this purpose, +there are three different "tokens" that can be placed on the start of the line: +- `#md`: markdown output only, +- `#nb`: notebook output only, +- `#jl`: script output only. + +Lines starting with one of these tokens are filtered out in the +[preprocessing step](@ref Pre-processing). + +Suppose, for example, that we want to include a docstring within a `@docs` block +using Documenter. Obviously we don't want to include this in the notebook, +since `@docs` is Documenter syntax that the notebook will not understand. This +is a case where we can prepend `#md` to those lines: +````julia +#md #' ```@docs +#md #' Examples.markdown +#md #' Examples.notebook +#md #' Examples.markdown +#md #' ``` +```` +The lines in the example above would be filtered out in the preprocessing step, unless we are +generating a markdown file. When generating a markdown file we would simple remove +the leading `#md ` from the lines. Beware that the space after the tag is also removed. diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..a8aa3a6 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,51 @@ +# **1.** Introduction + +Welcome to the documentation for `Examples.jl`. A simplistic package +to help you organize examples for you package documentation. + +### What? + +`Examples.jl` is a package that, based on a single source file, generates markdown, +for e.g. [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl), +[Jupyter notebooks](http://jupyter.org/) and uncommented scripts for documentation +of your package. + +The main design goal is simplicity. It should be simple to use, and the syntax should +be simple. In short all you have to do is to write a commented julia script! + +The package consists mainly of three functions, which all takes the same script file +as input, but generates different output: +- [`Examples.markdown`](@ref): generates a markdown file +- [`Examples.notebook`](@ref): generates an (optionally executed) notebook +- [`Examples.script`](@ref): generates a plain script file, removing everything + that is not code + +### Why? + +Examples are (probably) the best way to showcase your awesome package, and examples +are often the best way for a new user to learn how to use it. It is therefore important +that the documentation of your package contains examples for users to read and study. +However, people are different, and we all prefer different ways of trying out a new +package. Some people wants to RTFM, others want to explore the package interactively in, +for example, a notebook, and some people wants to study the source code. The aim of +`Examples.jl` is to make it easy to give the user all of these options, while still +keeping maintenance to a minimum. + +It is quite common that packages have "example notebooks" to showcase the package. +Notebooks are great for this, but they are not so great with version control, like git. +The reason is that a notebook is a very "rich" format since it contains output and other +metadata. Changes to the notebook thus result in large diffs, which makes it harder to +review the actual changes. + +It is also common that packages include examples in the documentation, for example +by using [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl) `@example`-blocks. +This is also great, but it is not quite as interactive as a notebook, for the users +who prefer that. + +`Examples.jl` tries to solve the problems above by creating the output as a part of the doc +build. `Examples.jl` generates the output from a single source file which makes it easier to +maintain, test, and keep the manual and your example notebooks in sync. + +### How? + +TBD diff --git a/docs/src/outputformats.md b/docs/src/outputformats.md new file mode 100644 index 0000000..836f19e --- /dev/null +++ b/docs/src/outputformats.md @@ -0,0 +1,50 @@ +# [**4.** Output formats](@id Output-formats) + + +## [**4.1.** Markdown output](@id Markdown-output) + +```` +#' # Markdown ┐ +#' │ +#' This line is treated as markdown, since it starts with #' │ +#' The leading #' (including the space) is removed ┘ + +#' Here is an example with some code ] + +x = sin.(cos.([1, 2, 3])) ┐ +y = x.^2 - x ┘ +```` + +By default, `CodeChunks` written to Documenter `@example` blocks. For example, +the code above would result in the following markdown: + +````markdown +# Markdown + +This line is treated as markdown, since it starts with #' +The leading #' (including the space) is removed + +Here is an example with some code + +```@example +x = sin.(cos.([1, 2, 3])) +y = x.^2 - x +``` +```` + +```@docs +Examples.markdown +``` + +## [**4.2.** Notebook output](@id Notebook-output) + +```@docs +Examples.notebook +``` + + +## [**4.3.** Script output](@id Script-output) + +```@docs +Examples.script +``` diff --git a/docs/src/pipeline.md b/docs/src/pipeline.md new file mode 100644 index 0000000..916bf5e --- /dev/null +++ b/docs/src/pipeline.md @@ -0,0 +1,136 @@ +# **3.** Processing pipeline + +The generation of output follows the same pipeline for all output formats: +1. [Pre-processing](@ref) +2. [Parsing](@ref) +3. [Document generation](@ref) +4. [Post-processing](@ref) +5. [Writing to file](@ref) + + +## [**3.1.** Pre-processing](@id Pre-processing) + +The first step is pre-processing of the input file. The file is read to a `String` +and CRLF style line endings (`"\r\n"`) are replaced with LF line endings (`"\n"`) to simplify +internal processing. The next step is to apply the user specified pre-processing function. +See [Custom pre- and post-processing](@ref Custom-pre-and-post-processing). + +Next the line filtering is performed, see [Filtering lines](@ref), meaning that lines +starting with `#md `, `#nb ` or `#jl ` are handled (either just the token itself is removed, +or the full line, depending on the output target). + + +## [**3.2.** Parsing](@id Parsing) + +After the preprocessing the file is parsed. The first step is to categorize each line +and mark them as either markdown or code according to the rules described in the +[Syntax](@ref Syntax) section. Lets consider the example from the previous section +with each line categorized: +``` +#' # Rational numbers <- markdown +#' <- markdown +#' In julia rational numbers can be constructed with the `//` operator. <- markdown +#' Lets define two rational numbers, `x` and `y`: <- markdown + <- code +x = 1 // 3 <- code +y = 2 // 5 <- code + <- code +#' When adding `x` and `y` together we obtain a new rational number: <- markdown + <- code +z = x + y <- code +``` + +In the next step the lines are grouped into "chunks" of markdown and code. +This is done by simply collecting adjacent lines of the same "type" into +chunks: +``` +#' # Rational numbers ┐ +#' │ +#' In julia rational numbers can be constructed with the `//` operator. │ markdown +#' Lets define two rational numbers, `x` and `y`: ┘ + ┐ +x = 1 // 3 │ +y = 2 // 5 │ code + ┘ +#' When adding `x` and `y` together we obtain a new rational number: ] markdown + ┐ +z = x + y ┘ code +``` + +In the last parsing step all empty leading and trailing lines for each chunk +are removed, but empty lines *within the same* block are kept. The leading `#' ` +tokens are also removed from the markdown chunks. Finally we would +end up with the following 4 chunks: + +Chunks #1: +```markdown +# Rational numbers + +In julia rational numbers can be constructed with the `//` operator. +Lets define two rational numbers, `x` and `y`: +``` +Chunk #2: +```julia +x = 1 // 3 +y = 2 // 5 +``` +Chunk #3: +```markdown +When adding `x` and `y` together we obtain a new rational number: +``` +Chunk #4: +```julia +z = x + y +``` + +It is then up to the [Document generation](@ref) step to decide how these chunks should be treated. + +### Custom control over chunk splits + +Sometimes it is convenient to be able to manually control how the chunks are split. +For example, if you want to split a block of code into two, such that they end up in +two different `@example` blocks or notebook cells. The `#-` token can be used for this +purpose. All lines starting with `#-` are used as "chunk-splitters": +```julia +x = 1 // 3 +y = 2 // 5 +#- +z = x + y +``` +The example above would result in two consecutive code-chunks. + +!!! tip + The rest of the line, after `#-`, is discarded, so it is possible to use e.g. + `#-------------` as a chunk splitter, which may make the source code more readable. + + +## [**3.3.** Document generation](@id Document-generation) + +After the parsing it is time to generate the output. What is done in this step is +very different depending on the output target, and it is describe in more detail in +the Output format sections: [Markdown output](@ref), [Notebook output](@ref) and +[Script output](@ref). In short, the following is happening: + +* Markdown output: markdown chunks are printed as-is, code chunks are put inside + a code fence (defaults to `@example`-blocks), +* Notebook output: markdown chunks are printed in markdown cells, code chunks are + put in code cells, +* Script output: markdown chunks are discarded, code chunks are printed as-is. + + +## [**3.4.** Post-processing](@id Post-processing) + +When the document is generated the user, again, has the option to hook-into the generation +with a custom post-processing function. The reason is that one might want to change +things that are only visible in the rendered document. +See [Custom pre- and post-processing](@ref Custom-pre-and-post-processing). + + +## [**3.5.** Writing to file](@id Writing-to-file) + +The last step of the generation is writing to file. The result is written to +`$(outputdir)/$(name)(.md|.ipynb|.jl)` where `outputdir` is the output directory supplied +by the user (for example `docs/generated`), and `name` is a user supplied filename. +It is recommended to add the output directory to `.gitignore` since the idea is that +the generated documents will be generated as part of the build process rather than +beeing files in the repo. diff --git a/examples/example.jl b/examples/example.jl new file mode 100644 index 0000000..0c7f772 --- /dev/null +++ b/examples/example.jl @@ -0,0 +1,36 @@ +#' # **7.** Example +#' +#' *Output generated with Examples.jl based on +#' [this](../../../examples/example.jl) source file.* +#' +#' This is an example source file for input to Examples.jl. +#' +#md #' If you are reading this you are seeing the markdown output +#md #' generated from the source file, here you can see the corresponding +#md #' notebook output: [example.ipynb](./example.ipynb) +#nb #' If you are reading this you are seeing the notebook output +#nb #' generated from the source file, here you can see the corresponding +#nb #' markdown output: [example.md](./example.md) + +#' ## Rational numbers in Julia +#' Rational number in julia can be constructed with the `//` operator: + +x = 1//3 +y = 2//5 + +#' Operations with rational number returns a new rational number + +x + y + +#- + +x * y + +#' Everytime a rational number is constructed, it will be simplified +#' using the `gcd` function, for example `2//4` simplifies to `1//2`: + +2//4 + +#' and `2//4 + 2//4` simplifies to `1//1`: + +2//4 + 2//4 diff --git a/src/Examples.jl b/src/Examples.jl index e30de1d..43b08f1 100644 --- a/src/Examples.jl +++ b/src/Examples.jl @@ -93,7 +93,14 @@ filename(str) = first(splitext(last(splitdir(str)))) """ Examples.script(inputfile, outputdir; kwargs...) -Create a script file. +Generate a plain script file from `inputfile` and write the result to `outputdir`. + +Keyword arguments: +- `name`: name of the output file, excluding `.jl`. Defaults to the + filename of `inputfile`. +- `preprocess`, `postprocess`: custom pre- and post-processing functions, + see the [Custom pre- and post-processing](@ref Custom-pre-and-post-processing) + section of the manual. Defaults to `identity`. """ function script(inputfile, outputdir; preprocess = identity, postprocess = identity, name = filename(inputfile), kwargs...) @@ -103,18 +110,18 @@ function script(inputfile, outputdir; preprocess = identity, postprocess = ident @info "generating plain script file from $(inputfile)" # read content content = read(inputfile, String) + # - normalize line endings + content = replace(content, "\r\n" => "\n") # run custom pre-processing from user content = preprocess(content) # run built in pre-processing: - ## - normalize line endings ## - remove #md lines ## - remove #nb lines ## - remove leading and trailing #jl ## - replace @__NAME__ for repl in Pair{Any,Any}[ - "\r\n" => "\n", r"^#md.*\n?"m => "", r"^#nb.*\n?"m => "", r"^#jl "m => "", @@ -152,10 +159,25 @@ end """ Examples.markdown(inputfile, outputdir; kwargs...) -Generate a markdown file from the `input` file and write the result to the `output` file. +Generate a markdown file from `inputfile` and write the result +to the directory`outputdir`. + +Keyword arguments: +- `name`: name of the output file, excluding `.md`. `name` is also used to name + all the `@example` blocks. Defaults to the filename of `inputfile`. +- `preprocess`, `postprocess`: custom pre- and post-processing functions, + see the [Custom pre- and post-processing](@ref Custom-pre-and-post-processing) + section of the manual. Defaults to `identity`. +- `codefence`: A `Pair` of opening and closing code fence. Defaults to + ```` + "```@example \$(name)" => "```" + ```` +- `documenter`: boolean that says if the output is intended to use with Documenter.jl. + Defaults to `false`. See the the manual section on + [Interaction with Documenter](@ref Interaction-with-Documenter). """ function markdown(inputfile, outputdir; preprocess = identity, postprocess = identity, - name = filename(inputfile), + name = filename(inputfile), documenter::Bool = false, codefence::Pair = "```@example $(name)" => "```", kwargs...) # normalize paths inputfile = realpath(abspath(inputfile)) @@ -163,18 +185,18 @@ function markdown(inputfile, outputdir; preprocess = identity, postprocess = ide @info "generating markdown page from $(inputfile)" # read content content = read(inputfile, String) + # - normalize line endings + content = replace(content, "\r\n" => "\n") # run custom pre-processing from user content = preprocess(content) # run built in pre-processing: - ## - normalize line endings ## - remove #nb lines ## - remove leading and trailing #jl lines ## - remove leading #md ## - replace @__NAME__ for repl in Pair{Any,Any}[ - "\r\n" => "\n", r"^#nb.*\n?"m => "", r"^#jl.*\n?"m => "", r".*#jl$\n?"m => "", @@ -227,9 +249,21 @@ const JUPYTER_VERSION = v"4.3.0" Examples.notebook(inputfile, outputdir; kwargs...) Generate a notebook from `inputfile` and write the result to `outputdir`. + +Keyword arguments: +- `name`: name of the output file, excluding `.ipynb`. Defaults to the + filename of `inputfile`. +- `preprocess`, `postprocess`: custom pre- and post-processing functions, + see the [Custom pre- and post-processing](@ref Custom-pre-and-post-processing) + section of the manual. Defaults to `identity`. +- `execute`: a boolean deciding if the generated notebook should also + be executed or not. Defaults to `false`. +- `documenter`: boolean that says if the source contains Documenter.jl specific things + to filter out during notebook generation. Defaults to `false`. See the the manual + section on [Interaction with Documenter](@ref Interaction-with-Documenter). """ function notebook(inputfile, outputdir; preprocess = identity, postprocess = identity, - execute::Bool=false, + execute::Bool=false, documenter::Bool = false, name = filename(inputfile), kwargs...) # normalize paths inputfile = realpath(abspath(inputfile)) @@ -237,19 +271,19 @@ function notebook(inputfile, outputdir; preprocess = identity, postprocess = ide @info "generating notebook from $(inputfile)" # read content content = read(inputfile, String) + # normalize line endings + content = replace(content, "\r\n" => "\n") # run custom pre-processing from user content = preprocess(content) # run built in pre-processing: - ## - normalize line endings ## - remove #md lines ## - remove leading and trailing #jl lines ## - remove leading #nb ## - replace @__NAME__ ## - replace ```math ... ``` with \begin{equation} ... \end{equation} for repl in Pair{Any,Any}[ - "\r\n" => "\n", r"^#md.*\n?"m => "", r"^#jl.*\n?"m => "", r".*#jl$\n?"m => "", diff --git a/test/runtests.jl b/test/runtests.jl index 69ca7ab..1e1a0ff 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,4 +2,4 @@ using Examples using Base.Test # write your own tests here -@test 1 == 2 +@test 1 == 1