Support for REPL-simulated test cases

The implementation is based on the code from Documenter.jl.

Allows to define test cases with REPL-simulated syntax:

    julia> 1
    1

This is useful as allows to define new test cases by simply copy pasting
real REPL sessions.

Author: andreypopp

src/NarrativeTest.jl

@@ -125,6 +125,7 @@ struct TestCase <: AbstractTestCase
     code::TextBlock
     pre::Union{TextBlock,Nothing}
     expect::Union{TextBlock,Nothing}
+    repl::Bool
 end
 
 location(test::TestCase) = test.loc
@@ -537,29 +538,36 @@ function parsemd!(stack::Vector{TextBlock})
         line = pop!(stack)
         if isfence(line)
             # Extract a fenced block.
+            isrepl = false
             fenceloc = line.loc
             lang = strip(line.val[4:end])
             jlstack = TextBlock[]
             while !isempty(stack) && !isfence(stack[end])
-                push!(jlstack, pop!(stack))
+                block = pop!(stack)
+                isrepl = isrepl || startswith(block.val, "julia>")
+                push!(jlstack, block)
             end
             if isempty(stack)
                 push!(suite, BrokenTestCase(fenceloc, "incomplete fenced code block"))
             else
                 pop!(stack)
                 if isempty(lang)
                     reverse!(jlstack)
-                    append!(suite, parsejl!(jlstack))
+                    append!(suite, isrepl ? parsejlrepl!(jlstack) : parsejl!(jlstack))
                 end
             end
         elseif isindent(line) && !isblank(line)
             # Extract an indented block.
-            jlstack = TextBlock[unindent(line)]
+            block = unindent(line)
+            isrepl = startswith(block.val, "julia>")
+            jlstack = TextBlock[block]
             while !isempty(stack) && (isindent(stack[end]) || isblank(stack[end]))
-                push!(jlstack, unindent(pop!(stack)))
+                block = unindent(pop!(stack))
+                isrepl = isrepl || startswith(block.val, "julia> ")
+                push!(jlstack, block)
             end
             reverse!(jlstack)
-            append!(suite, parsejl!(jlstack))
+            append!(suite, isrepl ? parsejlrepl!(jlstack) : parsejl!(jlstack))
         elseif isadmonition(line)
             # Skip an indented admonition block.
             while !isempty(stack) && (isindent(stack[end]) || isblank(stack[end]))
@@ -588,6 +596,78 @@ function parsejl!(stack::Vector{TextBlock})
     return suite
 end
 
+const PROMPT_REGEX = r"^julia>(?: (.*))?$"
+const SOURCE_REGEX = r"^       (.*)$"
+
+function parsejlrepl!(stack::Vector{TextBlock})
+    reverse!(stack)
+    code = TextBlock[]
+    expect = TextBlock[]
+    buf = IOBuffer()
+    processing_prompt = false
+    function savecode!(loc)
+        data = consumebuf!(buf)
+        if data !== nothing
+            push!(code, TextBlock(loc, strip(data)))
+        else
+            push!(code, TextBlock(loc, ""))
+        end
+    end
+    function saveexpect!(loc)
+        data = consumebuf!(buf)
+        if data !== nothing
+            push!(expect, TextBlock(loc, rstrip(data)))
+        end
+    end
+    while true
+        line = popfirst!(stack)
+        prompt = match(PROMPT_REGEX, line.val)
+        if prompt === nothing
+            source = match(SOURCE_REGEX, line.val)
+            if source === nothing
+                processing_prompt = false
+                savecode!(line.loc)
+                println(buf, rstrip(line.val))
+                takeuntil!(PROMPT_REGEX, buf, stack)
+            else
+                processing_prompt = true
+                println(buf, source[1])
+            end
+        else
+            processing_prompt = true
+            saveexpect!(line.loc)
+            if prompt[1] !== nothing
+                println(buf, prompt[1])
+            end
+        end
+        if isempty(stack)
+            processing_prompt ? savecode!(line.loc) : saveexpect!(line.loc)
+            for _ in length(code):length(expect) + 1
+              push!(expect, TextBlock(line.loc, ""))
+            end
+            break
+        end
+    end
+    [isempty(c.val) ?
+     BrokenTestCase(c.loc, "empty test case") :
+     TestCase(c.loc, c, nothing, e, true) for (c, e) in zip(code, expect)]
+end
+
+function consumebuf!(buf)
+    n = bytesavailable(seekstart(buf))
+    n > 0 ? String(take!(buf)) : nothing
+end
+
+function takeuntil!(r, buf, lines::Vector{TextBlock})
+    while !isempty(lines)
+        if !occursin(r, lines[1].val)
+            println(buf, rstrip(popfirst!(lines).val))
+        else
+            break
+        end
+    end
+end
+
 # Extract a test case from Julia source.
 
 function parsecase!(stack::Vector{TextBlock})
@@ -638,7 +718,7 @@ function parsecase!(stack::Vector{TextBlock})
         end
     end
     !isempty(code) || return BrokenTestCase(loc, "missing test code")
-    return TestCase(loc, collapse(code), collapse(pre), collapse(expect))
+    return TestCase(loc, collapse(code), collapse(pre), collapse(expect), false)
 end
 
 # Run a single test case.
@@ -703,7 +783,11 @@ function runtest(test::TestCase; subs=common_subs(), mod=nothing)
                         body = asexpr(test.code)
                         ans = Core.eval(mod, body)
                         if ans !== nothing && !no_output
-                            Base.invokelatest(show, io, ans)
+                            if test.repl
+                              Base.invokelatest(show, io, "text/plain", ans)
+                            else
+                              Base.invokelatest(show, io, ans)
+                            end
                         end
                     end
                 catch exc
@@ -746,10 +830,11 @@ end
 runtest(test::BrokenTestCase; subs=common_subs(), mod=nothing) =
     Error(test)
 
-runtest(loc, code; pre=nothing, expect=nothing, subs=common_subs(), mod=nothing) =
+runtest(loc, code; pre=nothing, expect=nothing, subs=common_subs(), mod=nothing, repl=false) =
     runtest(TestCase(loc, TextBlock(loc, code),
                      pre !== nothing ? TextBlock(loc, pre) : nothing,
-                     expect !== nothing ? TextBlock(loc, expect) : nothing),
+                     expect !== nothing ? TextBlock(loc, expect) : nothing,
+                     repl),
             subs=subs, mod=mod)
 
 # Convert expected output block to a regex.

test/index.md

@@ -384,6 +384,178 @@ It is also an error if a multi-line output block is not closed.
         incomplete multiline comment block
     =#
 
+It is possible to define tests using syntax simulating REPL session (in fact
+REPL session can be copy pasted as-is to become test cases), just prefix a line
+in a code block with `julia> `:
+
+    suite = parsemd(
+        @__FILE__,
+        IOBuffer("""
+            These test cases are embedded in an indented code block.
+
+                julia> (3+4)*6
+                42
+
+                julia> 2+2
+                5
+
+            The following test cases are embedded in a fenced code block.
+            ```
+            julia> print(2^16)
+            65526
+
+            julia> sqrt(-1)
+            0.0 + 1.0im
+            ```
+            """))
+    foreach(display, suite)
+    #=>
+    Test case at …/index.md:4
+        (3+4)*6
+    Expected output:
+        42
+    Test case at …/index.md:7
+        2+2
+    Expected output:
+        5
+    Test case at …/index.md:12
+        print(2^16)
+    Expected output:
+        65526
+    Test case at …/index.md:15
+        sqrt(-1)
+    Expected output:
+        0.0 + 1.0im
+    =#
+
+Expected output can be missing:
+
+    suite = parsemd(
+        @__FILE__,
+        IOBuffer("""
+            The indented code block below is missing few expected outputs:
+
+                julia> 1
+
+                julia> 2
+                2
+
+                julia> 3
+
+            Now fenced code block with missing few expected outputs:
+
+            ```
+            julia> 1
+
+            julia> 2
+            2
+
+            julia> 3
+            ```
+            """))
+    foreach(display, suite)
+    #=>
+    Test case at …/index.md:4
+        1
+    Expected output:
+
+    Test case at …/index.md:6
+        2
+    Expected output:
+        2
+    Test case at …/index.md:9
+        3
+    Expected output:
+
+    Test case at …/index.md:14
+        1
+    Expected output:
+
+    Test case at …/index.md:16
+        2
+    Expected output:
+        2
+    Test case at …/index.md:18
+        3
+    Expected output:
+
+    =#
+
+REPL-like test cases can span multiple lines as in real REPL if we keep
+indentation:
+
+    suite = parsemd(
+        @__FILE__,
+        IOBuffer("""
+            Multiline code blocks:
+
+                julia> [1 2;
+                        3 4]
+                2×2 Matrix{Int64}:
+                 1  2
+                 3  4
+
+                julia> 1
+                       2
+                2
+
+                julia> "no
+                        output?"
+
+            """))
+    foreach(display, suite)
+    #=>
+    Test case at …/index.md:5
+        [1 2;
+         3 4]
+    Expected output:
+        2×2 Matrix{Int64}:
+         1  2
+         3  4
+    Test case at …/index.md:11
+        1
+        2
+    Expected output:
+        2
+    Test case at …/index.md:15
+        "no
+         output?"
+    Expected output:
+    =#
+
+Empty prompt with REPL-like test cases is considered as invalid:
+
+    suite = parsemd(
+        @__FILE__,
+        IOBuffer("""
+            Some test cases are invalid below as they are missing code
+            to execute:
+
+                julia> 1
+
+                julia>
+
+                julia> 3
+                3
+
+                julia>
+
+            """))
+    foreach(display, suite)
+    #=>
+    Test case at …/index.md:5
+        1
+    Expected output:
+
+    Error at …/index.md:7
+        empty test case
+    Test case at …/index.md:9
+        3
+    Expected output:
+        3
+    Error at …/index.md:12
+        empty test case
+    =#
 
 ## Running one test