add variable interpolation

docs/source/usage.md

@@ -67,23 +67,64 @@ You can then use, e.g.,
 
 ### IPython magic
 
-In IPython (and therefore in Jupyter), you can directly execute Julia
-code using `%%julia` magic:
+In IPython (and therefore in Jupyter), you can directly execute Julia code using `%julia` magic:
 
-```
+```python
 In [1]: %load_ext julia.magic
 Initializing Julia interpreter. This may take some time...
 
-In [2]: %%julia
-   ...: Base.banner(IOContext(stdout, :color=>true))
-               _
-   _       _ _(_)_     |  Documentation: https://docs.julialang.org
-  (_)     | (_) (_)    |
-   _ _   _| |_  __ _   |  Type "?" for help, "]?" for Pkg help.
-  | | | | | | |/ _` |  |
-  | | |_| | | | (_| |  |  Version 1.0.1 (2018-09-29)
- _/ |\__'_|_|_|\__'_|  |  Official https://julialang.org/ release
-|__/                   |
+In [2]: %julia [1 2; 3 4] .+ 1 
+Out[2]: 
+array([[2, 3],
+       [4, 5]], dtype=int64)
+```
+
+You can call Python code from inside of `%julia` blocks via `$var` for accessing single variables or `py"..."` for more complex expressions:
+
+```julia
+In [3]: arr = [1, 2, 3]
+
+In [4]: %julia $arr .+ 1
+Out[4]: 
+array([2, 3, 4], dtype=int64)
+
+In [5]: %julia sum(py"[x**2 for x in arr]")
+Out[5]: 14
+```
+
+Inside of strings and quote blocks, `$var` and `py"..."` don't call Python and instead retain their usual Julia behavior. To call Python code in these cases, you can "escape" one extra time:
+
+```julia
+In [6]: foo = "Python"
+        %julia foo = "Julia"
+        %julia ("this is $foo", "this is $($foo)")
+Out[6]: ('this is Julia', 'this is Python')
+```
+
+Expressions in macro arguments also always retain the Julia behavior:
+
+```julia
+In [7]: %julia @eval $foo
+Out[7]: 'Julia'
+```
+
+Results are automatically converted between equivalent Python/Julia types (should they exist). You can turn this off by appending `o` to the Python string:
+
+```python
+In [8]: %julia typeof(py"1"), typeof(py"1"o)
+Out[8]: (<PyCall.jlwrap Int64>, <PyCall.jlwrap PyObject>)
+```
+
+Code inside `%julia` blocks obeys the Python scope:
+
+```python
+In [9]: x = "global"
+   ...: def f():
+   ...:     x = "local"
+   ...:     ret = %julia py"x"
+   ...:     return ret
+   ...: f()
+Out[9]: 'local'
 ```
 
 #### IPython configuration

src/julia/magic.py

@@ -29,6 +29,12 @@
 from .core import Julia, JuliaError
 from .tools import redirect_output_streams
 
+try:
+    from IPython.core.magic import no_var_expand
+except ImportError:
+    def no_var_expand(f):
+        return f
+
 #-----------------------------------------------------------------------------
 # Main classes
 #-----------------------------------------------------------------------------
@@ -89,6 +95,7 @@ def __init__(self, shell):
         self._julia = Julia(init_julia=True)
         print()
 
+    @no_var_expand
     @line_cell_magic
     def julia(self, line, cell=None):
         """
@@ -97,14 +104,17 @@ def julia(self, line, cell=None):
         """
         src = compat.unicode_type(line if cell is None else cell)
 
-        try:
-            ans = self._julia.eval(src)
-        except JuliaError as e:
-            print(e, file=sys.stderr)
-            ans = None
-
-        return ans
-
+        # We assume the caller's frame is the first parent frame not in the
+        # IPython module. This seems to work with IPython back to ~v5, and
+        # is at least somewhat immune to future IPython internals changes,
+        # although by no means guaranteed to be perfect.
+        caller_frame = sys._getframe(3)
+        while caller_frame.f_globals.get('__name__').startswith("IPython"):
+            caller_frame = caller_frame.f_back
+
+        return self._julia.eval("""
+        _PyJuliaHelper.@prepare_for_pyjulia_call begin %s end
+        """%src)(self.shell.user_ns, caller_frame.f_locals)
 
 # Add to the global docstring the class information.
 __doc__ = __doc__.format(

src/julia/pyjulia_helper.jl

@@ -1,5 +1,9 @@
 module _PyJuliaHelper
 
+using PyCall
+using PyCall: pyeval_, Py_eval_input, Py_file_input
+using PyCall.MacroTools: isexpr, walk
+
 if VERSION < v"0.7-"
 nameof(m::Module) = ccall(:jl_module_name, Ref{Symbol}, (Any,), m)
 
@@ -44,6 +48,66 @@ if VERSION >= v"0.7-"
     end
 end
 
+
+# takes an expression like `$foo + 1` and turns it into a pyfunction
+# `(globals,locals) -> convert(PyAny, pyeval_("foo",globals,locals,PyAny)) + 1`
+# so that Python code can call it and just pass the appropriate globals/locals
+# dicts to perform the interpolation. 
+macro prepare_for_pyjulia_call(ex)
+
+    # f(x, quote_depth) should return a transformed expression x and whether to
+    # recurse into the new expression. quote_depth keeps track of how deep
+    # inside of nested quote objects we arepyeval
+    function stoppable_walk(f, x, quote_depth=1)
+        (fx, recurse) = f(x, quote_depth)
+        if isexpr(fx,:quote)
+            quote_depth += 1
+        end
+        if isexpr(fx,:$)
+            quote_depth -= 1
+        end
+        walk(fx, (recurse ? (x -> stoppable_walk(f,x,quote_depth)) : identity), identity)
+    end
+
+    function make_pyeval(globals, locals, expr::Union{String,Symbol}, options...)
+        code = string(expr)
+        T = length(options) == 1 && 'o' in options[1] ? PyObject : PyAny
+        input_type = '\n' in code ? Py_file_input : Py_eval_input
+        :($convert($T, $pyeval_($code, $globals, $locals, $input_type)))
+    end
+
+    function insert_pyevals(globals, locals, ex)
+        stoppable_walk(ex) do x, quote_depth
+            if quote_depth==1 && isexpr(x, :$)
+                if x.args[1] isa Symbol
+                    make_pyeval(globals, locals, x.args[1]), false
+                else
+                    error("""syntax error in: \$($(string(x.args[1])))
+                    Use py"..." instead of \$(...) for interpolating Python expressions.""")
+                end
+            elseif quote_depth==1 && isexpr(x, :macrocall)
+                if x.args[1]==Symbol("@py_str")
+                    # in Julia 0.7+, x.args[2] is a LineNumberNode, so filter it out
+                    # in a way that's compatible with Julia 0.6:
+                    make_pyeval(globals, locals, filter(s->(s isa String), x.args[2:end])...), false
+                else
+                    x, false
+                end
+            else
+                x, true
+            end
+        end
+    end
+
+    esc(quote
+        $pyfunction(
+            (globals, locals)->Base.eval(Main, $insert_pyevals(globals, locals, $(QuoteNode(ex)))),
+            $PyObject, $PyObject
+        )
+    end)
+end
+
+
 module IOPiper
 
 const orig_stdin  = Ref{IO}()

test/test_magic.py

@@ -1,15 +1,33 @@
-from IPython.testing.globalipapp import get_ipython
-from julia import magic
+from textwrap import dedent
+
 import pytest
+from IPython.testing import globalipapp
 
+from julia import magic
 
 @pytest.fixture
-def julia_magics(julia):
-    return magic.JuliaMagics(shell=get_ipython())
-
-
-def test_register_magics(julia):
-    magic.load_ipython_extension(get_ipython())
+def julia_magics():
+    return magic.JuliaMagics(shell=globalipapp.get_ipython())
+
+@pytest.fixture
+def run_cell(julia_magics):
+    # a more convenient way to run strings (possibly with magic) as if they were
+    # an IPython cell
+    def run_cell_impl(cell):
+        cell = dedent(cell).strip()
+        if cell.startswith("%%"):
+            return julia_magics.shell.run_cell_magic("julia","",cell.replace("%%julia","").strip())
+        else:
+            exec_result = julia_magics.shell.run_cell(cell)
+            if exec_result.error_in_exec:
+                raise exec_result.error_in_exec
+            else:
+                return exec_result.result
+    return run_cell_impl
+
+
+def test_register_magics():
+    magic.load_ipython_extension(globalipapp.get_ipython())
 
 
 def test_success_line(julia_magics):
@@ -23,14 +41,93 @@ def test_success_cell(julia_magics):
 
 
 def test_failure_line(julia_magics):
-    ans = julia_magics.julia('pop!([])')
-    assert ans is None
+    with pytest.raises(Exception):
+        julia_magics.julia('pop!([])')
 
 
 def test_failure_cell(julia_magics):
-    ans = julia_magics.julia(None, '1 += 1')
-    assert ans is None
-
+    with pytest.raises(Exception):
+        julia_magics.julia(None, '1 += 1')
+
+
+# In IPython, $x does a string interpolation handled by IPython itself for
+# *line* magic, which prior to IPython 7.3 could not be turned off. However,
+# even prior to IPython 7.3, *cell* magic never did the string interpolation, so
+# below, any time we need to test $x interpolation, do it as cell magic so it
+# works on IPython < 7.3
+
+def test_interp_var(run_cell):
+    run_cell("x=1")
+    assert run_cell("""
+    %%julia
+    $x
+    """) == 1
+
+def test_interp_expr(run_cell):
+    assert run_cell("""
+    x=1
+    %julia py"x+1"
+    """) == 2
+
+def test_bad_interp(run_cell):
+    with pytest.raises(Exception):
+        assert run_cell("""
+        %%julia
+        $(x+1)
+        """)
+
+def test_string_interp(run_cell):
+    run_cell("foo='python'")
+    assert run_cell("""
+    %%julia 
+    foo="julia"
+    "$foo", "$($foo)"
+    """) == ('julia','python')
+
+def test_expr_interp(run_cell):
+    run_cell("foo='python'")
+    assert run_cell("""
+    %%julia 
+    foo="julia"
+    :($foo), :($($foo))
+    """) == ('julia','python')
+
+def test_expr_py_interp(run_cell):
+    assert "baz" in str(run_cell("""
+    %julia :(py"baz")
+    """))
+
+def test_macro_esc(run_cell):
+    assert run_cell("""
+    %%julia
+    x = 1
+    @eval y = $x
+    y
+    """) == 1
+
+def test_type_conversion(run_cell):
+    assert run_cell("""
+    %julia py"1" isa Integer && py"1"o isa PyObject
+    """) == True
+
+def test_local_scope(run_cell):
+    assert run_cell("""
+    x = "global"
+    def f():
+        x = "local"
+        ret = %julia py"x"
+        return ret
+    f()
+    """) == "local"
+
+def test_global_scope(run_cell):
+    assert run_cell("""
+    x = "global"
+    def f():
+        ret = %julia py"x"
+        return ret
+    f()
+    """) == "global"
 
 def test_revise_error():
     from julia.ipy import revise