Add support for referring imported names (#1170)

Fixes #1154

Author: chrisrink10

CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+* Added support for referring imported Python names as by `from ... import ...` (#1154)
 
 ## [v0.3.8]
 ### Added
@@ -18,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  * Fix an issue where consecutive reader comment forms would not be ignored (#1207)
 
 ## [v0.3.7]
-### Fixed 
+### Fixed
  * Fix a regression introduced in #1176 where the testrunner couldn't handle relative paths in `sys.path`, causing `basilisp test` to fail when no arugments were provided (#1204)
  * Fix a bug where `basilisp.process/exec` could deadlock reading process output if that output exceeded the buffer size (#1202)
  * Fix `basilisp boostrap` issue on MS-Windows where the boostrap file loaded too early, before Basilisp was in `sys.path` (#1208)

docs/pyinterop.rst

@@ -49,13 +49,39 @@ Submodules will be available under the full, dot-separated name.
 
 To avoid name clashes from the above, you may alias imports (as in native Python code) using the same syntax as ``require``.
 Both top-level modules and submodules may be aliased: ``(import [module.sub :as sm])``.
-Note that none of the other convenience features or flags from :lpy:fn:`require` are available, so you will not be able to, say, refer unqualified module members into the current Namespace.
 
 .. code-block::
 
     (import [os.path :as path])
     (path/exists "test.txt") ;;=> false
 
+As with Basilisp ``refers`` (and as in Python), it is possible to refer individual module members by name into the current namespace using the ``:refer`` option.
+It is also possible to refer all module members into the namespace using ``:refer :all``.
+
+.. code-block::
+
+   (import [math :refer [sqrt pi]])
+   pi  ;; 3.141592653589793
+
+   (import [statistics :refer :all])
+   mean  ;; <function mean at 0x...>
+
+.. warning::
+
+   Basilisp refers names into the current module in different conceptual namespaces and resolves names against those namespaces in order of precedence, preferring Basilisp members first.
+   Referred Python module members may not resolve if other names take precedence within the current namespace context.
+
+   .. code-block::
+
+      (import [datetime :as dt :refer :all])
+
+      ;; This name using the module alias directly will guarantee we are referencing
+      ;; the module member `datetime.time` (a class)
+      dt/time  ;; <class 'datetime.time'>
+
+      ;; ...whereas this reference prefers the `basilisp.core` function `time`
+      time  ;; <function time at 0x...>
+
 .. note::
 
    Users should generally prefer to use the :lpy:fn:`ns` macro for importing modules into their namespace, rather than using the :lpy:fn:`import` form directly.
@@ -65,14 +91,9 @@ Note that none of the other convenience features or flags from :lpy:fn:`require`
       (ns myproject.ns
        (:import [os.path :as path]))
 
-.. warning::
-
-   Unlike in Python, imported module names and aliases cannot be referred to directly in Basilisp code.
-   Module and Namespace names are resolved separately from local names and will not resolve as unqualified names.
-
 .. seealso::
 
-   :lpy:form:`import`, :lpy:fn:`import`, :lpy:fn:`ns-imports`, :lpy:fn:`ns-map`
+   :lpy:form:`import`, :lpy:fn:`import`, :lpy:fn:`ns-imports`, :lpy:fn:`ns-import-refers`, :lpy:fn:`ns-map`
 
 .. seealso::
 

src/basilisp/core.lpy

@@ -4915,6 +4915,11 @@
   [ns]
   (.-imports (the-ns ns)))
 
+(defn ^:inline ns-import-refers
+  "Return a set of Python module members which are referred in the current namespace."
+  [ns]
+  (.-import-refers (the-ns ns)))
+
 (defn ^:inline ns-interns
   "Return a map of symbols to Vars which are interned in the current namespace."
   [ns]
@@ -4950,14 +4955,26 @@
 (defn ns-map
   "Return a map of all the mapped symbols in the namespace.
 
-  Includes the return values of :lpy:fn:`ns-interns` and :lpy:fn:`ns-refers` in one
-  map."
+  Includes the return values of :lpy:fn:`ns-interns`, :lpy:fn:`ns-refers`, and
+  :lpy:fn:`ns-imports` in one map.
+
+  .. note::
+
+     It is possible that the same symbol is mapped to an interned name, a referred
+     name, an imported name, or an import-referred name. They are stored in separate
+     collections in each namespace and Basilisp symbol resolution rules determine the
+     precedence. This function returns the combined set of all mapped symbols in a
+     single map and therefore symbols defined in multiple maps will only show one
+     value. Inspecting the contents of each of the component maps will show all
+     mappings, including those that were overwritten by the merge operation in this
+     function."
   ([] (ns-map *ns*))
   ([ns]
    (let [resolved-ns (the-ns ns)]
      (merge
-      (ns-interns resolved-ns)
-      (ns-refers resolved-ns)))))
+      (ns-imports resolved-ns)
+      (ns-refers resolved-ns)
+      (ns-interns resolved-ns)))))
 
 (defn ^:inline ns-resolve
   "Return the Var which will be resolved by the symbol in the given namespace."
@@ -4974,7 +4991,13 @@
   "Import Python modules by name.
 
   Modules may be specified either as symbols naming the full module path or as a
-  vector taking the form ``[full.module.path :as alias]``\\.
+  vector taking the form ``[full.module.path & opts]``\\. The ``opts`` should be
+  pairs of a keyword and a value from below, similar to :lpy:fn:`import`:
+
+  - ``:as name`` which will alias the imported module to the symbol name
+  - ``:refer [& syms]`` which will refer module members in the local namespace
+    directly
+  - ``:refer :all`` which will refer all module members from the namespace directly
 
   Note that unlike in Python, ``import`` ed Python module names are always hoisted to
   the current Namespace, so imported names will be available within a Namespace even

src/basilisp/lang/compiler/analyzer.py

@@ -175,9 +175,11 @@
 FINALLY = kw.keyword("finally")
 
 # Constants used in analyzing
+ALL = kw.keyword("all")
 AS = kw.keyword("as")
 IMPLEMENTS = kw.keyword("implements")
 INTERFACE = kw.keyword("interface")
+REFER = kw.keyword("refer")
 STAR_STAR = sym.symbol("**")
 _DOUBLE_DOT_MACRO_NAME = ".."
 _BUILTINS_NS = "python"
@@ -2571,8 +2573,8 @@ def _do_warn_on_import_or_require_name_clash(
 def _import_ast(form: ISeq, ctx: AnalyzerContext) -> Import:
     assert form.first == SpecialForm.IMPORT
 
-    aliases = []
-    for f in form.rest:
+    aliases, refers, refer_all = [], [], False
+    for f in form.rest:  # pylint: disable=too-many-nested-blocks
         if isinstance(f, sym.Symbol):
             module_name = f
             module_alias = None
@@ -2588,40 +2590,85 @@ def _import_ast(form: ISeq, ctx: AnalyzerContext) -> Import:
                 symbol_table=ctx.symbol_table.context_boundary,
             )
         elif isinstance(f, vec.PersistentVector):
-            if len(f) != 3:
+            if len(f) < 1:
                 raise ctx.AnalyzerException(
-                    "import alias must take the form: [module :as alias]", form=f
+                    "import alias must take the form: [module :as alias :refer [...]]",
+                    form=f,
                 )
             module_name = f.val_at(0)  # type: ignore[assignment]
             if not isinstance(module_name, sym.Symbol):
                 raise ctx.AnalyzerException(
                     "Python module name must be a symbol", form=f
                 )
-            if not AS == f.val_at(1):
+
+            try:
+                opts = lmap.hash_map(*f[1:])
+            except IndexError:
                 raise ctx.AnalyzerException(
-                    "expected :as alias for Python import", form=f
-                )
-            module_alias_sym = f.val_at(2)
-            if not isinstance(module_alias_sym, sym.Symbol):
+                    "Expected options: ':as alias' or ':refer [...]'", form=f
+                ) from None
+
+            if not {AS, REFER}.issuperset(set(opts.keys())):
                 raise ctx.AnalyzerException(
-                    "Python module alias must be a symbol", form=f
+                    f"Unexpected import options: {lset.set(opts.keys())}", form=f
                 )
-            module_alias = module_alias_sym.name
-            if "." in module_alias:
-                raise ctx.AnalyzerException(
-                    "Python module alias must not contain '.'", form=f
+
+            if (module_alias_sym := opts.val_at(AS)) is not None:
+                if not isinstance(module_alias_sym, sym.Symbol):
+                    raise ctx.AnalyzerException(
+                        "Python module alias must be a symbol", form=f
+                    )
+                module_alias = module_alias_sym.name
+                if "." in module_alias:
+                    raise ctx.AnalyzerException(
+                        "Python module alias must not contain '.'", form=f
+                    )
+
+                ctx.put_new_symbol(
+                    module_alias_sym,
+                    Binding(
+                        form=module_alias_sym,
+                        name=module_alias,
+                        local=LocalType.IMPORT,
+                        env=ctx.get_node_env(),
+                    ),
+                    symbol_table=ctx.symbol_table.context_boundary,
                 )
+            else:
+                module_alias = module_name.name
 
-            ctx.put_new_symbol(
-                module_alias_sym,
-                Binding(
-                    form=module_alias_sym,
-                    name=module_alias,
-                    local=LocalType.IMPORT,
-                    env=ctx.get_node_env(),
-                ),
-                symbol_table=ctx.symbol_table.context_boundary,
-            )
+            if (module_refers := opts.val_at(REFER)) is not None:
+                if ALL == module_refers:
+                    refer_all = True
+                else:
+                    if not isinstance(module_refers, vec.PersistentVector):
+                        raise ctx.AnalyzerException(
+                            "Python module refers must be a vector of symbols",
+                            form=module_refers,
+                        )
+
+                    if len(module_refers) == 0:
+                        raise ctx.AnalyzerException(
+                            "Must refer at least one name", form=module_refers
+                        )
+
+                    for refer in module_refers:
+                        if not isinstance(refer, sym.Symbol):
+                            raise ctx.AnalyzerException(
+                                "Python module refer name must be a symbol", form=refer
+                            )
+                        refers.append(refer.name)
+
+                        ctx.put_new_symbol(
+                            refer,
+                            Binding(
+                                form=refer,
+                                name=refer.name,
+                                local=LocalType.IMPORT,
+                                env=ctx.get_node_env(),
+                            ),
+                            symbol_table=ctx.symbol_table.context_boundary,
+                        )
         else:
             raise ctx.AnalyzerException("symbol or vector expected for import*", form=f)
 
@@ -2643,6 +2690,8 @@ def _import_ast(form: ISeq, ctx: AnalyzerContext) -> Import:
     return Import(
         form=form,
         aliases=aliases,
+        refers=refers,
+        refer_all=refer_all,
         env=ctx.get_node_env(pos=ctx.syntax_position),
     )
 
@@ -3738,7 +3787,7 @@ def __resolve_namespaced_symbol(  # pylint: disable=too-many-branches
 
 def __resolve_bare_symbol(
     ctx: AnalyzerContext, form: sym.Symbol
-) -> Union[Const, MaybeClass, VarRef]:
+) -> Union[Const, HostField, MaybeClass, VarRef]:
     """Resolve a non-namespaced symbol into a Python name or a local Basilisp Var."""
     assert form.ns is None
 
@@ -3767,6 +3816,25 @@ def __resolve_bare_symbol(
             env=ctx.get_node_env(pos=ctx.syntax_position),
         )
 
+    maybe_import_refer_module = current_ns.get_import_refer(form)
+    if maybe_import_refer_module is not None:
+        refer_module = current_ns.get_import(maybe_import_refer_module)
+        # For referred imports, we want to generate a fully qualified reference
+        # to the object, so we don't have to pollute the module with more names.
+        # The user won't know the difference.
+        return HostField(
+            form=form,
+            field=munge(form.name),
+            target=MaybeClass(
+                form=maybe_import_refer_module,
+                class_=munge(maybe_import_refer_module.name),
+                target=refer_module,
+                env=ctx.get_node_env(pos=ctx.syntax_position),
+            ),
+            is_assignable=False,
+            env=ctx.get_node_env(pos=ctx.syntax_position),
+        )
+
     # Allow users to resolve imported module names directly
     maybe_import = current_ns.get_import(form)
     if maybe_import is not None:

src/basilisp/lang/compiler/generator.py

@@ -2409,6 +2409,62 @@ def _import_to_py_ast(ctx: GeneratorContext, node: Import) -> GeneratedPyAST[ast
                 ),
             )
         )
+
+        refers: Optional[ast.expr] = None
+        if node.refer_all:
+            key, val = genname("k"), genname("v")
+            refers = ast.DictComp(
+                key=ast.Call(
+                    func=_NEW_SYM_FN_NAME,
+                    args=[ast.Name(id=key, ctx=ast.Load())],
+                    keywords=[],
+                ),
+                value=ast.Name(id=val, ctx=ast.Load()),
+                generators=[
+                    ast.comprehension(
+                        target=ast.Tuple(
+                            elts=[
+                                ast.Name(id=key, ctx=ast.Store()),
+                                ast.Name(id=val, ctx=ast.Store()),
+                            ],
+                            ctx=ast.Store(),
+                        ),
+                        iter=ast.Call(
+                            func=ast.Attribute(
+                                value=ast.Call(
+                                    func=ast.Name(id="vars", ctx=ast.Load()),
+                                    args=[ast.Name(id=py_import_alias, ctx=ast.Load())],
+                                    keywords=[],
+                                ),
+                                attr="items",
+                                ctx=ast.Load(),
+                            ),
+                            args=[],
+                            keywords=[],
+                        ),
+                        ifs=[],
+                        is_async=0,
+                    )
+                ],
+            )
+        elif node.refers:
+            refer_keys: list[Optional[ast.expr]] = []
+            refer_vals: list[ast.expr] = []
+            for refer in node.refers:
+                refer_keys.append(
+                    ast.Call(
+                        func=_NEW_SYM_FN_NAME, args=[ast.Constant(refer)], keywords=[]
+                    )
+                )
+                refer_vals.append(
+                    ast.Attribute(
+                        value=ast.Name(id=py_import_alias, ctx=ast.Load()),
+                        attr=refer,
+                        ctx=ast.Load(),
+                    )
+                )
+            refers = ast.Dict(keys=refer_keys, values=refer_vals)
+
         last = ast.Name(id=py_import_alias, ctx=ast.Load())
 
         deps.append(
@@ -2437,7 +2493,11 @@ def _import_to_py_ast(ctx: GeneratorContext, node: Import) -> GeneratedPyAST[ast
                         ),
                     )
                 ),
-                keywords=[],
+                keywords=(
+                    [ast.keyword(arg="refers", value=refers)]
+                    if refers is not None
+                    else []
+                ),
             )
         )
 

src/basilisp/lang/compiler/nodes.py

@@ -612,6 +612,8 @@ class If(Node[SpecialForm]):
 class Import(Node[SpecialForm]):
     form: SpecialForm
     aliases: Iterable["ImportAlias"]
+    refers: Iterable[str]
+    refer_all: bool
     env: NodeEnv = attr.field(hash=False)
     children: Sequence[kw.Keyword] = vec.EMPTY
     op: NodeOp = NodeOp.IMPORT