[mypyc] Promotion to Value Type optimization

mypy/options.py

@@ -75,7 +75,8 @@ class BuildType:
 PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes"
 NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax"
 INLINE_TYPEDDICT: Final = "InlineTypedDict"
-INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT))
+MYPYC_VALUE_TYPES: Final = "MypycValueTypes"
+INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT, MYPYC_VALUE_TYPES))
 COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, NEW_GENERIC_SYNTAX))
 
 

mypyc/__main__.py

@@ -24,7 +24,7 @@
 from mypyc.build import mypycify
 
 setup(name='mypyc_output',
-      ext_modules=mypycify({}, opt_level="{}", debug_level="{}", strict_dunder_typing={}),
+      ext_modules=mypycify({}, opt_level="{}", debug_level="{}", strict_dunder_typing={}, compiler_type="{}"),
 )
 """
 
@@ -39,10 +39,15 @@ def main() -> None:
     opt_level = os.getenv("MYPYC_OPT_LEVEL", "3")
     debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1")
     strict_dunder_typing = bool(int(os.getenv("MYPYC_STRICT_DUNDER_TYPING", "0")))
+    compiler = os.getenv("MYPYC_COMPILER", "")
 
     setup_file = os.path.join(build_dir, "setup.py")
     with open(setup_file, "w") as f:
-        f.write(setup_format.format(sys.argv[1:], opt_level, debug_level, strict_dunder_typing))
+        f.write(
+            setup_format.format(
+                sys.argv[1:], opt_level, debug_level, strict_dunder_typing, compiler
+            )
+        )
 
     # We don't use run_setup (like we do in the test suite) because it throws
     # away the error code from distutils, and we don't care about the slight

mypyc/build.py

@@ -31,7 +31,7 @@
 from mypy.errors import CompileError
 from mypy.fscache import FileSystemCache
 from mypy.main import process_options
-from mypy.options import Options
+from mypy.options import MYPYC_VALUE_TYPES, Options
 from mypy.util import write_junit_xml
 from mypyc.codegen import emitmodule
 from mypyc.common import RUNTIME_C_FILES, shared_lib_name
@@ -418,6 +418,11 @@ def mypyc_build(
         paths, only_compile_paths, compiler_options, fscache
     )
 
+    # Enable value types option via mypy options
+    compiler_options.experimental_value_types = (
+        MYPYC_VALUE_TYPES in options.enable_incomplete_feature
+    )
+
     # We generate a shared lib if there are multiple modules or if any
     # of the modules are in package. (Because I didn't want to fuss
     # around with making the single module code handle packages.)
@@ -471,6 +476,7 @@ def mypycify(
     target_dir: str | None = None,
     include_runtime_files: bool | None = None,
     strict_dunder_typing: bool = False,
+    compiler_type: str | None = None,
 ) -> list[Extension]:
     """Main entry point to building using mypyc.
 
@@ -496,7 +502,7 @@ def mypycify(
         separate: Should compiled modules be placed in separate extension modules.
                   If False, all modules are placed in a single shared library.
                   If True, every module is placed in its own library.
-                  Otherwise separate should be a list of
+                  Otherwise, separate should be a list of
                   (file name list, optional shared library name) pairs specifying
                   groups of files that should be placed in the same shared library
                   (while all other modules will be placed in its own library).
@@ -513,6 +519,9 @@ def mypycify(
         strict_dunder_typing: If True, force dunder methods to have the return type
                               of the method strictly, which can lead to more
                               optimization opportunities. Defaults to False.
+        compiler_type: The distutils compiler. If None or empty, the default compiler
+                       is used. Some possible values are 'msvc', 'mingw32', 'unix'.
+                       Defaults to None.
     """
 
     # Figure out our configuration
@@ -541,17 +550,18 @@ def mypycify(
     # Create a compiler object so we can make decisions based on what
     # compiler is being used. typeshed is missing some attributes on the
     # compiler object so we give it type Any
-    compiler: Any = ccompiler.new_compiler()
+    compiler: Any = ccompiler.new_compiler(compiler=compiler_type or None, verbose=verbose)
     sysconfig.customize_compiler(compiler)
 
     build_dir = compiler_options.target_dir
 
     cflags: list[str] = []
-    if compiler.compiler_type == "unix":
+    if compiler.compiler_type in ("mingw32", "unix"):
         cflags += [
             f"-O{opt_level}",
             f"-g{debug_level}",
             "-Werror",
+            "-Wno-missing-braces",
             "-Wno-unused-function",
             "-Wno-unused-label",
             "-Wno-unreachable-code",

mypyc/codegen/emit.py

@@ -23,6 +23,7 @@
 from mypyc.ir.ops import BasicBlock, Value
 from mypyc.ir.rtypes import (
     RInstance,
+    RInstanceValue,
     RPrimitive,
     RTuple,
     RType,
@@ -322,6 +323,8 @@ def c_undefined_value(self, rtype: RType) -> str:
             return rtype.c_undefined
         elif isinstance(rtype, RTuple):
             return self.tuple_undefined_value(rtype)
+        elif isinstance(rtype, RInstanceValue):
+            return self.rinstance_value_undefined_value(rtype)
         assert False, rtype
 
     def c_error_value(self, rtype: RType) -> str:
@@ -358,14 +361,17 @@ def bitmap_field(self, index: int) -> str:
             return "bitmap"
         return f"bitmap{n + 1}"
 
-    def attr_bitmap_expr(self, obj: str, cl: ClassIR, index: int) -> str:
+    def attr_bitmap_expr(self, obj: str, cl: RInstance, index: int) -> str:
         """Return reference to the attribute definedness bitmap."""
-        cast = f"({cl.struct_name(self.names)} *)"
         attr = self.bitmap_field(index)
-        return f"({cast}{obj})->{attr}"
+        if cl.is_unboxed:
+            return f"{obj}.{attr}"
+        else:
+            cast = f"({cl.struct_name(self.names)} *)"
+            return f"({cast}{obj})->{attr}"
 
     def emit_attr_bitmap_set(
-        self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str
+        self, value: str, obj: str, rtype: RType, cl: RInstance, attr: str
     ) -> None:
         """Mark an attribute as defined in the attribute bitmap.
 
@@ -374,20 +380,20 @@ def emit_attr_bitmap_set(
         """
         self._emit_attr_bitmap_update(value, obj, rtype, cl, attr, clear=False)
 
-    def emit_attr_bitmap_clear(self, obj: str, rtype: RType, cl: ClassIR, attr: str) -> None:
+    def emit_attr_bitmap_clear(self, obj: str, rtype: RType, cl: RInstance, attr: str) -> None:
         """Mark an attribute as undefined in the attribute bitmap.
 
         Unlike emit_attr_bitmap_set, clear unconditionally.
         """
         self._emit_attr_bitmap_update("", obj, rtype, cl, attr, clear=True)
 
     def _emit_attr_bitmap_update(
-        self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str, clear: bool
+        self, value: str, obj: str, rtype: RType, cl: RInstance, attr: str, clear: bool
     ) -> None:
         if value:
             check = self.error_value_check(rtype, value, "==")
             self.emit_line(f"if (unlikely({check})) {{")
-        index = cl.bitmap_attrs.index(attr)
+        index = cl.class_ir.bitmap_attrs.index(attr)
         mask = 1 << (index & (BITMAP_BITS - 1))
         bitmap = self.attr_bitmap_expr(obj, cl, index)
         if clear:
@@ -407,26 +413,28 @@ def emit_undefined_attr_check(
         compare: str,
         obj: str,
         attr: str,
-        cl: ClassIR,
+        cl: RInstance,
         *,
         unlikely: bool = False,
     ) -> None:
         check = self.error_value_check(rtype, attr_expr, compare)
         if unlikely:
             check = f"unlikely({check})"
         if rtype.error_overlap:
-            index = cl.bitmap_attrs.index(attr)
+            index = cl.class_ir.bitmap_attrs.index(attr)
+            attr_expr = self.attr_bitmap_expr(obj, cl, index)
             bit = 1 << (index & (BITMAP_BITS - 1))
-            attr = self.bitmap_field(index)
-            obj_expr = f"({cl.struct_name(self.names)} *){obj}"
-            check = f"{check} && !(({obj_expr})->{attr} & {bit})"
+            check = f"{check} && !({attr_expr} & {bit})"
+
         self.emit_line(f"if ({check}) {{")
 
     def error_value_check(self, rtype: RType, value: str, compare: str) -> str:
         if isinstance(rtype, RTuple):
             return self.tuple_undefined_check_cond(
                 rtype, value, self.c_error_value, compare, check_exception=False
             )
+        elif isinstance(rtype, RInstanceValue):
+            return f"{value}.vtable {compare} NULL"
         else:
             return f"{value} {compare} {self.c_error_value(rtype)}"
 
@@ -458,6 +466,9 @@ def tuple_undefined_check_cond(
             return self.tuple_undefined_check_cond(
                 item_type, tuple_expr_in_c + f".f{i}", c_type_compare_val, compare
             )
+        elif isinstance(item_type, RInstanceValue):
+            check = f"{tuple_expr_in_c}.f{i}.vtable {compare} NULL"
+            return check
         else:
             check = f"{tuple_expr_in_c}.f{i} {compare} {c_type_compare_val(item_type)}"
             if rtuple.error_overlap and check_exception:
@@ -468,6 +479,10 @@ def tuple_undefined_value(self, rtuple: RTuple) -> str:
         """Undefined tuple value suitable in an expression."""
         return f"({rtuple.struct_name}) {self.c_initializer_undefined_value(rtuple)}"
 
+    def rinstance_value_undefined_value(self, rinstance_value: RInstanceValue) -> str:
+        """Undefined value for an unboxed instance."""
+        return f"(({rinstance_value.struct_name(self.names)}){self.c_initializer_undefined_value(rinstance_value)})"
+
     def c_initializer_undefined_value(self, rtype: RType) -> str:
         """Undefined value represented in a form suitable for variable initialization."""
         if isinstance(rtype, RTuple):
@@ -477,6 +492,8 @@ def c_initializer_undefined_value(self, rtype: RType) -> str:
                 return f"{{ {int_rprimitive.c_undefined} }}"
             items = ", ".join([self.c_initializer_undefined_value(t) for t in rtype.types])
             return f"{{ {items} }}"
+        elif isinstance(rtype, RInstanceValue):
+            return "{0}"
         else:
             return self.c_undefined_value(rtype)
 
@@ -489,6 +506,8 @@ def declare_tuple_struct(self, tuple_type: RTuple) -> None:
                 # XXX other types might eventually need similar behavior
                 if isinstance(typ, RTuple):
                     dependencies.add(typ.struct_name)
+                if isinstance(typ, RInstanceValue):
+                    dependencies.add(typ.struct_name(self.names))
 
             self.context.declarations[tuple_type.struct_name] = HeaderDeclaration(
                 self.tuple_c_declaration(tuple_type), dependencies=dependencies, is_type=True
@@ -510,6 +529,10 @@ def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None:
         elif isinstance(rtype, RTuple):
             for i, item_type in enumerate(rtype.types):
                 self.emit_inc_ref(f"{dest}.f{i}", item_type)
+        elif isinstance(rtype, RInstanceValue):
+            if rtype.is_refcounted:
+                for attr, attr_type in rtype.class_ir.all_attributes().items():
+                    self.emit_inc_ref(f"{dest}.{self.attr(attr)}", attr_type)
         elif not rtype.is_unboxed:
             # Always inline, since this is a simple op
             self.emit_line("CPy_INCREF(%s);" % dest)
@@ -535,6 +558,10 @@ def emit_dec_ref(
         elif isinstance(rtype, RTuple):
             for i, item_type in enumerate(rtype.types):
                 self.emit_dec_ref(f"{dest}.f{i}", item_type, is_xdec=is_xdec, rare=rare)
+        elif isinstance(rtype, RInstanceValue):
+            if rtype.is_refcounted:
+                for attr, attr_type in rtype.class_ir.all_attributes().items():
+                    self.emit_dec_ref(f"{dest}.{self.attr(attr)}", attr_type, is_xdec=is_xdec)
         elif not rtype.is_unboxed:
             if rare:
                 self.emit_line(f"CPy_{x}DecRef({dest});")
@@ -987,7 +1014,17 @@ def emit_unbox(
             self.emit_line("}")
             if optional:
                 self.emit_line("}")
+        elif isinstance(typ, RInstanceValue):
+            if declare_dest:
+                self.emit_line(f"{self.ctype(typ)} {dest};")
+            if optional:
+                self.emit_line(f"if ({src} == NULL) {{")
+                self.emit_line(f"{dest} = {self.c_error_value(typ)};")
+                self.emit_line("} else {")
 
+            self.emit_line(f"{dest} = *({self.ctype(typ)} *){src};")
+            if optional:
+                self.emit_line("}")
         else:
             assert False, "Unboxing not implemented: %s" % typ
 
@@ -1041,6 +1078,31 @@ def emit_box(
                     inner_name = self.temp_name()
                     self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True)
                     self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});")
+        elif isinstance(typ, RInstanceValue):
+            cl = typ.class_ir
+            generate_full = not cl.is_trait and not cl.builtin_base
+            assert generate_full, "Only full classes can be boxed"  # only those have setup method
+            name_prefix = cl.name_prefix(self.names)
+            setup_name = f"{name_prefix}_setup"
+            py_type_struct = self.type_struct_name(cl)
+            temp_dest = self.temp_name()
+            self.emit_line(
+                f"{self.ctype_spaced(typ)}*{temp_dest} = ({self.ctype_spaced(typ)}*){setup_name}({py_type_struct});"
+            )
+            self.emit_line(f"if (unlikely({temp_dest} == NULL))")
+            self.emit_line("    CPyError_OutOfMemory();")
+            if cl.bitmap_attrs:
+                n_fields = (len(cl.bitmap_attrs) - 1) // BITMAP_BITS + 1
+                for i in range(n_fields):
+                    attr_name = self.bitmap_field(i * BITMAP_BITS)
+                    self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box")
+            for attr, attr_type in cl.all_attributes().items():
+                attr_name = self.attr(attr)
+                self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box")
+                if attr_type.is_refcounted:
+                    self.emit_inc_ref(f"{temp_dest}->{attr_name}", attr_type)
+
+            self.emit_line(f"{declaration}{dest} = (PyObject *){temp_dest};")
         else:
             assert not typ.is_unboxed
             # Type is boxed -- trivially just assign.
@@ -1054,6 +1116,8 @@ def emit_error_check(self, value: str, rtype: RType, failure: str) -> None:
             else:
                 cond = self.tuple_undefined_check_cond(rtype, value, self.c_error_value, "==")
                 self.emit_line(f"if ({cond}) {{")
+        elif isinstance(rtype, RInstanceValue):
+            self.emit_line(f"if ({value}.vtable == NULL) {{")
         elif rtype.error_overlap:
             # The error value is also valid as a normal value, so we need to also check
             # for a raised exception.
@@ -1078,6 +1142,9 @@ def emit_gc_visit(self, target: str, rtype: RType) -> None:
         elif isinstance(rtype, RTuple):
             for i, item_type in enumerate(rtype.types):
                 self.emit_gc_visit(f"{target}.f{i}", item_type)
+        elif isinstance(rtype, RInstanceValue):
+            for attr, attr_type in rtype.class_ir.all_attributes().items():
+                self.emit_gc_visit(f"{target}.{self.attr(attr)}", attr_type)
         elif self.ctype(rtype) == "PyObject *":
             # The simplest case.
             self.emit_line(f"Py_VISIT({target});")
@@ -1102,6 +1169,9 @@ def emit_gc_clear(self, target: str, rtype: RType) -> None:
         elif isinstance(rtype, RTuple):
             for i, item_type in enumerate(rtype.types):
                 self.emit_gc_clear(f"{target}.f{i}", item_type)
+        elif isinstance(rtype, RInstanceValue):
+            for attr, attr_type in rtype.class_ir.all_attributes().items():
+                self.emit_gc_clear(f"{target}.{self.attr(attr)}", attr_type)
         elif self.ctype(rtype) == "PyObject *" and self.c_undefined_value(rtype) == "NULL":
             # The simplest case.
             self.emit_line(f"Py_CLEAR({target});")