From 6868d348e027dc82372de8f659d935275dfba9a4 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:47:10 -0500 Subject: [PATCH 01/21] Promote immutable RInstances to value types --- mypyc/codegen/emit.py | 41 ++++++++++++++++++++++++++ mypyc/codegen/emitclass.py | 39 ++++++++++++++++++------- mypyc/codegen/emitfunc.py | 42 ++++++++++++++++++++++++-- mypyc/codegen/emitmodule.py | 3 ++ mypyc/ir/class_ir.py | 24 +++++++++++++-- mypyc/ir/rtypes.py | 47 +++++++++++++++++++++++++++++- mypyc/irbuild/builder.py | 4 +-- mypyc/irbuild/callable_class.py | 4 +-- mypyc/irbuild/env_class.py | 39 ++++++++++++------------- mypyc/irbuild/function.py | 20 +++++-------- mypyc/irbuild/generator.py | 14 ++++----- mypyc/irbuild/mapper.py | 3 +- mypyc/irbuild/prepare.py | 12 +++++--- mypyc/irbuild/util.py | 20 +++++++++++++ mypyc/rt_subtype.py | 4 +++ mypyc/sametype.py | 4 +++ mypyc/subtype.py | 4 +++ mypyc/transform/value_type_init.py | 35 ++++++++++++++++++++++ 18 files changed, 290 insertions(+), 69 deletions(-) create mode 100644 mypyc/transform/value_type_init.py diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index fce6896e8d11..604bd1c76a93 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -23,6 +23,7 @@ from mypyc.ir.ops import BasicBlock, Value from mypyc.ir.rtypes import ( RInstance, + RInstanceValue, RPrimitive, RTuple, RType, @@ -322,6 +323,8 @@ def c_undefined_value(self, rtype: RType) -> str: return rtype.c_undefined elif isinstance(rtype, RTuple): return self.tuple_undefined_value(rtype) + elif isinstance(rtype, RInstanceValue): + return self.rinstance_value_undefined_value(rtype) assert False, rtype def c_error_value(self, rtype: RType) -> str: @@ -427,6 +430,8 @@ def error_value_check(self, rtype: RType, value: str, compare: str) -> str: return self.tuple_undefined_check_cond( rtype, value, self.c_error_value, compare, check_exception=False ) + elif isinstance(rtype, RInstanceValue): + return f"{value}.vtable {compare} NULL" else: return f"{value} {compare} {self.c_error_value(rtype)}" @@ -468,6 +473,10 @@ def tuple_undefined_value(self, rtuple: RTuple) -> str: """Undefined tuple value suitable in an expression.""" return f"({rtuple.struct_name}) {self.c_initializer_undefined_value(rtuple)}" + def rinstance_value_undefined_value(self, rinstance_value: RInstanceValue) -> str: + """Undefined value for an unboxed instance.""" + return f"(({rinstance_value.struct_name(self.names)}){self.c_initializer_undefined_value(rinstance_value)})" + def c_initializer_undefined_value(self, rtype: RType) -> str: """Undefined value represented in a form suitable for variable initialization.""" if isinstance(rtype, RTuple): @@ -477,6 +486,8 @@ def c_initializer_undefined_value(self, rtype: RType) -> str: return f"{{ {int_rprimitive.c_undefined} }}" items = ", ".join([self.c_initializer_undefined_value(t) for t in rtype.types]) return f"{{ {items} }}" + elif isinstance(rtype, RInstanceValue): + return "{0}" else: return self.c_undefined_value(rtype) @@ -987,7 +998,17 @@ def emit_unbox( self.emit_line("}") if optional: self.emit_line("}") + elif isinstance(typ, RInstanceValue): + if declare_dest: + self.emit_line(f"{self.ctype(typ)} {dest};") + if optional: + self.emit_line(f"if ({src} == NULL) {{") + self.emit_line(f"{dest} = {self.c_error_value(typ)};") + self.emit_line("} else {") + self.emit_line(f"{dest} = *({self.ctype(typ)} *){src};") + if optional: + self.emit_line("}") else: assert False, "Unboxing not implemented: %s" % typ @@ -1041,6 +1062,24 @@ def emit_box( inner_name = self.temp_name() self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") + elif isinstance(typ, RInstanceValue): + cl = typ.class_ir + generate_full = not cl.is_trait and not cl.builtin_base + assert generate_full, "Only full classes can be boxed" # only those have setup method + name_prefix = cl.name_prefix(self.names) + setup_name = f"{name_prefix}_setup" + py_type_struct = self.type_struct_name(cl) + temp_dest = self.temp_name() + self.emit_line( + f"{self.ctype_spaced(typ)}*{temp_dest} = ({self.ctype_spaced(typ)}*){setup_name}({py_type_struct});" + ) + self.emit_line(f"if (unlikely({temp_dest} == NULL))") + self.emit_line(" CPyError_OutOfMemory();") + for attr, attr_type in cl.attributes.items(): + attr_name = self.attr(attr) + self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box") + + self.emit_line(f"{declaration}{dest} = (PyObject *){temp_dest};") else: assert not typ.is_unboxed # Type is boxed -- trivially just assign. @@ -1054,6 +1093,8 @@ def emit_error_check(self, value: str, rtype: RType, failure: str) -> None: else: cond = self.tuple_undefined_check_cond(rtype, value, self.c_error_value, "==") self.emit_line(f"if ({cond}) {{") + elif isinstance(rtype, RInstanceValue): + self.emit_line(f"if ({value}.vtable == NULL) {{") elif rtype.error_overlap: # The error value is also valid as a normal value, so we need to also check # for a raised exception. diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 3ab6932546a6..6327e7f0311b 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -294,7 +294,10 @@ def emit_line() -> None: # Declare setup method that allocates and initializes an object. type is the # type of the class being initialized, which could be another class if there # is an interpreted subclass. - emitter.emit_line(f"static PyObject *{setup_name}(PyTypeObject *type);") + emitter.context.declarations[setup_name] = HeaderDeclaration( + f"PyObject *{setup_name}(PyTypeObject *type);", needs_export=True + ) + assert cl.ctor is not None emitter.emit_line(native_function_header(cl.ctor, emitter) + ";") @@ -394,6 +397,7 @@ def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None: if attr not in bitmap_attrs: lines.append(f"{BITMAP_TYPE} {attr};") bitmap_attrs.append(attr) + for attr, rtype in base.attributes.items(): if (attr, rtype) not in seen_attrs: lines.append(f"{emitter.ctype_spaced(rtype)}{emitter.attr(attr)};") @@ -555,7 +559,7 @@ def generate_setup_for_class( emitter: Emitter, ) -> None: """Generate a native function that allocates an instance of a class.""" - emitter.emit_line("static PyObject *") + emitter.emit_line("PyObject *") emitter.emit_line(f"{func_name}(PyTypeObject *type)") emitter.emit_line("{") emitter.emit_line(f"{cl.struct_name(emitter.names)} *self;") @@ -586,7 +590,7 @@ def generate_setup_for_class( # We don't need to set this field to NULL since tp_alloc() already # zero-initializes `self`. - if value != "NULL": + if value not in ("NULL", "0"): emitter.emit_line(rf"self->{emitter.attr(attr)} = {value};") # Initialize attributes to default values, if necessary @@ -615,10 +619,15 @@ def generate_constructor_for_class( """Generate a native function that allocates and initializes an instance of a class.""" emitter.emit_line(f"{native_function_header(fn, emitter)}") emitter.emit_line("{") - emitter.emit_line(f"PyObject *self = {setup_name}({emitter.type_struct_name(cl)});") - emitter.emit_line("if (self == NULL)") - emitter.emit_line(" return NULL;") - args = ", ".join(["self"] + [REG_PREFIX + arg.name for arg in fn.sig.args]) + if cl.is_value_type: + emitter.emit_line(f"{cl.struct_name(emitter.names)} self = {{0}};") + emitter.emit_line(f"self.vtable = {vtable_name};") + args = ", ".join(["(PyObject*)&self"] + [REG_PREFIX + arg.name for arg in fn.sig.args]) + else: + emitter.emit_line(f"PyObject *self = {setup_name}({emitter.type_struct_name(cl)});") + emitter.emit_line("if (self == NULL)") + emitter.emit_line(" return NULL;") + args = ", ".join(["self"] + [REG_PREFIX + arg.name for arg in fn.sig.args]) if init_fn is not None: emitter.emit_line( "char res = {}{}{}({});".format( @@ -629,8 +638,12 @@ def generate_constructor_for_class( ) ) emitter.emit_line("if (res == 2) {") - emitter.emit_line("Py_DECREF(self);") - emitter.emit_line("return NULL;") + if cl.is_value_type: + emitter.emit_line("self.vtable = NULL;") + emitter.emit_line("return self;") + else: + emitter.emit_line("Py_DECREF(self);") + emitter.emit_line("return NULL;") emitter.emit_line("}") # If there is a nontrivial ctor that we didn't define, invoke it via tp_init @@ -638,8 +651,12 @@ def generate_constructor_for_class( emitter.emit_line(f"int res = {emitter.type_struct_name(cl)}->tp_init({args});") emitter.emit_line("if (res < 0) {") - emitter.emit_line("Py_DECREF(self);") - emitter.emit_line("return NULL;") + if cl.is_value_type: + emitter.emit_line("self.vtable = NULL;") + emitter.emit_line("return self;") + else: + emitter.emit_line("Py_DECREF(self);") + emitter.emit_line("return NULL;") emitter.emit_line("}") emitter.emit_line("return self;") diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 6088fb06dd32..246eddea21ef 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -71,11 +71,14 @@ ) from mypyc.ir.pprint import generate_names_for_ir from mypyc.ir.rtypes import ( + PyObject, RArray, RInstance, + RInstanceValue, RStruct, RTuple, RType, + c_pyssize_t_rprimitive, is_int32_rprimitive, is_int64_rprimitive, is_int_rprimitive, @@ -84,6 +87,18 @@ ) +def struct_type(class_ir: ClassIR, emitter: Emitter) -> RStruct: + """Return the struct type for this instance type.""" + python_fields: list[tuple[str, RType]] = [ + ("head", PyObject), + ("vtable", c_pyssize_t_rprimitive), + ] + class_fields = list(class_ir.attributes.items()) + attr_names = [emitter.attr(name) for name, _ in python_fields + class_fields] + attr_types = [rtype for _, rtype in python_fields + class_fields] + return RStruct(class_ir.struct_name(emitter.names), attr_names, attr_types) + + def native_function_type(fn: FuncIR, emitter: Emitter) -> str: args = ", ".join(emitter.ctype(arg.type) for arg in fn.args) or "void" ret = emitter.ctype(fn.ret_type) @@ -229,6 +244,8 @@ def visit_branch(self, op: Branch) -> None: cond = self.emitter.tuple_undefined_check_cond( typ, self.reg(op.value), self.c_error_value, compare ) + elif isinstance(typ, RInstanceValue): + cond = f"{self.reg(op.value)}.vtable {compare} NULL" else: cond = f"{self.reg(op.value)} {compare} {self.c_error_value(typ)}" else: @@ -352,7 +369,11 @@ def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> st if op.class_type.class_ir.is_trait: assert not decl_cl.is_trait cast = f"({decl_cl.struct_name(self.emitter.names)} *)" - return f"({cast}{obj})->{self.emitter.attr(op.attr)}" + + if op.obj.type.is_unboxed: + return f"{obj}.{self.emitter.attr(op.attr)}" + else: + return f"({cast}{obj})->{self.emitter.attr(op.attr)}" def visit_get_attr(self, op: GetAttr) -> None: dest = self.reg(op) @@ -383,8 +404,18 @@ def visit_get_attr(self, op: GetAttr) -> None: else: # Otherwise, use direct or offset struct access. attr_expr = self.get_attr_expr(obj, op, decl_cl) - self.emitter.emit_line(f"{dest} = {attr_expr};") always_defined = cl.is_always_defined(op.attr) + # This steals the reference to src, so we don't need to increment the arg + if isinstance(attr_rtype, RInstance) and attr_rtype.class_ir.is_value_type: + # special case for value types, it is unboxed in the struct + struct_name = attr_rtype.class_ir.struct_name(self.names) + temp = self.emitter.temp_name() + self.emitter.emit_line(f"{struct_name} {temp} = {attr_expr};") + self.emitter.emit_line(f"{dest} = (PyObject *)&{temp};") + always_defined = True + else: + self.emitter.emit_line(f"{dest} = {attr_expr};") + merged_branch = None if not always_defined: self.emitter.emit_undefined_attr_check( @@ -481,7 +512,12 @@ def visit_set_attr(self, op: SetAttr) -> None: self.emitter.emit_attr_bitmap_set(src, obj, attr_rtype, cl, op.attr) # This steals the reference to src, so we don't need to increment the arg - self.emitter.emit_line(f"{attr_expr} = {src};") + if isinstance(attr_rtype, RInstance) and attr_rtype.class_ir.is_value_type: + # special case for value types, it is unboxed in the struct + struct_name = attr_rtype.class_ir.struct_name(self.names) + self.emitter.emit_line(f"{attr_expr} = *({struct_name} *)({src});") + else: + self.emitter.emit_line(f"{attr_expr} = {src};") if op.error_kind == ERR_FALSE: self.emitter.emit_line(f"{dest} = 1;") diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 1d8708912de5..92a6ee6f7434 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -63,6 +63,7 @@ from mypyc.transform.lower import lower_ir from mypyc.transform.refcount import insert_ref_count_opcodes from mypyc.transform.uninit import insert_uninit_checks +from mypyc.transform.value_type_init import patch_value_type_init_methods # All of the modules being compiled are divided into "groups". A group # is a set of modules that are placed into the same shared library. @@ -231,6 +232,8 @@ def compile_scc_to_ir( for module in modules.values(): for fn in module.functions: + # Patch init methods for Value Types + patch_value_type_init_methods(fn, compiler_options) # Insert uninit checks. insert_uninit_checks(fn) # Insert exception handling. diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 94bf714b28d4..a27e51188b61 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -7,7 +7,7 @@ from mypyc.common import PROPSET_PREFIX, JsonDict from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature from mypyc.ir.ops import DeserMaps, Value -from mypyc.ir.rtypes import RInstance, RType, deserialize_type +from mypyc.ir.rtypes import RInstance, RInstanceValue, RType, deserialize_type from mypyc.namegen import NameGenerator, exported_name # Some notes on the vtable layout: Each concrete class has a vtable @@ -94,6 +94,8 @@ def __init__( is_abstract: bool = False, is_ext_class: bool = True, is_final_class: bool = False, + is_immutable: bool = False, + is_value_type: bool = False, ) -> None: self.name = name self.module_name = module_name @@ -102,6 +104,7 @@ def __init__( self.is_abstract = is_abstract self.is_ext_class = is_ext_class self.is_final_class = is_final_class + self.is_immutable = is_immutable # An augmented class has additional methods separate from what mypyc generates. # Right now the only one is dataclasses. self.is_augmented = False @@ -114,6 +117,8 @@ def __init__( # Does this class need getseters to be generated for its attributes? (getseters are also # added if is_generated is False) self.needs_getseters = False + # A value type is a class that can be passed by value instead of by reference. + self.is_value_type = self.is_ext_class and self.is_immutable and not self.has_dict and is_value_type # Is this class declared as serializable (supports copy.copy # and pickle) using @mypyc_attr(serializable=True)? # @@ -131,8 +136,10 @@ def __init__( # of the object for that class. We currently only support this # in a few ad-hoc cases. self.builtin_base: str | None = None + # The RType for instances of this class + self.rtype = RInstanceValue(self) if self.is_value_type else RInstance(self) # Default empty constructor - self.ctor = FuncDecl(name, None, module_name, FuncSignature([], RInstance(self))) + self.ctor = FuncDecl(name, None, module_name, FuncSignature([], self.rtype)) # Attributes defined in the class (not inherited) self.attributes: dict[str, RType] = {} # Deletable attributes @@ -202,7 +209,8 @@ def __repr__(self) -> str: "name={self.name}, module_name={self.module_name}, " "is_trait={self.is_trait}, is_generated={self.is_generated}, " "is_abstract={self.is_abstract}, is_ext_class={self.is_ext_class}, " - "is_final_class={self.is_final_class}" + "is_final_class={self.is_final_class}, is_immutable={self.is_immutable}, " + "is_value_type={self.is_value_type}" ")".format(self=self) ) @@ -283,6 +291,12 @@ def name_prefix(self, names: NameGenerator) -> str: def struct_name(self, names: NameGenerator) -> str: return f"{exported_name(self.fullname)}Object" + def struct_name2(self) -> str: + return f"{exported_name(self.fullname)}Object" + + def struct_data_name(self) -> str: + return f"{exported_name(self.fullname)}Data" + def get_method_and_class( self, name: str, *, prefer_method: bool = False ) -> tuple[FuncIR, ClassIR] | None: @@ -352,6 +366,8 @@ def serialize(self) -> JsonDict: "is_generated": self.is_generated, "is_augmented": self.is_augmented, "is_final_class": self.is_final_class, + "is_immutable": self.is_immutable, + "is_value_type": self.is_value_type, "inherits_python": self.inherits_python, "has_dict": self.has_dict, "allow_interpreted_subclasses": self.allow_interpreted_subclasses, @@ -408,6 +424,8 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir.is_ext_class = data["is_ext_class"] ir.is_augmented = data["is_augmented"] ir.is_final_class = data["is_final_class"] + ir.is_immutable = data["is_immutable"] + ir.is_value_type = data["is_value_type"] ir.inherits_python = data["inherits_python"] ir.has_dict = data["has_dict"] ir.allow_interpreted_subclasses = data["allow_interpreted_subclasses"] diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 53e3cee74e56..031cd058a0cb 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -98,6 +98,10 @@ def deserialize_type(data: JsonDict | str, ctx: DeserMaps) -> RType: return RVoid() else: assert False, f"Can't find class {data}" + elif data[".class"] == "RInstanceValue": + class_ = deserialize_type(data["class"], ctx) + assert isinstance(class_, RInstance) + return RInstanceValue(class_.class_ir) elif data[".class"] == "RTuple": return RTuple.deserialize(data, ctx) elif data[".class"] == "RUnion": @@ -116,6 +120,10 @@ def visit_rprimitive(self, typ: RPrimitive) -> T: def visit_rinstance(self, typ: RInstance) -> T: raise NotImplementedError + @abstractmethod + def visit_rinstance_value(self, typ: RInstanceValue) -> T: + raise NotImplementedError + @abstractmethod def visit_runion(self, typ: RUnion) -> T: raise NotImplementedError @@ -574,6 +582,9 @@ class TupleNameVisitor(RTypeVisitor[str]): def visit_rinstance(self, t: RInstance) -> str: return "O" + def visit_rinstance_value(self, typ: RInstanceValue) -> T: + return "O" + def visit_runion(self, t: RUnion) -> str: return "O" @@ -798,7 +809,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RStruct: class RInstance(RType): """Instance of user-defined class (compiled to C extension class). - The runtime representation is 'PyObject *', and these are always + The runtime representation is typically 'PyObject *', and these are boxed and thus reference-counted. These support fast method calls and fast attribute access using @@ -851,6 +862,40 @@ def serialize(self) -> str: return self.name +class RInstanceValue(RInstance): + """ + Fixed-length unboxed Value Type. + + These are used to represent unboxed values of RInstance which match + the Value Type constraints. + """ + + is_unboxed = True + is_refcounted = False + + def __init__(self, class_ir: ClassIR) -> None: + super().__init__(class_ir) + self._ctype = self.class_ir.struct_name2() + + def accept(self, visitor: RTypeVisitor[T]) -> T: + return visitor.visit_rinstance_value(self) + + def attr_type(self, name: str) -> RType: + return self.class_ir.attr_type(name) + + def __repr__(self) -> str: + return "" % self.name + + def __eq__(self, other: object) -> bool: + return isinstance(other, RInstanceValue) and other.name == self.name + + def __hash__(self) -> int: + return hash(self.name) + + def serialize(self) -> JsonDict: + return {".class": "RInstanceValue", "class": super().serialize()} + + class RUnion(RType): """union[x, ..., y]""" diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index a9e1ce471953..e84c91026191 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1181,7 +1181,7 @@ def enter_method( self.class_ir_stack.append(class_ir) self.ret_types[-1] = ret_type if self_type is None: - self_type = RInstance(class_ir) + self_type = class_ir.rtype self.add_argument(SELF_NAME, self_type) try: yield @@ -1239,7 +1239,7 @@ def add_self_to_env(self, cls: ClassIR) -> AssignmentTargetRegister: This is only useful if using enter() instead of enter_method(). """ - return self.add_local_reg(Var(SELF_NAME), RInstance(cls), is_arg=True) + return self.add_local_reg(Var(SELF_NAME), cls.rtype, is_arg=True) def add_target(self, symbol: SymbolNode, target: SymbolTarget) -> SymbolTarget: self.symtables[-1][symbol] = target diff --git a/mypyc/irbuild/callable_class.py b/mypyc/irbuild/callable_class.py index 599dbb81f767..62d8bc3cb917 100644 --- a/mypyc/irbuild/callable_class.py +++ b/mypyc/irbuild/callable_class.py @@ -10,7 +10,7 @@ from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import BasicBlock, Call, Register, Return, SetAttr, Value -from mypyc.ir.rtypes import RInstance, object_rprimitive +from mypyc.ir.rtypes import object_rprimitive from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.context import FuncInfo, ImplicitClass from mypyc.primitives.misc_ops import method_new_op @@ -67,7 +67,7 @@ class for the nested function. # If the enclosing class doesn't contain nested (which will happen if # this is a toplevel lambda), don't set up an environment. if builder.fn_infos[-2].contains_nested: - callable_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class) + callable_class_ir.attributes[ENV_ATTR_NAME] = builder.fn_infos[-2].env_class.rtype callable_class_ir.mro = [callable_class_ir] builder.fn_info.callable_class = ImplicitClass(callable_class_ir) builder.classes.append(callable_class_ir) diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index aa223fe20176..1a4615e15095 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -17,11 +17,12 @@ def g() -> int: from __future__ import annotations -from mypy.nodes import Argument, FuncDef, SymbolNode, Var -from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME, bitmap_name +from mypy.nodes import Argument, FuncDef, SymbolNode +from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME from mypyc.ir.class_ir import ClassIR +from mypyc.ir.func_ir import FuncSignature from mypyc.ir.ops import Call, GetAttr, SetAttr, Value -from mypyc.ir.rtypes import RInstance, bitmap_rprimitive, object_rprimitive +from mypyc.ir.rtypes import RInstance, object_rprimitive from mypyc.irbuild.builder import IRBuilder, SymbolTarget from mypyc.irbuild.context import FuncInfo, GeneratorClass, ImplicitClass from mypyc.irbuild.targets import AssignmentTargetAttr @@ -45,18 +46,18 @@ class is generated, the function environment has not yet been env_class = ClassIR( f"{builder.fn_info.namespaced_name()}_env", builder.module_name, is_generated=True ) - env_class.attributes[SELF_NAME] = RInstance(env_class) + env_class.attributes[SELF_NAME] = env_class.rtype if builder.fn_info.is_nested: # If the function is nested, its environment class must contain an environment # attribute pointing to its encapsulating functions' environment class. - env_class.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_infos[-2].env_class) + env_class.attributes[ENV_ATTR_NAME] = builder.fn_infos[-2].env_class.rtype env_class.mro = [env_class] builder.fn_info.env_class = env_class builder.classes.append(env_class) return env_class -def finalize_env_class(builder: IRBuilder) -> None: +def finalize_env_class(builder: IRBuilder, sig: FuncSignature) -> None: """Generate, instantiate, and set up the environment of an environment class.""" instantiate_env_class(builder) @@ -64,9 +65,9 @@ def finalize_env_class(builder: IRBuilder) -> None: # that were previously added to the environment with references to the function's # environment class. if builder.fn_info.is_nested: - add_args_to_env(builder, local=False, base=builder.fn_info.callable_class) + add_args_to_env(builder, sig, local=False, base=builder.fn_info.callable_class) else: - add_args_to_env(builder, local=False, base=builder.fn_info) + add_args_to_env(builder, sig, local=False, base=builder.fn_info) def instantiate_env_class(builder: IRBuilder) -> Value: @@ -91,7 +92,7 @@ def instantiate_env_class(builder: IRBuilder) -> Value: return curr_env_reg -def load_env_registers(builder: IRBuilder) -> None: +def load_env_registers(builder: IRBuilder, sig: FuncSignature) -> None: """Load the registers for the current FuncItem being visited. Adds the arguments of the FuncItem to the environment. If the @@ -99,7 +100,7 @@ def load_env_registers(builder: IRBuilder) -> None: loads all of the outer environments of the FuncItem into registers so that they can be used when accessing free variables. """ - add_args_to_env(builder, local=True) + add_args_to_env(builder, sig, local=True) fn_info = builder.fn_info fitem = fn_info.fitem @@ -170,25 +171,21 @@ def num_bitmap_args(builder: IRBuilder, args: list[Argument]) -> int: def add_args_to_env( builder: IRBuilder, + sig: FuncSignature, local: bool = True, base: FuncInfo | ImplicitClass | None = None, reassign: bool = True, ) -> None: fn_info = builder.fn_info - args = fn_info.fitem.arguments - nb = num_bitmap_args(builder, args) if local: - for arg in args: - rtype = builder.type_to_rtype(arg.variable.type) - builder.add_local_reg(arg.variable, rtype, is_arg=True) - for i in reversed(range(nb)): - builder.add_local_reg(Var(bitmap_name(i)), bitmap_rprimitive, is_arg=True) + for sarg, farg in zip(sig.args, fn_info.fitem.arguments): + assert sarg.name == farg.variable.name + builder.add_local_reg(farg.variable, sarg.type, is_arg=True) else: - for arg in args: - if is_free_variable(builder, arg.variable) or fn_info.is_generator: - rtype = builder.type_to_rtype(arg.variable.type) + for sarg, farg in zip(sig.args, fn_info.fitem.arguments): + if is_free_variable(builder, farg.variable) or fn_info.is_generator: assert base is not None, "base cannot be None for adding nonlocal args" - builder.add_var_to_env_class(arg.variable, rtype, base, reassign=reassign) + builder.add_var_to_env_class(farg.variable, sarg.type, base, reassign=reassign) def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: ImplicitClass) -> None: diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index c985e88b0e0c..c1c51bfd21fd 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -54,13 +54,7 @@ Unreachable, Value, ) -from mypyc.ir.rtypes import ( - RInstance, - bool_rprimitive, - dict_rprimitive, - int_rprimitive, - object_rprimitive, -) +from mypyc.ir.rtypes import bool_rprimitive, dict_rprimitive, int_rprimitive, object_rprimitive from mypyc.irbuild.builder import IRBuilder, SymbolTarget, gen_arg_defaults from mypyc.irbuild.callable_class import ( add_call_to_callable_class, @@ -260,7 +254,7 @@ def c() -> None: if builder.fn_info.is_generator: # Do a first-pass and generate a function that just returns a generator object. - gen_generator_func(builder) + gen_generator_func(builder, sig) args, _, blocks, ret_type, fn_info = builder.leave() func_ir, func_reg = gen_func_ir( builder, args, blocks, sig, fn_info, cdef, is_singledispatch @@ -268,7 +262,7 @@ def c() -> None: # Re-enter the FuncItem and visit the body of the function this time. builder.enter(fn_info) - setup_env_for_generator_class(builder) + setup_env_for_generator_class(builder, sig) load_outer_envs(builder, builder.fn_info.generator_class) top_level = builder.top_level_fn_info() if ( @@ -281,11 +275,11 @@ def c() -> None: create_switch_for_generator_class(builder) add_raise_exception_blocks_to_generator_class(builder, fitem.line) else: - load_env_registers(builder) + load_env_registers(builder, sig) gen_arg_defaults(builder) if builder.fn_info.contains_nested and not builder.fn_info.is_generator: - finalize_env_class(builder) + finalize_env_class(builder, sig) builder.ret_types[-1] = sig.ret_type @@ -655,7 +649,7 @@ def f(builder: IRBuilder, x: object) -> int: ... rt_args = list(base_sig.args) if target.decl.kind == FUNC_NORMAL: - rt_args[0] = RuntimeArg(base_sig.args[0].name, RInstance(cls)) + rt_args[0] = RuntimeArg(base_sig.args[0].name, cls.rtype) arg_info = get_args(builder, rt_args, line) args, arg_kinds, arg_names = arg_info.args, arg_info.arg_kinds, arg_info.arg_names @@ -757,7 +751,7 @@ def gen_glue_property( """ builder.enter() - rt_arg = RuntimeArg(SELF_NAME, RInstance(cls)) + rt_arg = RuntimeArg(SELF_NAME, cls.rtype) self_target = builder.add_self_to_env(cls) arg = builder.read(self_target, line) builder.ret_types[-1] = sig.ret_type diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 92f9abff467c..101b67acdeab 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -30,7 +30,7 @@ Unreachable, Value, ) -from mypyc.ir.rtypes import RInstance, int_rprimitive, object_rprimitive +from mypyc.ir.rtypes import int_rprimitive, object_rprimitive from mypyc.irbuild.builder import IRBuilder, gen_arg_defaults from mypyc.irbuild.context import FuncInfo, GeneratorClass from mypyc.irbuild.env_class import ( @@ -49,11 +49,11 @@ ) -def gen_generator_func(builder: IRBuilder) -> None: +def gen_generator_func(builder: IRBuilder, sig: FuncSignature) -> None: setup_generator_class(builder) - load_env_registers(builder) + load_env_registers(builder, sig) gen_arg_defaults(builder) - finalize_env_class(builder) + finalize_env_class(builder, sig) builder.add(Return(instantiate_generator_class(builder))) @@ -84,7 +84,7 @@ def setup_generator_class(builder: IRBuilder) -> ClassIR: name = f"{builder.fn_info.namespaced_name()}_gen" generator_class_ir = ClassIR(name, builder.module_name, is_generated=True) - generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class) + generator_class_ir.attributes[ENV_ATTR_NAME] = builder.fn_info.env_class.rtype generator_class_ir.mro = [generator_class_ir] builder.classes.append(generator_class_ir) @@ -312,7 +312,7 @@ def add_await_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None: builder.add(Return(builder.self())) -def setup_env_for_generator_class(builder: IRBuilder) -> None: +def setup_env_for_generator_class(builder: IRBuilder, sig: FuncSignature) -> None: """Populates the environment for a generator class.""" fitem = builder.fn_info.fitem cls = builder.fn_info.generator_class @@ -340,7 +340,7 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None: # Add arguments from the original generator function to the # environment of the generator class. - add_args_to_env(builder, local=False, base=cls, reassign=False) + add_args_to_env(builder, sig, local=False, base=cls, reassign=False) # Set the next label register for the generator class. cls.next_label_reg = builder.read(cls.next_label_target, fitem.line) diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 90ce0e16c741..331f172b3174 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -25,7 +25,6 @@ from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncSignature, RuntimeArg from mypyc.ir.rtypes import ( - RInstance, RTuple, RType, RUnion, @@ -93,7 +92,7 @@ def type_to_rtype(self, typ: Type | None) -> RType: elif typ.type.fullname == "builtins.range": return range_rprimitive elif typ.type in self.type_to_ir: - inst = RInstance(self.type_to_ir[typ.type]) + inst = self.type_to_ir[typ.type].rtype # Treat protocols as Union[protocol, object], so that we can do fast # method calls in the cases where the protocol is explicitly inherited from # and fall back to generic operations when it isn't. diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 40a40b79df49..d9276a0d9e8e 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -50,14 +50,16 @@ RuntimeArg, ) from mypyc.ir.ops import DeserMaps -from mypyc.ir.rtypes import RInstance, RType, dict_rprimitive, none_rprimitive, tuple_rprimitive +from mypyc.ir.rtypes import RType, dict_rprimitive, none_rprimitive, tuple_rprimitive from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.util import ( get_func_def, get_mypyc_attrs, is_dataclass, is_extension_class, + is_immutable, is_trait, + is_value_type, ) from mypyc.options import CompilerOptions from mypyc.sametype import is_same_type @@ -86,6 +88,8 @@ def build_type_map( is_trait(cdef), is_abstract=cdef.info.is_abstract, is_final_class=cdef.info.is_final, + is_immutable=is_immutable(cdef), + is_value_type=is_value_type(cdef), ) class_ir.is_ext_class = is_extension_class(cdef) if class_ir.is_ext_class: @@ -414,7 +418,7 @@ def add_property_methods_for_attribute_if_needed( def add_getter_declaration( ir: ClassIR, attr_name: str, attr_rtype: RType, module_name: str ) -> None: - self_arg = RuntimeArg("self", RInstance(ir), pos_only=True) + self_arg = RuntimeArg("self", ir.rtype, pos_only=True) sig = FuncSignature([self_arg], attr_rtype) decl = FuncDecl(attr_name, ir.name, module_name, sig, FUNC_NORMAL) decl.is_prop_getter = True @@ -426,7 +430,7 @@ def add_getter_declaration( def add_setter_declaration( ir: ClassIR, attr_name: str, attr_rtype: RType, module_name: str ) -> None: - self_arg = RuntimeArg("self", RInstance(ir), pos_only=True) + self_arg = RuntimeArg("self", ir.rtype, pos_only=True) value_arg = RuntimeArg("value", attr_rtype, pos_only=True) sig = FuncSignature([self_arg, value_arg], none_rprimitive) setter_name = PROPSET_PREFIX + attr_name @@ -461,7 +465,7 @@ def prepare_init_method(cdef: ClassDef, ir: ClassIR, module_name: str, mapper: M ) last_arg = len(init_sig.args) - init_sig.num_bitmap_args - ctor_sig = FuncSignature(init_sig.args[1:last_arg], RInstance(ir)) + ctor_sig = FuncSignature(init_sig.args[1:last_arg], ir.rtype) ir.ctor = FuncDecl(cdef.name, None, module_name, ctor_sig) mapper.func_to_decl[cdef.info] = ir.ctor diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index e27e509ad7fa..9bb146398076 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -81,6 +81,26 @@ def dataclass_type(cdef: ClassDef) -> str | None: return None +def is_immutable(cdef: ClassDef) -> bool: + """Check if a class is immutable by checking if all its variables are marked as Final.""" + for v in cdef.info.names.values(): + if ( + isinstance(v.node, Var) + and not v.node.is_classvar + and v.node.name not in ("__slots__", "__deletable__") + ): + if not v.node.is_final or v.node.is_settable_property: + return False + + return True + + +def is_value_type(cdef: ClassDef) -> bool: + val = get_mypyc_attrs(cdef).get("value_type", False) + assert isinstance(val, bool) + return val is True + + def get_mypyc_attr_literal(e: Expression) -> Any: """Convert an expression from a mypyc_attr decorator to a value. diff --git a/mypyc/rt_subtype.py b/mypyc/rt_subtype.py index 004e56ed75bc..95756af9e387 100644 --- a/mypyc/rt_subtype.py +++ b/mypyc/rt_subtype.py @@ -18,6 +18,7 @@ from mypyc.ir.rtypes import ( RArray, RInstance, + RInstanceValue, RPrimitive, RStruct, RTuple, @@ -50,6 +51,9 @@ def __init__(self, right: RType) -> None: def visit_rinstance(self, left: RInstance) -> bool: return is_subtype(left, self.right) + def visit_rinstance_value(self, left: RInstanceValue) -> bool: + return is_subtype(left, self.right) + def visit_runion(self, left: RUnion) -> bool: return not self.right.is_unboxed and is_subtype(left, self.right) diff --git a/mypyc/sametype.py b/mypyc/sametype.py index 1b811d4e9041..7acc222558a0 100644 --- a/mypyc/sametype.py +++ b/mypyc/sametype.py @@ -6,6 +6,7 @@ from mypyc.ir.rtypes import ( RArray, RInstance, + RInstanceValue, RPrimitive, RStruct, RTuple, @@ -50,6 +51,9 @@ def __init__(self, right: RType) -> None: def visit_rinstance(self, left: RInstance) -> bool: return isinstance(self.right, RInstance) and left.name == self.right.name + def visit_rinstance_value(self, left: RInstanceValue) -> bool: + return isinstance(self.right, RInstanceValue) and left.name == self.right.name + def visit_runion(self, left: RUnion) -> bool: if isinstance(self.right, RUnion): items = list(self.right.items) diff --git a/mypyc/subtype.py b/mypyc/subtype.py index 726a48d7a01d..5802aff6c1aa 100644 --- a/mypyc/subtype.py +++ b/mypyc/subtype.py @@ -5,6 +5,7 @@ from mypyc.ir.rtypes import ( RArray, RInstance, + RInstanceValue, RPrimitive, RStruct, RTuple, @@ -50,6 +51,9 @@ def __init__(self, right: RType) -> None: def visit_rinstance(self, left: RInstance) -> bool: return isinstance(self.right, RInstance) and self.right.class_ir in left.class_ir.mro + def visit_rinstance_value(self, left: RInstanceValue) -> bool: + return isinstance(self.right, RInstanceValue) and self.right.class_ir in left.class_ir.mro + def visit_runion(self, left: RUnion) -> bool: return all(is_subtype(item, self.right) for item in left.items) diff --git a/mypyc/transform/value_type_init.py b/mypyc/transform/value_type_init.py new file mode 100644 index 000000000000..7dd953be1a63 --- /dev/null +++ b/mypyc/transform/value_type_init.py @@ -0,0 +1,35 @@ +"""Transformation for changing the initialization method of a value type. + +This transformation changes the type of the self parameter of the __init__ method +of a value type to be a reference to the value type. This is necessary because +the __init__ method of a value type is called with the purpose of initializing +the attributes on the storage but the self parameter being passed by value +won't make the changes on the target object. +""" + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.rtypes import RInstance, RInstanceValue +from mypyc.options import CompilerOptions + + +def patch_value_type_init_methods(ir: FuncIR, options: CompilerOptions) -> None: + if ir.name != "__init__" or not ir.args or not ir.blocks: + return + + if not isinstance(ir.args[0].type, RInstanceValue): + return + + self_rtype: RInstanceValue = ir.args[0].type + cl = self_rtype.class_ir + + # ensure we are processing the __init__ method of a value type + if not cl.is_value_type or cl.get_method("__init__") is not ir: + return + + # patch the type of the self parameter to be a reference to the value type + ref_type = RInstance(cl) + # the refcounted flag is set to False because we only need to initialize the + # attributes of the value type, but it is not expected to be refcounted + ref_type.is_refcounted = False + ir.args[0].type = ref_type + ir.arg_regs[0].type = ref_type From 51379e2021b50ff35519de804e428b51917eb190 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Wed, 16 Oct 2024 23:20:07 -0500 Subject: [PATCH 02/21] Value types tests working --- mypyc/codegen/emit.py | 38 +++--- mypyc/codegen/emitclass.py | 37 ++++-- mypyc/codegen/emitfunc.py | 61 ++++----- mypyc/ir/class_ir.py | 24 ++-- mypyc/ir/rtypes.py | 21 ++-- mypyc/irbuild/classdef.py | 7 +- mypyc/irbuild/env_class.py | 23 ++-- mypyc/irbuild/prepare.py | 11 +- mypyc/irbuild/util.py | 19 +-- mypyc/test-data/run-valuetype.test | 194 +++++++++++++++++++++++++++++ mypyc/test/test_run.py | 1 + 11 files changed, 332 insertions(+), 104 deletions(-) create mode 100644 mypyc/test-data/run-valuetype.test diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 604bd1c76a93..ae9c4d326cbf 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -361,14 +361,22 @@ def bitmap_field(self, index: int) -> str: return "bitmap" return f"bitmap{n + 1}" - def attr_bitmap_expr(self, obj: str, cl: ClassIR, index: int) -> str: - """Return reference to the attribute definedness bitmap.""" - cast = f"({cl.struct_name(self.names)} *)" + def attr_bitmap_expr(self, obj: str, cl: RInstance, index: int) -> str: + """ + Return reference to the attribute definedness bitmap. + + If a_ref is True, assume object's type is a reference. + Otherwise, the object type is indicated by the class IR. + """ attr = self.bitmap_field(index) - return f"({cast}{obj})->{attr}" + if cl.is_unboxed: + return f"{obj}.{attr}" + else: + cast = f"({cl.struct_name(self.names)} *)" + return f"({cast}{obj})->{attr}" def emit_attr_bitmap_set( - self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str + self, value: str, obj: str, rtype: RType, cl: RInstance, attr: str ) -> None: """Mark an attribute as defined in the attribute bitmap. @@ -377,7 +385,7 @@ def emit_attr_bitmap_set( """ self._emit_attr_bitmap_update(value, obj, rtype, cl, attr, clear=False) - def emit_attr_bitmap_clear(self, obj: str, rtype: RType, cl: ClassIR, attr: str) -> None: + def emit_attr_bitmap_clear(self, obj: str, rtype: RType, cl: RInstance, attr: str) -> None: """Mark an attribute as undefined in the attribute bitmap. Unlike emit_attr_bitmap_set, clear unconditionally. @@ -385,12 +393,12 @@ def emit_attr_bitmap_clear(self, obj: str, rtype: RType, cl: ClassIR, attr: str) self._emit_attr_bitmap_update("", obj, rtype, cl, attr, clear=True) def _emit_attr_bitmap_update( - self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str, clear: bool + self, value: str, obj: str, rtype: RType, cl: RInstance, attr: str, clear: bool ) -> None: if value: check = self.error_value_check(rtype, value, "==") self.emit_line(f"if (unlikely({check})) {{") - index = cl.bitmap_attrs.index(attr) + index = cl.class_ir.bitmap_attrs.index(attr) mask = 1 << (index & (BITMAP_BITS - 1)) bitmap = self.attr_bitmap_expr(obj, cl, index) if clear: @@ -410,7 +418,7 @@ def emit_undefined_attr_check( compare: str, obj: str, attr: str, - cl: ClassIR, + cl: RInstance, *, unlikely: bool = False, ) -> None: @@ -418,11 +426,11 @@ def emit_undefined_attr_check( if unlikely: check = f"unlikely({check})" if rtype.error_overlap: - index = cl.bitmap_attrs.index(attr) + index = cl.class_ir.bitmap_attrs.index(attr) + attr_expr = self.attr_bitmap_expr(obj, cl, index) bit = 1 << (index & (BITMAP_BITS - 1)) - attr = self.bitmap_field(index) - obj_expr = f"({cl.struct_name(self.names)} *){obj}" - check = f"{check} && !(({obj_expr})->{attr} & {bit})" + check = f"{check} && !({attr_expr} & {bit})" + self.emit_line(f"if ({check}) {{") def error_value_check(self, rtype: RType, value: str, compare: str) -> str: @@ -500,6 +508,8 @@ def declare_tuple_struct(self, tuple_type: RTuple) -> None: # XXX other types might eventually need similar behavior if isinstance(typ, RTuple): dependencies.add(typ.struct_name) + if isinstance(typ, RInstanceValue): + dependencies.add(typ.struct_name(self.names)) self.context.declarations[tuple_type.struct_name] = HeaderDeclaration( self.tuple_c_declaration(tuple_type), dependencies=dependencies, is_type=True @@ -1075,7 +1085,7 @@ def emit_box( ) self.emit_line(f"if (unlikely({temp_dest} == NULL))") self.emit_line(" CPyError_OutOfMemory();") - for attr, attr_type in cl.attributes.items(): + for attr, attr_type in cl.all_attributes().items(): attr_name = self.attr(attr) self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box") diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 6327e7f0311b..68d3de0737f4 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -21,7 +21,7 @@ from mypyc.common import BITMAP_BITS, BITMAP_TYPE, NATIVE_PREFIX, PREFIX, REG_PREFIX from mypyc.ir.class_ir import ClassIR, VTableEntries from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR -from mypyc.ir.rtypes import RTuple, RType, object_rprimitive +from mypyc.ir.rtypes import RTuple, RType, object_rprimitive, RInstanceValue, RInstance from mypyc.namegen import NameGenerator from mypyc.sametype import is_same_type @@ -294,9 +294,14 @@ def emit_line() -> None: # Declare setup method that allocates and initializes an object. type is the # type of the class being initialized, which could be another class if there # is an interpreted subclass. - emitter.context.declarations[setup_name] = HeaderDeclaration( - f"PyObject *{setup_name}(PyTypeObject *type);", needs_export=True - ) + if cl.is_value_type: + # Value types will need this method be exported because it will be required + # when boxing the value type instance. + emitter.context.declarations[setup_name] = HeaderDeclaration( + f"PyObject *{setup_name}(PyTypeObject *type);", needs_export=True + ) + else: + emitter.emit_line(f"static PyObject *{setup_name}(PyTypeObject *type);") assert cl.ctor is not None emitter.emit_line(native_function_header(cl.ctor, emitter) + ";") @@ -949,11 +954,13 @@ def generate_getter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N always_defined = cl.is_always_defined(attr) and not rtype.is_refcounted if not always_defined: - emitter.emit_undefined_attr_check(rtype, attr_expr, "==", "self", attr, cl, unlikely=True) + clt = RInstance(cl) + emitter.emit_undefined_attr_check(rtype, attr_expr, "==", "self", attr, clt, unlikely=True) emitter.emit_line("PyErr_SetString(PyExc_AttributeError,") emitter.emit_line(f' "attribute {repr(attr)} of {repr(cl.name)} undefined");') emitter.emit_line("return NULL;") emitter.emit_line("}") + emitter.emit_inc_ref(f"self->{attr_field}", rtype) emitter.emit_box(f"self->{attr_field}", "retval", rtype, declare_dest=True) emitter.emit_line("return retval;") @@ -988,7 +995,8 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N if rtype.is_refcounted: attr_expr = f"self->{attr_field}" if not always_defined: - emitter.emit_undefined_attr_check(rtype, attr_expr, "!=", "self", attr, cl) + clt = RInstance(cl) + emitter.emit_undefined_attr_check(rtype, attr_expr, "!=", "self", attr, clt) emitter.emit_dec_ref(f"self->{attr_field}", rtype) if not always_defined: emitter.emit_line("}") @@ -1006,13 +1014,13 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_inc_ref("tmp", rtype) emitter.emit_line(f"self->{attr_field} = tmp;") if rtype.error_overlap and not always_defined: - emitter.emit_attr_bitmap_set("tmp", "self", rtype, cl, attr) + emitter.emit_attr_bitmap_set("tmp", "self", rtype, RInstance(cl), attr) if deletable: emitter.emit_line("} else") emitter.emit_line(f" self->{attr_field} = {emitter.c_undefined_value(rtype)};") if rtype.error_overlap: - emitter.emit_attr_bitmap_clear("self", rtype, cl, attr) + emitter.emit_attr_bitmap_clear("self", rtype, RInstance(cl), attr) emitter.emit_line("return 0;") emitter.emit_line("}") @@ -1027,19 +1035,22 @@ def generate_readonly_getter( ) ) emitter.emit_line("{") + + arg0 = func_ir.args[0].type + obj = "*self" if isinstance(arg0, RInstanceValue) else "(PyObject *)self" + if rtype.is_unboxed: emitter.emit_line( - "{}retval = {}{}((PyObject *) self);".format( - emitter.ctype_spaced(rtype), NATIVE_PREFIX, func_ir.cname(emitter.names) + "{}retval = {}{}({});".format( + emitter.ctype_spaced(rtype), NATIVE_PREFIX, func_ir.cname(emitter.names), obj ) ) emitter.emit_error_check("retval", rtype, "return NULL;") emitter.emit_box("retval", "retbox", rtype, declare_dest=True) emitter.emit_line("return retbox;") else: - emitter.emit_line( - f"return {NATIVE_PREFIX}{func_ir.cname(emitter.names)}((PyObject *) self);" - ) + emitter.emit_line(f"return {NATIVE_PREFIX}{func_ir.cname(emitter.names)}({obj});") + emitter.emit_line("}") diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 246eddea21ef..3c3537ba6331 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -15,7 +15,7 @@ TYPE_VAR_PREFIX, ) from mypyc.ir.class_ir import ClassIR -from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values +from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values, FUNC_NORMAL from mypyc.ir.ops import ( ERR_FALSE, NAMESPACE_MODULE, @@ -71,14 +71,12 @@ ) from mypyc.ir.pprint import generate_names_for_ir from mypyc.ir.rtypes import ( - PyObject, RArray, RInstance, RInstanceValue, RStruct, RTuple, RType, - c_pyssize_t_rprimitive, is_int32_rprimitive, is_int64_rprimitive, is_int_rprimitive, @@ -87,18 +85,6 @@ ) -def struct_type(class_ir: ClassIR, emitter: Emitter) -> RStruct: - """Return the struct type for this instance type.""" - python_fields: list[tuple[str, RType]] = [ - ("head", PyObject), - ("vtable", c_pyssize_t_rprimitive), - ] - class_fields = list(class_ir.attributes.items()) - attr_names = [emitter.attr(name) for name, _ in python_fields + class_fields] - attr_types = [rtype for _, rtype in python_fields + class_fields] - return RStruct(class_ir.struct_name(emitter.names), attr_names, attr_types) - - def native_function_type(fn: FuncIR, emitter: Emitter) -> str: args = ", ".join(emitter.ctype(arg.type) for arg in fn.args) or "void" ret = emitter.ctype(fn.ret_type) @@ -295,7 +281,7 @@ def visit_assign(self, op: Assign) -> None: # clang whines about self assignment (which we might generate # for some casts), so don't emit it. if dest != src: - # We sometimes assign from an integer prepresentation of a pointer + # We sometimes assign from an integer representation of a pointer # to a real pointer, and C compilers insist on a cast. if op.src.type.is_unboxed and not op.dest.type.is_unboxed: src = f"(void *){src}" @@ -406,20 +392,12 @@ def visit_get_attr(self, op: GetAttr) -> None: attr_expr = self.get_attr_expr(obj, op, decl_cl) always_defined = cl.is_always_defined(op.attr) # This steals the reference to src, so we don't need to increment the arg - if isinstance(attr_rtype, RInstance) and attr_rtype.class_ir.is_value_type: - # special case for value types, it is unboxed in the struct - struct_name = attr_rtype.class_ir.struct_name(self.names) - temp = self.emitter.temp_name() - self.emitter.emit_line(f"{struct_name} {temp} = {attr_expr};") - self.emitter.emit_line(f"{dest} = (PyObject *)&{temp};") - always_defined = True - else: - self.emitter.emit_line(f"{dest} = {attr_expr};") + self.emitter.emit_line(f"{dest} = {attr_expr};") merged_branch = None if not always_defined: self.emitter.emit_undefined_attr_check( - attr_rtype, dest, "==", obj, op.attr, cl, unlikely=True + attr_rtype, dest, "==", obj, op.attr, rtype, unlikely=True ) branch = self.next_branch() if branch is not None: @@ -466,7 +444,8 @@ def visit_set_attr(self, op: SetAttr) -> None: dest = self.reg(op) obj = self.reg(op.obj) src = self.reg(op.src) - rtype = op.class_type + rtype = op.obj.type + assert isinstance(rtype, RInstance) cl = rtype.class_ir attr_rtype, decl_cl = cl.attr_details(op.attr) if cl.get_method(op.attr): @@ -501,7 +480,7 @@ def visit_set_attr(self, op: SetAttr) -> None: always_defined = cl.is_always_defined(op.attr) if not always_defined: self.emitter.emit_undefined_attr_check( - attr_rtype, attr_expr, "!=", obj, op.attr, cl + attr_rtype, attr_expr, "!=", obj, op.attr, rtype ) self.emitter.emit_dec_ref(attr_expr, attr_rtype) if not always_defined: @@ -509,15 +488,10 @@ def visit_set_attr(self, op: SetAttr) -> None: elif attr_rtype.error_overlap and not cl.is_always_defined(op.attr): # If there is overlap with the error value, update bitmap to mark # attribute as defined. - self.emitter.emit_attr_bitmap_set(src, obj, attr_rtype, cl, op.attr) + self.emitter.emit_attr_bitmap_set(src, obj, attr_rtype, rtype, op.attr) # This steals the reference to src, so we don't need to increment the arg - if isinstance(attr_rtype, RInstance) and attr_rtype.class_ir.is_value_type: - # special case for value types, it is unboxed in the struct - struct_name = attr_rtype.class_ir.struct_name(self.names) - self.emitter.emit_line(f"{attr_expr} = *({struct_name} *)({src});") - else: - self.emitter.emit_line(f"{attr_expr} = {src};") + self.emitter.emit_line(f"{attr_expr} = {src};") if op.error_kind == ERR_FALSE: self.emitter.emit_line(f"{dest} = 1;") @@ -589,6 +563,20 @@ def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Va if method.decl.kind == FUNC_STATICMETHOD else [f"(PyObject *)Py_TYPE({obj})"] if method.decl.kind == FUNC_CLASSMETHOD else [obj] ) + need_box_obj = ( + method.decl.kind == FUNC_NORMAL + and rtype.is_unboxed + and not method.args[0].type.is_unboxed + ) + obj_boxed = "" + if need_box_obj: + # for cases where obj.method(...) is called and obj is unboxed, but method + # expects a boxed due inheritance or trait. e.g. obj is a value type + # but method comes from a parent which not + obj_boxed = self.temp_name() + self.emitter.emit_box(obj, obj_boxed, rtype, declare_dest=True) + obj_args = [obj_boxed] + args = ", ".join(obj_args + [self.reg(arg) for arg in op_args]) mtype = native_function_type(method, self.emitter) version = "_TRAIT" if rtype.class_ir.is_trait else "" @@ -613,6 +601,9 @@ def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Va ) ) + if need_box_obj: + self.emitter.emit_dec_ref(obj_boxed, RInstance(rtype.class_ir)) + def visit_inc_ref(self, op: IncRef) -> None: src = self.reg(op.src) self.emit_inc_ref(src, op.src.type) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index a27e51188b61..2181330189cb 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -94,7 +94,6 @@ def __init__( is_abstract: bool = False, is_ext_class: bool = True, is_final_class: bool = False, - is_immutable: bool = False, is_value_type: bool = False, ) -> None: self.name = name @@ -104,7 +103,8 @@ def __init__( self.is_abstract = is_abstract self.is_ext_class = is_ext_class self.is_final_class = is_final_class - self.is_immutable = is_immutable + # A value type is a class that can be passed by value instead of by reference. + self.is_value_type = is_value_type # An augmented class has additional methods separate from what mypyc generates. # Right now the only one is dataclasses. self.is_augmented = False @@ -117,8 +117,6 @@ def __init__( # Does this class need getseters to be generated for its attributes? (getseters are also # added if is_generated is False) self.needs_getseters = False - # A value type is a class that can be passed by value instead of by reference. - self.is_value_type = self.is_ext_class and self.is_immutable and not self.has_dict and is_value_type # Is this class declared as serializable (supports copy.copy # and pickle) using @mypyc_attr(serializable=True)? # @@ -136,8 +134,6 @@ def __init__( # of the object for that class. We currently only support this # in a few ad-hoc cases. self.builtin_base: str | None = None - # The RType for instances of this class - self.rtype = RInstanceValue(self) if self.is_value_type else RInstance(self) # Default empty constructor self.ctor = FuncDecl(name, None, module_name, FuncSignature([], self.rtype)) # Attributes defined in the class (not inherited) @@ -209,11 +205,14 @@ def __repr__(self) -> str: "name={self.name}, module_name={self.module_name}, " "is_trait={self.is_trait}, is_generated={self.is_generated}, " "is_abstract={self.is_abstract}, is_ext_class={self.is_ext_class}, " - "is_final_class={self.is_final_class}, is_immutable={self.is_immutable}, " - "is_value_type={self.is_value_type}" + "is_final_class={self.is_final_class}, is_value_type={self.is_value_type}" ")".format(self=self) ) + @property + def rtype(self) -> RType: + return RInstanceValue(self) if self.is_value_type else RInstance(self) + @property def fullname(self) -> str: return f"{self.module_name}.{self.name}" @@ -229,6 +228,13 @@ def vtable_entry(self, name: str) -> int: assert name in self.vtable, f"{self.name!r} has no attribute {name!r}" return self.vtable[name] + def all_attributes(self) -> dict[str, RType]: + """Return all attributes, including inherited ones. Not including properties.""" + result = {} + for ir in reversed(self.mro): + result.update(ir.attributes) + return result + def attr_details(self, name: str) -> tuple[RType, ClassIR]: for ir in self.mro: if name in ir.attributes: @@ -366,7 +372,6 @@ def serialize(self) -> JsonDict: "is_generated": self.is_generated, "is_augmented": self.is_augmented, "is_final_class": self.is_final_class, - "is_immutable": self.is_immutable, "is_value_type": self.is_value_type, "inherits_python": self.inherits_python, "has_dict": self.has_dict, @@ -424,7 +429,6 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir.is_ext_class = data["is_ext_class"] ir.is_augmented = data["is_augmented"] ir.is_final_class = data["is_final_class"] - ir.is_immutable = data["is_immutable"] ir.is_value_type = data["is_value_type"] ir.inherits_python = data["inherits_python"] ir.has_dict = data["has_dict"] diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 031cd058a0cb..c5142245e1b3 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -99,9 +99,7 @@ def deserialize_type(data: JsonDict | str, ctx: DeserMaps) -> RType: else: assert False, f"Can't find class {data}" elif data[".class"] == "RInstanceValue": - class_ = deserialize_type(data["class"], ctx) - assert isinstance(class_, RInstance) - return RInstanceValue(class_.class_ir) + return RInstanceValue.deserialize(data, ctx) elif data[".class"] == "RTuple": return RTuple.deserialize(data, ctx) elif data[".class"] == "RUnion": @@ -853,7 +851,11 @@ def __repr__(self) -> str: return "" % self.name def __eq__(self, other: object) -> bool: - return isinstance(other, RInstance) and other.name == self.name + return ( + isinstance(other, RInstance) + and not isinstance(other, RInstanceValue) + and other.name == self.name + ) def __hash__(self) -> int: return hash(self.name) @@ -880,9 +882,6 @@ def __init__(self, class_ir: ClassIR) -> None: def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rinstance_value(self) - def attr_type(self, name: str) -> RType: - return self.class_ir.attr_type(name) - def __repr__(self) -> str: return "" % self.name @@ -893,7 +892,13 @@ def __hash__(self) -> int: return hash(self.name) def serialize(self) -> JsonDict: - return {".class": "RInstanceValue", "class": super().serialize()} + return {".class": "RInstanceValue", "cls": super().serialize()} + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RInstanceValue: + dt = deserialize_type(data["cls"], ctx) + assert isinstance(dt, RInstance) + return RInstanceValue(dt.class_ir) class RUnion(RType): diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index bcc9594adcb9..55ad2400abea 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -2,10 +2,11 @@ from __future__ import annotations -import typing_extensions from abc import abstractmethod from typing import Callable, Final +import typing_extensions + from mypy.nodes import ( TYPE_VAR_TUPLE_KIND, AssignmentStmt, @@ -806,7 +807,9 @@ def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> None: # If __eq__ returns NotImplemented, then __ne__ should also not_implemented_block, regular_block = BasicBlock(), BasicBlock() - eqval = builder.add(MethodCall(builder.self(), "__eq__", [rhs_arg], line)) + method_arg_type = cls.get_method("__eq__").args[1].type + rhs = builder.coerce(rhs_arg, method_arg_type, line) + eqval = builder.add(MethodCall(builder.self(), "__eq__", [rhs], line)) not_implemented = builder.add( LoadAddress(not_implemented_op.type, not_implemented_op.src, line) ) diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 1a4615e15095..1031c64e4fb4 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -17,12 +17,12 @@ def g() -> int: from __future__ import annotations -from mypy.nodes import Argument, FuncDef, SymbolNode -from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME +from mypy.nodes import Argument, FuncDef, SymbolNode, Var +from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME, bitmap_name from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncSignature from mypyc.ir.ops import Call, GetAttr, SetAttr, Value -from mypyc.ir.rtypes import RInstance, object_rprimitive +from mypyc.ir.rtypes import RInstance, object_rprimitive, bitmap_rprimitive from mypyc.irbuild.builder import IRBuilder, SymbolTarget from mypyc.irbuild.context import FuncInfo, GeneratorClass, ImplicitClass from mypyc.irbuild.targets import AssignmentTargetAttr @@ -177,15 +177,20 @@ def add_args_to_env( reassign: bool = True, ) -> None: fn_info = builder.fn_info + args = fn_info.fitem.arguments + nb = num_bitmap_args(builder, args) if local: - for sarg, farg in zip(sig.args, fn_info.fitem.arguments): - assert sarg.name == farg.variable.name - builder.add_local_reg(farg.variable, sarg.type, is_arg=True) + for arg in args: + rtype = builder.type_to_rtype(arg.variable.type) + builder.add_local_reg(arg.variable, rtype, is_arg=True) + for i in reversed(range(nb)): + builder.add_local_reg(Var(bitmap_name(i)), bitmap_rprimitive, is_arg=True) else: - for sarg, farg in zip(sig.args, fn_info.fitem.arguments): - if is_free_variable(builder, farg.variable) or fn_info.is_generator: + for arg in args: + if is_free_variable(builder, arg.variable) or fn_info.is_generator: + rtype = builder.type_to_rtype(arg.variable.type) assert base is not None, "base cannot be None for adding nonlocal args" - builder.add_var_to_env_class(farg.variable, sarg.type, base, reassign=reassign) + builder.add_var_to_env_class(arg.variable, rtype, base, reassign=reassign) def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: ImplicitClass) -> None: diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index d9276a0d9e8e..e17e2c583bfe 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -57,9 +57,8 @@ get_mypyc_attrs, is_dataclass, is_extension_class, - is_immutable, is_trait, - is_value_type, + is_value_type, is_immutable, ) from mypyc.options import CompilerOptions from mypyc.sametype import is_same_type @@ -88,10 +87,14 @@ def build_type_map( is_trait(cdef), is_abstract=cdef.info.is_abstract, is_final_class=cdef.info.is_final, - is_immutable=is_immutable(cdef), + is_ext_class=is_extension_class(cdef), is_value_type=is_value_type(cdef), ) - class_ir.is_ext_class = is_extension_class(cdef) + + if class_ir.is_value_type and (not is_immutable(cdef) or not cdef.info.is_final): + # Because the value type have semantic differences we can not just ignore it + errors.error("Value types must be immutable and final", module.path, cdef.line) + if class_ir.is_ext_class: class_ir.deletable = cdef.info.deletable_attributes.copy() # If global optimizations are disabled, turn of tracking of class children diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index 9bb146398076..10109981d71f 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -82,15 +82,16 @@ def dataclass_type(cdef: ClassDef) -> str | None: def is_immutable(cdef: ClassDef) -> bool: - """Check if a class is immutable by checking if all its variables are marked as Final.""" - for v in cdef.info.names.values(): - if ( - isinstance(v.node, Var) - and not v.node.is_classvar - and v.node.name not in ("__slots__", "__deletable__") - ): - if not v.node.is_final or v.node.is_settable_property: - return False + """Check if a class is immutable by checking if all its attributes are marked as Final.""" + for c in cdef.info.mro: + for v in c.names.values(): + if ( + isinstance(v.node, Var) + and not v.node.is_classvar + and v.node.name not in ("__slots__", "__deletable__") + ): + if not v.node.is_final or v.node.is_settable_property: + return False return True diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test new file mode 100644 index 000000000000..92af04587b54 --- /dev/null +++ b/mypyc/test-data/run-valuetype.test @@ -0,0 +1,194 @@ +[case testValueTypeBasic] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + self.y: Final = y + + def __add__(self, other: "Vector2I") -> "Vector2I": + return Vector2I(self.x + other.x, self.y + other.y) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Vector2I): + return NotImplemented + return self.x == other.x and self.y == other.y + +def test_basic_value_type() -> None: + a = Vector2I(1, 2) + b = Vector2I(3, 4) + c = a.__add__(b) + assert c == Vector2I(4, 6) + +def test_value_type_operator_is() -> None: + # The 'is' operator have undefined result for value types. + # however it must not raise an error. + Vector2I(1, 2) is Vector2I(1, 2) + +[case testValueTypeInheritance] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +class A: + def __init__(self, x: i32) -> None: + self.x: Final = x + + def method(self) -> i32: + return self.x + 4 + + def method2(self) -> i32: + return self.x + 4 + + @property + def prop(self) -> i32: + return self.x + 4 + + @property + def prop2(self) -> i32: + return self.x + 4 + +@final +@mypyc_attr(value_type=True) +class B(A): + def __init__(self, x: i32, y: i32) -> None: + super().__init__(x + 1) + self.y: Final = y + + def method2(self) -> i32: + return self.x + 7 + + @property + def prop2(self) -> i32: + return self.x + 7 + + +def test_value_type_inheritance() -> None: + # Tests with inheritance overriding methods and properties + assert A(1).x == 1 + assert B(1, 3).x == 2 + assert B(1, 3).y == 3 + assert A(1).method() == 5 + assert A(1).method2() == 5 + assert A(1).prop == 5 + assert A(1).prop2 == 5 + assert B(1, 3).method() == 6 + assert B(1, 3).method2() == 9 + assert B(1, 3).prop == 6 + assert B(1, 3).prop2 == 9 + + +def test_value_type_inheritance_is() -> None: + # however it must not raise an error. + assert not (B(1, 3) is A(1)) + assert not (A(1) is B(1, 3)) + assert B(1, 3) is not A(1) + assert A(1) is not B(1, 3) + +def f(a: A) -> i32: + return a.x + +def fcast(a: object) -> i32: + assert isinstance(a, A) + return a.x + +def fcast2(a: object) -> i32: + assert isinstance(a, B) + return a.x + +def test_value_type_boxing() -> None: + assert f(A(1)) == 1 + assert f(B(1, 3)) == 2 + assert fcast(A(1)) == 1 + assert fcast(B(1, 3)) == 2 + assert fcast2(B(1, 3)) == 2 + +[case testValueTypeNest] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + self.y: Final = y + +@final +@mypyc_attr(value_type=True) +class Fig: + def __init__(self, name: str, v: Vector2I) -> None: + self.name: Final = name + self.pos: Final = v + +def test_value_type_nest() -> None: + f = Fig("fig", Vector2I(1, 2)) + assert f.name == "fig" + assert f.pos.x == 1 + assert f.pos.y == 2 + +[case testValueTypeNest2] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + self.y: Final = y + +# NON VALUE TYPE +class Fig: + def __init__(self, name: str, v: Vector2I) -> None: + self.name = name + self.pos = v + +def test_value_type_nest() -> None: + f = Fig("fig", Vector2I(1, 2)) + assert f.name == "fig" + assert f.pos.x == 1 + assert f.pos.y == 2 + +[case testValueTypeInSeq] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class A: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + +def test_value_type_in_list() -> None: + a = A(1, 2) + l = [a, a] + assert l[0].x == 1 + assert l[1].x == 1 + +def test_value_type_in_list2() -> None: + l = [A(1, 2), A(1, 2)] + assert l[0].x == 1 + assert l[1].x == 1 + +def test_value_type_in_list3() -> None: + l = [A(1, 2), A(5, 7)] + assert l[0].x == 1 + assert l[1].x == 5 + assert l[0] != A(1, 2) + assert l[1] != A(5, 7) + +def test_value_type_in_list4() -> None: + l: list[object] = [A(1, 2), "test"] + assert l[0] != A(1, 2) + assert isinstance(l[0], A) + assert l[0].x == 1 + assert l[1] == "test" + +def test_value_type_in_tuple() -> None: + l = (A(1, 2), "test") + assert l[0] != A(1, 2) + assert l[0].x == 1 + assert l[1] == "test" diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 668e5b124841..ebf59c8a4f1f 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -67,6 +67,7 @@ "run-attrs.test", "run-python37.test", "run-python38.test", + "run-valuetype.test", ] if sys.version_info >= (3, 10): From 1072d2502c6516a60496dd6f6adb15e496265298 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Wed, 16 Oct 2024 23:26:10 -0500 Subject: [PATCH 03/21] Fix mypy issues --- mypyc/ir/rtypes.py | 6 +++--- mypyc/irbuild/classdef.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index c5142245e1b3..905b17ddb2ba 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -580,7 +580,7 @@ class TupleNameVisitor(RTypeVisitor[str]): def visit_rinstance(self, t: RInstance) -> str: return "O" - def visit_rinstance_value(self, typ: RInstanceValue) -> T: + def visit_rinstance_value(self, typ: RInstanceValue) -> str: return "O" def visit_runion(self, t: RUnion) -> str: @@ -860,7 +860,7 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash(self.name) - def serialize(self) -> str: + def serialize(self) -> JsonDict | str: return self.name @@ -891,7 +891,7 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash(self.name) - def serialize(self) -> JsonDict: + def serialize(self) -> JsonDict | str: return {".class": "RInstanceValue", "cls": super().serialize()} @classmethod diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 55ad2400abea..3ac2ed7b3575 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -807,7 +807,9 @@ def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> None: # If __eq__ returns NotImplemented, then __ne__ should also not_implemented_block, regular_block = BasicBlock(), BasicBlock() - method_arg_type = cls.get_method("__eq__").args[1].type + eq_func_ir = cls.get_method("__eq__") + assert eq_func_ir is not None + method_arg_type = eq_func_ir.args[1].type rhs = builder.coerce(rhs_arg, method_arg_type, line) eqval = builder.add(MethodCall(builder.self(), "__eq__", [rhs], line)) not_implemented = builder.add( From a1b95a38d6071ad91364d9a30641e1028a971b89 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 06:08:24 -0500 Subject: [PATCH 04/21] Fix code formatting --- mypyc/irbuild/prepare.py | 3 ++- mypyc/test-data/run-valuetype.test | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index e17e2c583bfe..59fd5f4997a5 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -57,8 +57,9 @@ get_mypyc_attrs, is_dataclass, is_extension_class, + is_immutable, is_trait, - is_value_type, is_immutable, + is_value_type, ) from mypyc.options import CompilerOptions from mypyc.sametype import is_same_type diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test index 92af04587b54..08cf804ace7e 100644 --- a/mypyc/test-data/run-valuetype.test +++ b/mypyc/test-data/run-valuetype.test @@ -153,7 +153,7 @@ def test_value_type_nest() -> None: assert f.pos.y == 2 [case testValueTypeInSeq] -from typing import final, Final +from typing import List, final, Final from mypy_extensions import mypyc_attr, i32 @final @@ -181,7 +181,7 @@ def test_value_type_in_list3() -> None: assert l[1] != A(5, 7) def test_value_type_in_list4() -> None: - l: list[object] = [A(1, 2), "test"] + l: List[object] = [A(1, 2), "test"] assert l[0] != A(1, 2) assert isinstance(l[0], A) assert l[0].x == 1 From f93e96bfcf8201070cb178756db2823064ea60b4 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 06:18:06 -0500 Subject: [PATCH 05/21] Fix is_immutable --- mypyc/irbuild/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index 10109981d71f..8b3a692d0e32 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -84,11 +84,13 @@ def dataclass_type(cdef: ClassDef) -> str | None: def is_immutable(cdef: ClassDef) -> bool: """Check if a class is immutable by checking if all its attributes are marked as Final.""" for c in cdef.info.mro: + if c.fullname == "builtins.object": + continue for v in c.names.values(): if ( isinstance(v.node, Var) and not v.node.is_classvar - and v.node.name not in ("__slots__", "__deletable__") + and v.node.name not in ("__slots__", "__deletable__", "__doc__", "__dict__") ): if not v.node.is_final or v.node.is_settable_property: return False From a77e1e14271ea46737c85b311d262e7c217cceb4 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 06:29:46 -0500 Subject: [PATCH 06/21] Fix code formatting --- mypyc/codegen/emitclass.py | 2 +- mypyc/codegen/emitfunc.py | 9 ++++++++- mypyc/irbuild/classdef.py | 3 +-- mypyc/irbuild/env_class.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 68d3de0737f4..84b012c84bc4 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -21,7 +21,7 @@ from mypyc.common import BITMAP_BITS, BITMAP_TYPE, NATIVE_PREFIX, PREFIX, REG_PREFIX from mypyc.ir.class_ir import ClassIR, VTableEntries from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR -from mypyc.ir.rtypes import RTuple, RType, object_rprimitive, RInstanceValue, RInstance +from mypyc.ir.rtypes import RInstance, RInstanceValue, RTuple, RType, object_rprimitive from mypyc.namegen import NameGenerator from mypyc.sametype import is_same_type diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 3c3537ba6331..c238be4a65bd 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -15,7 +15,14 @@ TYPE_VAR_PREFIX, ) from mypyc.ir.class_ir import ClassIR -from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values, FUNC_NORMAL +from mypyc.ir.func_ir import ( + FUNC_CLASSMETHOD, + FUNC_NORMAL, + FUNC_STATICMETHOD, + FuncDecl, + FuncIR, + all_values, +) from mypyc.ir.ops import ( ERR_FALSE, NAMESPACE_MODULE, diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 3ac2ed7b3575..3f12bafa77c6 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -2,11 +2,10 @@ from __future__ import annotations +import typing_extensions from abc import abstractmethod from typing import Callable, Final -import typing_extensions - from mypy.nodes import ( TYPE_VAR_TUPLE_KIND, AssignmentStmt, diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 1031c64e4fb4..2f22b311ed34 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -22,7 +22,7 @@ def g() -> int: from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncSignature from mypyc.ir.ops import Call, GetAttr, SetAttr, Value -from mypyc.ir.rtypes import RInstance, object_rprimitive, bitmap_rprimitive +from mypyc.ir.rtypes import RInstance, bitmap_rprimitive, object_rprimitive from mypyc.irbuild.builder import IRBuilder, SymbolTarget from mypyc.irbuild.context import FuncInfo, GeneratorClass, ImplicitClass from mypyc.irbuild.targets import AssignmentTargetAttr From 0ff55899fda88930d7c5090d4b8f4b69df088a40 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 06:37:02 -0500 Subject: [PATCH 07/21] Modify some doc strings --- mypyc/codegen/emit.py | 7 +------ mypyc/ir/rtypes.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index ae9c4d326cbf..cd0cb97af6be 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -362,12 +362,7 @@ def bitmap_field(self, index: int) -> str: return f"bitmap{n + 1}" def attr_bitmap_expr(self, obj: str, cl: RInstance, index: int) -> str: - """ - Return reference to the attribute definedness bitmap. - - If a_ref is True, assume object's type is a reference. - Otherwise, the object type is indicated by the class IR. - """ + """Return reference to the attribute definedness bitmap.""" attr = self.bitmap_field(index) if cl.is_unboxed: return f"{obj}.{attr}" diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 905b17ddb2ba..eaf7aa87d9f9 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -807,7 +807,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RStruct: class RInstance(RType): """Instance of user-defined class (compiled to C extension class). - The runtime representation is typically 'PyObject *', and these are + The runtime representation is 'PyObject *', and these are always boxed and thus reference-counted. These support fast method calls and fast attribute access using From 7e8a71b8031dd85f43e4fd15d058989f3e655bf4 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:27:16 -0500 Subject: [PATCH 08/21] Add tests for dictionaries and dunders --- mypyc/codegen/emit.py | 2 + mypyc/codegen/emitwrapper.py | 22 +- mypyc/test-data/run-valuetype.test | 459 +++++++++++++++++------------ 3 files changed, 285 insertions(+), 198 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index cd0cb97af6be..14bddf31f8e4 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1083,6 +1083,8 @@ def emit_box( for attr, attr_type in cl.all_attributes().items(): attr_name = self.attr(attr) self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box") + if attr_type.is_refcounted: + self.emit_inc_ref(temp_dest, attr_type) self.emit_line(f"{declaration}{dest} = (PyObject *){temp_dest};") else: diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 45c6c7a05867..1d2aa061a688 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -131,6 +131,16 @@ def make_format_string(func_name: str | None, groups: dict[ArgKind, list[Runtime return format +def unbox_value_if_required(src: str, dest: str, dest_type: RType, emitter: Emitter) -> bool: + """Declares a new dest variable and unbox if needed. Assumes src is boxed.""" + if dest_type.is_unboxed: + emitter.emit_unbox(src, dest, dest_type, declare_dest=True) + return True + else: + emitter.emit_line(f"{emitter.ctype_spaced(dest_type)}{dest} = {src};") + return False + + def generate_wrapper_function( fn: FuncIR, emitter: Emitter, source_path: str, module_name: str ) -> None: @@ -548,8 +558,9 @@ def generate_hash_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: """Generates a wrapper for native __hash__ methods.""" name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" emitter.emit_line(f"static Py_ssize_t {name}(PyObject *self) {{") + unbox_value_if_required("self", "obj_self", fn.args[0].type, emitter) emitter.emit_line( - "{}retval = {}{}{}(self);".format( + "{}retval = {}{}{}(obj_self);".format( emitter.ctype_spaced(fn.ret_type), emitter.get_group_prefix(fn.decl), NATIVE_PREFIX, @@ -575,8 +586,9 @@ def generate_len_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: """Generates a wrapper for native __len__ methods.""" name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" emitter.emit_line(f"static Py_ssize_t {name}(PyObject *self) {{") + unbox_value_if_required("self", "obj_self", fn.args[0].type, emitter) emitter.emit_line( - "{}retval = {}{}{}(self);".format( + "{}retval = {}{}{}(obj_self);".format( emitter.ctype_spaced(fn.ret_type), emitter.get_group_prefix(fn.decl), NATIVE_PREFIX, @@ -600,8 +612,9 @@ def generate_bool_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: """Generates a wrapper for native __bool__ methods.""" name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" emitter.emit_line(f"static int {name}(PyObject *self) {{") + unbox_value_if_required("self", "obj_self", fn.args[0].type, emitter) emitter.emit_line( - "{}val = {}{}(self);".format( + "{}val = {}{}(obj_self);".format( emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names) ) ) @@ -722,8 +735,9 @@ def generate_contains_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" emitter.emit_line(f"static int {name}(PyObject *self, PyObject *obj_item) {{") generate_arg_check("item", fn.args[1].type, emitter, ReturnHandler("-1")) + unbox_value_if_required("self", "obj_self", fn.args[0].type, emitter) emitter.emit_line( - "{}val = {}{}(self, arg_item);".format( + "{}val = {}{}(obj_self, arg_item);".format( emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names) ) ) diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test index 08cf804ace7e..ee5013077b8d 100644 --- a/mypyc/test-data/run-valuetype.test +++ b/mypyc/test-data/run-valuetype.test @@ -1,194 +1,265 @@ -[case testValueTypeBasic] -from typing import final, Final -from mypy_extensions import mypyc_attr, i32 - -@final -@mypyc_attr(value_type=True) -class Vector2I: - def __init__(self, x: i32, y: i32) -> None: - self.x: Final = x - self.y: Final = y - - def __add__(self, other: "Vector2I") -> "Vector2I": - return Vector2I(self.x + other.x, self.y + other.y) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Vector2I): - return NotImplemented - return self.x == other.x and self.y == other.y - -def test_basic_value_type() -> None: - a = Vector2I(1, 2) - b = Vector2I(3, 4) - c = a.__add__(b) - assert c == Vector2I(4, 6) - -def test_value_type_operator_is() -> None: - # The 'is' operator have undefined result for value types. - # however it must not raise an error. - Vector2I(1, 2) is Vector2I(1, 2) - -[case testValueTypeInheritance] -from typing import final, Final -from mypy_extensions import mypyc_attr, i32 - -class A: - def __init__(self, x: i32) -> None: - self.x: Final = x - - def method(self) -> i32: - return self.x + 4 - - def method2(self) -> i32: - return self.x + 4 - - @property - def prop(self) -> i32: - return self.x + 4 - - @property - def prop2(self) -> i32: - return self.x + 4 - -@final -@mypyc_attr(value_type=True) -class B(A): - def __init__(self, x: i32, y: i32) -> None: - super().__init__(x + 1) - self.y: Final = y - - def method2(self) -> i32: - return self.x + 7 - - @property - def prop2(self) -> i32: - return self.x + 7 - - -def test_value_type_inheritance() -> None: - # Tests with inheritance overriding methods and properties - assert A(1).x == 1 - assert B(1, 3).x == 2 - assert B(1, 3).y == 3 - assert A(1).method() == 5 - assert A(1).method2() == 5 - assert A(1).prop == 5 - assert A(1).prop2 == 5 - assert B(1, 3).method() == 6 - assert B(1, 3).method2() == 9 - assert B(1, 3).prop == 6 - assert B(1, 3).prop2 == 9 - - -def test_value_type_inheritance_is() -> None: - # however it must not raise an error. - assert not (B(1, 3) is A(1)) - assert not (A(1) is B(1, 3)) - assert B(1, 3) is not A(1) - assert A(1) is not B(1, 3) - -def f(a: A) -> i32: - return a.x - -def fcast(a: object) -> i32: - assert isinstance(a, A) - return a.x - -def fcast2(a: object) -> i32: - assert isinstance(a, B) - return a.x - -def test_value_type_boxing() -> None: - assert f(A(1)) == 1 - assert f(B(1, 3)) == 2 - assert fcast(A(1)) == 1 - assert fcast(B(1, 3)) == 2 - assert fcast2(B(1, 3)) == 2 - -[case testValueTypeNest] -from typing import final, Final -from mypy_extensions import mypyc_attr, i32 - -@final -@mypyc_attr(value_type=True) -class Vector2I: - def __init__(self, x: i32, y: i32) -> None: - self.x: Final = x - self.y: Final = y - -@final -@mypyc_attr(value_type=True) -class Fig: - def __init__(self, name: str, v: Vector2I) -> None: - self.name: Final = name - self.pos: Final = v - -def test_value_type_nest() -> None: - f = Fig("fig", Vector2I(1, 2)) - assert f.name == "fig" - assert f.pos.x == 1 - assert f.pos.y == 2 - -[case testValueTypeNest2] -from typing import final, Final -from mypy_extensions import mypyc_attr, i32 - -@final -@mypyc_attr(value_type=True) -class Vector2I: - def __init__(self, x: i32, y: i32) -> None: - self.x: Final = x - self.y: Final = y - -# NON VALUE TYPE -class Fig: - def __init__(self, name: str, v: Vector2I) -> None: - self.name = name - self.pos = v - -def test_value_type_nest() -> None: - f = Fig("fig", Vector2I(1, 2)) - assert f.name == "fig" - assert f.pos.x == 1 - assert f.pos.y == 2 - -[case testValueTypeInSeq] -from typing import List, final, Final -from mypy_extensions import mypyc_attr, i32 - -@final -@mypyc_attr(value_type=True) -class A: - def __init__(self, x: i32, y: i32) -> None: - self.x: Final = x - -def test_value_type_in_list() -> None: - a = A(1, 2) - l = [a, a] - assert l[0].x == 1 - assert l[1].x == 1 - -def test_value_type_in_list2() -> None: - l = [A(1, 2), A(1, 2)] - assert l[0].x == 1 - assert l[1].x == 1 - -def test_value_type_in_list3() -> None: - l = [A(1, 2), A(5, 7)] - assert l[0].x == 1 - assert l[1].x == 5 - assert l[0] != A(1, 2) - assert l[1] != A(5, 7) - -def test_value_type_in_list4() -> None: - l: List[object] = [A(1, 2), "test"] - assert l[0] != A(1, 2) - assert isinstance(l[0], A) - assert l[0].x == 1 - assert l[1] == "test" - -def test_value_type_in_tuple() -> None: - l = (A(1, 2), "test") - assert l[0] != A(1, 2) - assert l[0].x == 1 - assert l[1] == "test" +[case testValueTypeBasic] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + self.y: Final = y + + def __add__(self, other: "Vector2I") -> "Vector2I": + return Vector2I(self.x + other.x, self.y + other.y) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Vector2I): + return NotImplemented + return self.x == other.x and self.y == other.y + +def test_basic_value_type() -> None: + a = Vector2I(1, 2) + b = Vector2I(3, 4) + c = a.__add__(b) + assert c == Vector2I(4, 6) + +def test_value_type_operator_is() -> None: + # The 'is' operator have undefined result for value types. + # however it must not raise an error. + Vector2I(1, 2) is Vector2I(1, 2) + +[case testValueTypeInheritance] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +class A: + def __init__(self, x: i32) -> None: + self.x: Final = x + + def method(self) -> i32: + return self.x + 4 + + def method2(self) -> i32: + return self.x + 4 + + @property + def prop(self) -> i32: + return self.x + 4 + + @property + def prop2(self) -> i32: + return self.x + 4 + +@final +@mypyc_attr(value_type=True) +class B(A): + def __init__(self, x: i32, y: i32) -> None: + super().__init__(x + 1) + self.y: Final = y + + def method2(self) -> i32: + return self.x + 7 + + @property + def prop2(self) -> i32: + return self.x + 7 + + +def test_value_type_inheritance() -> None: + # Tests with inheritance overriding methods and properties + assert A(1).x == 1 + assert B(1, 3).x == 2 + assert B(1, 3).y == 3 + assert A(1).method() == 5 + assert A(1).method2() == 5 + assert A(1).prop == 5 + assert A(1).prop2 == 5 + assert B(1, 3).method() == 6 + assert B(1, 3).method2() == 9 + assert B(1, 3).prop == 6 + assert B(1, 3).prop2 == 9 + + +def test_value_type_inheritance_is() -> None: + # however it must not raise an error. + assert not (B(1, 3) is A(1)) + assert not (A(1) is B(1, 3)) + assert B(1, 3) is not A(1) + assert A(1) is not B(1, 3) + +def f(a: A) -> i32: + return a.x + +def fcast(a: object) -> i32: + assert isinstance(a, A) + return a.x + +def fcast2(a: object) -> i32: + assert isinstance(a, B) + return a.x + +def test_value_type_boxing() -> None: + assert f(A(1)) == 1 + assert f(B(1, 3)) == 2 + assert fcast(A(1)) == 1 + assert fcast(B(1, 3)) == 2 + assert fcast2(B(1, 3)) == 2 + +[case testValueTypeNest] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + self.y: Final = y + +@final +@mypyc_attr(value_type=True) +class Fig: + def __init__(self, name: str, v: Vector2I) -> None: + self.name: Final = name + self.pos: Final = v + +def test_value_type_nest() -> None: + f = Fig("fig", Vector2I(1, 2)) + assert f.name == "fig" + assert f.pos.x == 1 + assert f.pos.y == 2 + +[case testValueTypeNest2] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + self.y: Final = y + +# NON VALUE TYPE +class Fig: + def __init__(self, name: str, v: Vector2I) -> None: + self.name = name + self.pos = v + +def test_value_type_nest() -> None: + f = Fig("fig", Vector2I(1, 2)) + assert f.name == "fig" + assert f.pos.x == 1 + assert f.pos.y == 2 + +[case testValueTypeInSeq] +from typing import List, final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class A: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + +def test_value_type_in_list() -> None: + a = A(1, 2) + l = [a, a] + assert l[0].x == 1 + assert l[1].x == 1 + +def test_value_type_in_list2() -> None: + l = [A(1, 2), A(1, 2)] + assert l[0].x == 1 + assert l[1].x == 1 + +def test_value_type_in_list3() -> None: + l = [A(1, 2), A(5, 7)] + assert l[0].x == 1 + assert l[1].x == 5 + assert l[0] != A(1, 2) + assert l[1] != A(5, 7) + +def test_value_type_in_list4() -> None: + l: List[object] = [A(1, 2), "test"] + assert l[0] != A(1, 2) + assert isinstance(l[0], A) + assert l[0].x == 1 + assert l[1] == "test" + +def test_value_type_in_tuple() -> None: + l = (A(1, 2), "test") + assert l[0] != A(1, 2) + assert l[0].x == 1 + assert l[1] == "test" + +[case testValueTypeInDictAndDunders] +from typing import Dict, final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class A: + def __init__(self, x: i32, y: i32) -> None: + self.x: Final = x + + def __hash__(self) -> int: + return hash(self.x) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, A): + return NotImplemented + return self.x == other.x + + def __len__(self) -> int: + return 1 + + def __lt__(self, other: object) -> bool: + if not isinstance(other, A): + return NotImplemented + return self.x < other.x + + def __gt__(self, other: object) -> bool: + if not isinstance(other, A): + return NotImplemented + return self.x > other.x + + def __le__(self, other: object) -> bool: + if not isinstance(other, A): + return NotImplemented + return self.x <= other.x + + def __ge__(self, other: object) -> bool: + if not isinstance(other, A): + return NotImplemented + return self.x >= other.x + + def __contains__(self, item: object) -> bool: + return isinstance(item, i32) and item == self.x + +def test_value_type_in_key() -> None: + a = A(1, 2) + d = {a: 1} + assert d[A(1,5)] == 1 + +def test_value_type_in_val() -> None: + a = A(1, 2) + d = {1: a} + assert d[1].x == 1 + +def test_len() -> None: + assert len(A(1, 2)) == 1 + +def test_cmp() -> None: + a = A(1, 2) + b = A(2, 3) + assert a < b + assert a <= b + assert b > a + assert b >= a + assert not a > b + assert not a >= b + assert not b < a + assert not b <= a + assert 1 in a + assert 2 not in a From 67caabb6a8bdbc117dcb6c551c1ab214a36e28ac Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:26:29 -0500 Subject: [PATCH 09/21] Add tests for str and repr --- mypyc/codegen/emitclass.py | 5 +-- mypyc/codegen/emitwrapper.py | 14 ++++++++ mypyc/ir/class_ir.py | 3 -- mypyc/irbuild/prepare.py | 34 +++++++++++++++++-- mypyc/test-data/run-valuetype.test | 54 ++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 8 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 84b012c84bc4..7b1bc6412887 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -17,6 +17,7 @@ generate_len_wrapper, generate_richcompare_wrapper, generate_set_del_item_wrapper, + generate_str_wrapper, ) from mypyc.common import BITMAP_BITS, BITMAP_TYPE, NATIVE_PREFIX, PREFIX, REG_PREFIX from mypyc.ir.class_ir import ClassIR, VTableEntries @@ -44,8 +45,8 @@ def wrapper_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: SLOT_DEFS: SlotTable = { "__init__": ("tp_init", lambda c, t, e: generate_init_for_class(c, t, e)), "__call__": ("tp_call", lambda c, t, e: generate_call_wrapper(c, t, e)), - "__str__": ("tp_str", native_slot), - "__repr__": ("tp_repr", native_slot), + "__str__": ("tp_str", generate_str_wrapper), + "__repr__": ("tp_repr", generate_str_wrapper), "__next__": ("tp_iternext", native_slot), "__iter__": ("tp_iter", native_slot), "__hash__": ("tp_hash", generate_hash_wrapper), diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 1d2aa061a688..24d05671f702 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -554,6 +554,20 @@ def generate_get_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: return name +def generate_str_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generates a wrapper for native __str__ and __repr__ methods.""" + name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" + emitter.emit_line(f"static PyObject* {name}(PyObject *self) {{") + unbox_value_if_required("self", "obj_self", fn.args[0].type, emitter) + emitter.emit_line( + "return {}{}{}(obj_self);".format( + emitter.get_group_prefix(fn.decl), NATIVE_PREFIX, fn.cname(emitter.names) + ) + ) + emitter.emit_line("}") + return name + + def generate_hash_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: """Generates a wrapper for native __hash__ methods.""" name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 2181330189cb..a0a28e9f3071 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -300,9 +300,6 @@ def struct_name(self, names: NameGenerator) -> str: def struct_name2(self) -> str: return f"{exported_name(self.fullname)}Object" - def struct_data_name(self) -> str: - return f"{exported_name(self.fullname)}Data" - def get_method_and_class( self, name: str, *, prefer_method: bool = False ) -> tuple[FuncIR, ClassIR] | None: diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 59fd5f4997a5..b7706de8c39f 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -79,6 +79,8 @@ def build_type_map( module_classes = [node for node in module.defs if isinstance(node, ClassDef)] classes.extend([(module, cdef) for cdef in module_classes]) + module_by_fullname = {module.fullname: module for module in modules} + # Collect all class mappings so that we can bind arbitrary class name # references even if there are import cycles. for module, cdef in classes: @@ -92,9 +94,8 @@ def build_type_map( is_value_type=is_value_type(cdef), ) - if class_ir.is_value_type and (not is_immutable(cdef) or not cdef.info.is_final): - # Because the value type have semantic differences we can not just ignore it - errors.error("Value types must be immutable and final", module.path, cdef.line) + if class_ir.is_value_type: + check_value_type(cdef, errors, module_by_fullname) if class_ir.is_ext_class: class_ir.deletable = cdef.info.deletable_attributes.copy() @@ -144,6 +145,33 @@ def build_type_map( ) +def check_value_type( + cdef: ClassDef, errors: Errors, module_by_fullname: dict[str, MypyFile] +) -> None: + if not is_immutable(cdef) or not cdef.info.is_final: + module = module_by_fullname.get(cdef.info.module_name) + # Because the value type have semantic differences we can not just ignore it + errors.error("Value types must be immutable and final", module.path, cdef.line) + + for mtd_name in ( + "__iter__", + "__next__", + "__enter__", + "__exit__", + "__getitem__", + "__setitem__", + "__delitem__", + ): + mtd = cdef.info.get_method(mtd_name) + if mtd is not None: + module = module_by_fullname.get(mtd.info.module_name) + errors.error( + f"Value types must not define method '{mtd_name}'", + module.path if module else "", + mtd.line, + ) + + def is_from_module(node: SymbolNode, module: MypyFile) -> bool: return node.fullname == module.fullname + "." + node.name diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test index ee5013077b8d..68bf068c49c6 100644 --- a/mypyc/test-data/run-valuetype.test +++ b/mypyc/test-data/run-valuetype.test @@ -237,6 +237,15 @@ class A: def __contains__(self, item: object) -> bool: return isinstance(item, i32) and item == self.x + def __bool__(self) -> bool: + return self.x != 0 + + def __str__(self) -> str: + return f"A({self.x})" + + def __repr__(self) -> str: + return f"A({self.x})" + def test_value_type_in_key() -> None: a = A(1, 2) d = {a: 1} @@ -246,10 +255,34 @@ def test_value_type_in_val() -> None: a = A(1, 2) d = {1: a} assert d[1].x == 1 + assert d[1] == A(1, 2) + +def test_value_type_in_set() -> None: + a = A(1, 2) + s = {a} + assert A(1, 2) in s + +def test_hash() -> None: + a = A(1, 2) + assert hash(a) == hash(1) + +def test_eq() -> None: + assert A(1, 2) == A(1, 2) + assert A(1, 2) != A(2, 3) + assert A(1, 2) != 1 + assert 1 != A(1, 2) + +def test_bool() -> None: + assert bool(A(1, 2)) + assert not bool(A(0, 2)) def test_len() -> None: assert len(A(1, 2)) == 1 +def test_str() -> None: + assert str(A(1, 2)) == "A(1)" + assert repr(A(1, 2)) == "A(1)" + def test_cmp() -> None: a = A(1, 2) b = A(2, 3) @@ -263,3 +296,24 @@ def test_cmp() -> None: assert not b <= a assert 1 in a assert 2 not in a + +[case testValueTypeLifetime] +from typing import final, Final, NamedTuple +from mypy_extensions import mypyc_attr + +class A(NamedTuple): + n: str + +@final +@mypyc_attr(value_type=True) +class B: + def __init__(self, a: A) -> None: + self.a: Final = a + +def f() -> B: + # A instance is created in the function scope + # should be keep alive after the function returns + return B(A(n="test")) + +def test_value_type_lifetime() -> None: + assert f().a.n == "test" From 689c446fd7a641c787eecd030427722061f79d55 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:46:31 -0500 Subject: [PATCH 10/21] Copy bitmap fields on box too --- mypyc/codegen/emit.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 14bddf31f8e4..cbffecb5bcac 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1080,6 +1080,11 @@ def emit_box( ) self.emit_line(f"if (unlikely({temp_dest} == NULL))") self.emit_line(" CPyError_OutOfMemory();") + if cl.bitmap_attrs: + n_fields = (len(cl.bitmap_attrs) - 1) // BITMAP_BITS + 1 + for i in range(n_fields): + attr_name = self.bitmap_field(i * BITMAP_BITS) + self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box") for attr, attr_type in cl.all_attributes().items(): attr_name = self.attr(attr) self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box") From 4fde6f9f9ad57bb2dce85daa5f46c8ec3327eb51 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:48:07 -0500 Subject: [PATCH 11/21] Fix error string --- mypyc/irbuild/prepare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index b7706de8c39f..aea3a3790f1a 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -150,8 +150,9 @@ def check_value_type( ) -> None: if not is_immutable(cdef) or not cdef.info.is_final: module = module_by_fullname.get(cdef.info.module_name) + path = module.path if module else "" # Because the value type have semantic differences we can not just ignore it - errors.error("Value types must be immutable and final", module.path, cdef.line) + errors.error("Value types must be immutable and final", path, cdef.line) for mtd_name in ( "__iter__", From 85757478ea6f6a79961ccacaabf88cdd75c46d10 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Fri, 18 Oct 2024 19:11:22 -0500 Subject: [PATCH 12/21] Add more testing --- mypyc/irbuild/env_class.py | 12 +++++------- mypyc/irbuild/function.py | 4 ++-- mypyc/irbuild/generator.py | 6 +++--- mypyc/irbuild/ll_builder.py | 1 + mypyc/test-data/run-valuetype.test | 17 +++++++++++++++++ mypyc/test/test_run.py | 18 +++++++++++++++++- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 2f22b311ed34..9ba639bde399 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -20,7 +20,6 @@ def g() -> int: from mypy.nodes import Argument, FuncDef, SymbolNode, Var from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME, bitmap_name from mypyc.ir.class_ir import ClassIR -from mypyc.ir.func_ir import FuncSignature from mypyc.ir.ops import Call, GetAttr, SetAttr, Value from mypyc.ir.rtypes import RInstance, bitmap_rprimitive, object_rprimitive from mypyc.irbuild.builder import IRBuilder, SymbolTarget @@ -57,7 +56,7 @@ class is generated, the function environment has not yet been return env_class -def finalize_env_class(builder: IRBuilder, sig: FuncSignature) -> None: +def finalize_env_class(builder: IRBuilder) -> None: """Generate, instantiate, and set up the environment of an environment class.""" instantiate_env_class(builder) @@ -65,9 +64,9 @@ def finalize_env_class(builder: IRBuilder, sig: FuncSignature) -> None: # that were previously added to the environment with references to the function's # environment class. if builder.fn_info.is_nested: - add_args_to_env(builder, sig, local=False, base=builder.fn_info.callable_class) + add_args_to_env(builder, local=False, base=builder.fn_info.callable_class) else: - add_args_to_env(builder, sig, local=False, base=builder.fn_info) + add_args_to_env(builder, local=False, base=builder.fn_info) def instantiate_env_class(builder: IRBuilder) -> Value: @@ -92,7 +91,7 @@ def instantiate_env_class(builder: IRBuilder) -> Value: return curr_env_reg -def load_env_registers(builder: IRBuilder, sig: FuncSignature) -> None: +def load_env_registers(builder: IRBuilder) -> None: """Load the registers for the current FuncItem being visited. Adds the arguments of the FuncItem to the environment. If the @@ -100,7 +99,7 @@ def load_env_registers(builder: IRBuilder, sig: FuncSignature) -> None: loads all of the outer environments of the FuncItem into registers so that they can be used when accessing free variables. """ - add_args_to_env(builder, sig, local=True) + add_args_to_env(builder, local=True) fn_info = builder.fn_info fitem = fn_info.fitem @@ -171,7 +170,6 @@ def num_bitmap_args(builder: IRBuilder, args: list[Argument]) -> int: def add_args_to_env( builder: IRBuilder, - sig: FuncSignature, local: bool = True, base: FuncInfo | ImplicitClass | None = None, reassign: bool = True, diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 87adbf8d7d28..06fa047d20bf 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -275,11 +275,11 @@ def c() -> None: create_switch_for_generator_class(builder) add_raise_exception_blocks_to_generator_class(builder, fitem.line) else: - load_env_registers(builder, sig) + load_env_registers(builder) gen_arg_defaults(builder) if builder.fn_info.contains_nested and not builder.fn_info.is_generator: - finalize_env_class(builder, sig) + finalize_env_class(builder) builder.ret_types[-1] = sig.ret_type diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index 101b67acdeab..01a0317821d5 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -51,9 +51,9 @@ def gen_generator_func(builder: IRBuilder, sig: FuncSignature) -> None: setup_generator_class(builder) - load_env_registers(builder, sig) + load_env_registers(builder) gen_arg_defaults(builder) - finalize_env_class(builder, sig) + finalize_env_class(builder) builder.add(Return(instantiate_generator_class(builder))) @@ -340,7 +340,7 @@ def setup_env_for_generator_class(builder: IRBuilder, sig: FuncSignature) -> Non # Add arguments from the original generator function to the # environment of the generator class. - add_args_to_env(builder, sig, local=False, base=cls, reassign=False) + add_args_to_env(builder, local=False, base=cls, reassign=False) # Set the next label register for the generator class. cls.next_label_reg = builder.read(cls.next_label_target, fitem.line) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index b08a53c1b135..a507afd9a981 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -1440,6 +1440,7 @@ def dunder_op(self, lreg: Value, rreg: Value | None, op: str, line: int) -> Valu if op == "in" and method_name == "__contains__": # contains needs to swap the r and l values + assert rreg, "'in' operator is binary and must have a valid right value" tmp = lreg lreg = rreg rreg = tmp diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test index 68bf068c49c6..453924381d7a 100644 --- a/mypyc/test-data/run-valuetype.test +++ b/mypyc/test-data/run-valuetype.test @@ -246,6 +246,23 @@ class A: def __repr__(self) -> str: return f"A({self.x})" +def test_direct_dunders() -> None: + a = A(1, 2) + assert a.__hash__() == hash(1) + assert a.__eq__(A(1, 2)) + assert not a.__eq__(A(2, 3)) + assert a.__len__() == 1 + assert a.__lt__(A(2, 3)) + assert a.__le__(A(2, 3)) + assert not a.__gt__(A(2, 3)) + assert not a.__ge__(A(2, 3)) + assert a.__contains__(1) + assert not a.__contains__(2) + assert a.__bool__() + assert not A(0, 1).__bool__() + assert a.__str__() == "A(1)" + assert a.__repr__() == "A(1)" + def test_value_type_in_key() -> None: a = A(1, 2) d = {a: 1} diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index c5c5185422a7..3c521da84999 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -413,7 +413,23 @@ class TestRunStrictDunderTyping(TestRun): strict_dunder_typing = True test_name_suffix = "_dunder_typing" - files = ["run-dunders.test", "run-floats.test"] + files = [ + "run-async.test", + "run-bools.test", + "run-bytes.test", + "run-classes.test", + "run-dunders.test", + "run-exceptions.test", + "run-floats.test", + "run-functions.test", + "run-generators.test", + "run-i16.test", + "run-i32.test", + "run-i64.test", + "run-integers.test", + "run-lists.test", + "run-traits.test", + ] + (["run-match.test"] if sys.version_info >= (3, 10) else []) def fix_native_line_number(message: str, fnam: str, delta: int) -> str: From c482cec13db40f786af4c07898db6db26381b508 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Sat, 23 Nov 2024 16:41:54 -0500 Subject: [PATCH 13/21] Address PR comments --- mypyc/codegen/emitfunc.py | 26 +--------- mypyc/ir/rtypes.py | 5 +- mypyc/irbuild/prepare.py | 11 +++++ mypyc/test-data/run-valuetype.test | 77 ------------------------------ mypyc/test/test_run.py | 14 +----- 5 files changed, 18 insertions(+), 115 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index c238be4a65bd..ec9fae3d5c26 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -15,14 +15,7 @@ TYPE_VAR_PREFIX, ) from mypyc.ir.class_ir import ClassIR -from mypyc.ir.func_ir import ( - FUNC_CLASSMETHOD, - FUNC_NORMAL, - FUNC_STATICMETHOD, - FuncDecl, - FuncIR, - all_values, -) +from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values from mypyc.ir.ops import ( ERR_FALSE, NAMESPACE_MODULE, @@ -570,20 +563,6 @@ def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Va if method.decl.kind == FUNC_STATICMETHOD else [f"(PyObject *)Py_TYPE({obj})"] if method.decl.kind == FUNC_CLASSMETHOD else [obj] ) - need_box_obj = ( - method.decl.kind == FUNC_NORMAL - and rtype.is_unboxed - and not method.args[0].type.is_unboxed - ) - obj_boxed = "" - if need_box_obj: - # for cases where obj.method(...) is called and obj is unboxed, but method - # expects a boxed due inheritance or trait. e.g. obj is a value type - # but method comes from a parent which not - obj_boxed = self.temp_name() - self.emitter.emit_box(obj, obj_boxed, rtype, declare_dest=True) - obj_args = [obj_boxed] - args = ", ".join(obj_args + [self.reg(arg) for arg in op_args]) mtype = native_function_type(method, self.emitter) version = "_TRAIT" if rtype.class_ir.is_trait else "" @@ -608,9 +587,6 @@ def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Va ) ) - if need_box_obj: - self.emitter.emit_dec_ref(obj_boxed, RInstance(rtype.class_ir)) - def visit_inc_ref(self, op: IncRef) -> None: src = self.reg(op.src) self.emit_inc_ref(src, op.src.type) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index eaf7aa87d9f9..b3aa24c2ff3d 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -873,7 +873,6 @@ class RInstanceValue(RInstance): """ is_unboxed = True - is_refcounted = False def __init__(self, class_ir: ClassIR) -> None: super().__init__(class_ir) @@ -882,6 +881,10 @@ def __init__(self, class_ir: ClassIR) -> None: def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rinstance_value(self) + @property + def is_refcounted(self) -> bool: + return any(t.is_refcounted for t in self.class_ir.all_attributes().values()) + def __repr__(self) -> str: return "" % self.name diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index f644bfd2b3db..27f00848b695 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -156,6 +156,17 @@ def check_value_type( # Because the value type have semantic differences we can not just ignore it errors.error("Value types must be immutable and final", path, cdef.line) + for base in cdef.info.mro: + if base is cdef.info or (base.module_name == "builtins" and base.name == "object"): + continue + + module = module_by_fullname.get(base.module_name) + errors.error( + "Value types must not inherit from other types", + module.path if module else "", + cdef.line, + ) + for mtd_name in ( "__iter__", "__next__", diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test index 453924381d7a..0440d9d6efba 100644 --- a/mypyc/test-data/run-valuetype.test +++ b/mypyc/test-data/run-valuetype.test @@ -28,83 +28,6 @@ def test_value_type_operator_is() -> None: # however it must not raise an error. Vector2I(1, 2) is Vector2I(1, 2) -[case testValueTypeInheritance] -from typing import final, Final -from mypy_extensions import mypyc_attr, i32 - -class A: - def __init__(self, x: i32) -> None: - self.x: Final = x - - def method(self) -> i32: - return self.x + 4 - - def method2(self) -> i32: - return self.x + 4 - - @property - def prop(self) -> i32: - return self.x + 4 - - @property - def prop2(self) -> i32: - return self.x + 4 - -@final -@mypyc_attr(value_type=True) -class B(A): - def __init__(self, x: i32, y: i32) -> None: - super().__init__(x + 1) - self.y: Final = y - - def method2(self) -> i32: - return self.x + 7 - - @property - def prop2(self) -> i32: - return self.x + 7 - - -def test_value_type_inheritance() -> None: - # Tests with inheritance overriding methods and properties - assert A(1).x == 1 - assert B(1, 3).x == 2 - assert B(1, 3).y == 3 - assert A(1).method() == 5 - assert A(1).method2() == 5 - assert A(1).prop == 5 - assert A(1).prop2 == 5 - assert B(1, 3).method() == 6 - assert B(1, 3).method2() == 9 - assert B(1, 3).prop == 6 - assert B(1, 3).prop2 == 9 - - -def test_value_type_inheritance_is() -> None: - # however it must not raise an error. - assert not (B(1, 3) is A(1)) - assert not (A(1) is B(1, 3)) - assert B(1, 3) is not A(1) - assert A(1) is not B(1, 3) - -def f(a: A) -> i32: - return a.x - -def fcast(a: object) -> i32: - assert isinstance(a, A) - return a.x - -def fcast2(a: object) -> i32: - assert isinstance(a, B) - return a.x - -def test_value_type_boxing() -> None: - assert f(A(1)) == 1 - assert f(B(1, 3)) == 2 - assert fcast(A(1)) == 1 - assert fcast(B(1, 3)) == 2 - assert fcast2(B(1, 3)) == 2 - [case testValueTypeNest] from typing import final, Final from mypy_extensions import mypyc_attr, i32 diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 3c521da84999..99a43a7a841f 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -414,22 +414,12 @@ class TestRunStrictDunderTyping(TestRun): strict_dunder_typing = True test_name_suffix = "_dunder_typing" files = [ - "run-async.test", "run-bools.test", - "run-bytes.test", - "run-classes.test", "run-dunders.test", - "run-exceptions.test", "run-floats.test", - "run-functions.test", - "run-generators.test", - "run-i16.test", - "run-i32.test", - "run-i64.test", "run-integers.test", - "run-lists.test", - "run-traits.test", - ] + (["run-match.test"] if sys.version_info >= (3, 10) else []) + "run-strings.test", + ] def fix_native_line_number(message: str, fnam: str, delta: int) -> str: From 73ddcbee1bccf38cf76ba7370181780c742cc5a7 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Sat, 23 Nov 2024 17:24:15 -0500 Subject: [PATCH 14/21] Fix mypy issue --- mypyc/ir/rtypes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index b3aa24c2ff3d..dfdbfa422a63 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -885,6 +885,10 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: def is_refcounted(self) -> bool: return any(t.is_refcounted for t in self.class_ir.all_attributes().values()) + @is_refcounted.setter + def is_refcounted(self, value: bool) -> None: + raise NotImplementedError("is_refcounted is read-only for RInstanceValue") + def __repr__(self) -> str: return "" % self.name From 587cfa7aa324bfdaa90979cfa9d558ae7158bd65 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Sat, 23 Nov 2024 17:48:01 -0500 Subject: [PATCH 15/21] Fix refcounted definition --- mypyc/ir/rtypes.py | 30 +++++++++++++++++++++--------- mypyc/transform/value_type_init.py | 3 +-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index dfdbfa422a63..67920df0e36e 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -49,8 +49,12 @@ class RType: # # TODO: This shouldn't be specific to C or a string c_undefined: str - # If unboxed: does the unboxed version use reference counting? - is_refcounted = True + + @property + def is_refcounted(self) -> bool: + # If unboxed: does the unboxed version use reference counting? + return True + # C type; use Emitter.ctype() to access _ctype: str # If True, error/undefined value overlaps with a valid value. To @@ -204,7 +208,7 @@ def __init__( self.name = name self.is_unboxed = is_unboxed - self.is_refcounted = is_refcounted + self._is_refcounted = is_refcounted self.is_native_int = is_native_int self.is_signed = is_signed self._ctype = ctype @@ -233,6 +237,10 @@ def __init__( else: assert False, "Unrecognized ctype: %r" % ctype + @property + def is_refcounted(self) -> bool: + return self._is_refcounted + def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rprimitive(self) @@ -822,12 +830,17 @@ class RInstance(RType): is_unboxed = False - def __init__(self, class_ir: ClassIR) -> None: + def __init__(self, class_ir: ClassIR, is_refcounted: bool = True) -> None: # name is used for formatting the name in messages and debug output # so we want the fullname for precision. self.name = class_ir.fullname self.class_ir = class_ir self._ctype = "PyObject *" + self._is_refcounted = is_refcounted + + @property + def is_refcounted(self) -> bool: + return self._is_refcounted def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rinstance(self) @@ -885,10 +898,6 @@ def accept(self, visitor: RTypeVisitor[T]) -> T: def is_refcounted(self) -> bool: return any(t.is_refcounted for t in self.class_ir.all_attributes().values()) - @is_refcounted.setter - def is_refcounted(self, value: bool) -> None: - raise NotImplementedError("is_refcounted is read-only for RInstanceValue") - def __repr__(self) -> str: return "" % self.name @@ -1005,7 +1014,10 @@ def __init__(self, item_type: RType, length: int) -> None: self.item_type = item_type # Number of items self.length = length - self.is_refcounted = False + + @property + def is_refcounted(self) -> bool: + return False def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rarray(self) diff --git a/mypyc/transform/value_type_init.py b/mypyc/transform/value_type_init.py index 7dd953be1a63..61131a75a2b2 100644 --- a/mypyc/transform/value_type_init.py +++ b/mypyc/transform/value_type_init.py @@ -27,9 +27,8 @@ def patch_value_type_init_methods(ir: FuncIR, options: CompilerOptions) -> None: return # patch the type of the self parameter to be a reference to the value type - ref_type = RInstance(cl) # the refcounted flag is set to False because we only need to initialize the # attributes of the value type, but it is not expected to be refcounted - ref_type.is_refcounted = False + ref_type = RInstance(cl, is_refcounted=False) ir.args[0].type = ref_type ir.arg_regs[0].type = ref_type From 317228feefa5b8559874c82b16c88f3cf2890af2 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Sat, 23 Nov 2024 17:59:50 -0500 Subject: [PATCH 16/21] Fix refcounted definition --- mypyc/ir/rtypes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 67920df0e36e..1a3779ac2e37 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -646,7 +646,7 @@ class RTuple(RType): def __init__(self, types: list[RType]) -> None: self.name = "tuple" self.types = tuple(types) - self.is_refcounted = any(t.is_refcounted for t in self.types) + self._is_refcounted = any(t.is_refcounted for t in self.types) # Generate a unique id which is used in naming corresponding C identifiers. # This is necessary since C does not have anonymous structural type equivalence # in the same way python can just assign a Tuple[int, bool] to a Tuple[int, bool]. @@ -656,6 +656,10 @@ def __init__(self, types: list[RType]) -> None: self._ctype = f"{self.struct_name}" self.error_overlap = all(t.error_overlap for t in self.types) and bool(self.types) + @property + def is_refcounted(self) -> bool: + return self._is_refcounted + def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rtuple(self) From 6a434b4535b6b2ae77744db741991df706dfd2c9 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Sun, 24 Nov 2024 03:59:10 -0500 Subject: [PATCH 17/21] Use feature flag in INCOMPLETE_FEATURES --- mypy/options.py | 3 ++- mypyc/build.py | 7 ++++++- mypyc/codegen/emit.py | 11 ++++++++++- mypyc/irbuild/prepare.py | 2 +- mypyc/options.py | 3 +++ mypyc/test/test_run.py | 3 ++- 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/mypy/options.py b/mypy/options.py index d315d297e023..ae702edecd59 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -75,7 +75,8 @@ class BuildType: PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" INLINE_TYPEDDICT: Final = "InlineTypedDict" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT)) +MYPYC_VALUE_TYPES: Final = "MypycValueTypes" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT, MYPYC_VALUE_TYPES)) COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, NEW_GENERIC_SYNTAX)) diff --git a/mypyc/build.py b/mypyc/build.py index 6d59113ef872..9267c3d5877e 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -31,7 +31,7 @@ from mypy.errors import CompileError from mypy.fscache import FileSystemCache from mypy.main import process_options -from mypy.options import Options +from mypy.options import MYPYC_VALUE_TYPES, Options from mypy.util import write_junit_xml from mypyc.codegen import emitmodule from mypyc.common import RUNTIME_C_FILES, shared_lib_name @@ -418,6 +418,11 @@ def mypyc_build( paths, only_compile_paths, compiler_options, fscache ) + # Enable value types option via mypy options + compiler_options.experimental_value_types = ( + MYPYC_VALUE_TYPES in options.enable_incomplete_feature + ) + # We generate a shared lib if there are multiple modules or if any # of the modules are in package. (Because I didn't want to fuss # around with making the single module code handle packages.) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index cbffecb5bcac..f04d7dd9129d 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -526,6 +526,9 @@ def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_inc_ref(f"{dest}.f{i}", item_type) + elif isinstance(rtype, RInstanceValue): + for i, (attr, attr_type) in enumerate(rtype.class_ir.all_attributes().items()): + self.emit_inc_ref(f"{dest}.{self.attr(attr)}", attr_type) elif not rtype.is_unboxed: # Always inline, since this is a simple op self.emit_line("CPy_INCREF(%s);" % dest) @@ -1089,7 +1092,7 @@ def emit_box( attr_name = self.attr(attr) self.emit_line(f"{temp_dest}->{attr_name} = {src}.{attr_name};", ann="box") if attr_type.is_refcounted: - self.emit_inc_ref(temp_dest, attr_type) + self.emit_inc_ref(f"{temp_dest}->{attr_name}", attr_type) self.emit_line(f"{declaration}{dest} = (PyObject *){temp_dest};") else: @@ -1131,6 +1134,9 @@ def emit_gc_visit(self, target: str, rtype: RType) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_gc_visit(f"{target}.f{i}", item_type) + elif isinstance(rtype, RInstanceValue): + for i, (attr, attr_type) in enumerate(rtype.class_ir.all_attributes().items()): + self.emit_gc_visit(f"{target}.{self.attr(attr)}", attr_type) elif self.ctype(rtype) == "PyObject *": # The simplest case. self.emit_line(f"Py_VISIT({target});") @@ -1155,6 +1161,9 @@ def emit_gc_clear(self, target: str, rtype: RType) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_gc_clear(f"{target}.f{i}", item_type) + elif isinstance(rtype, RInstanceValue): + for i, (attr, attr_type) in enumerate(rtype.class_ir.all_attributes().items()): + self.emit_gc_clear(f"{target}.{self.attr(attr)}", attr_type) elif self.ctype(rtype) == "PyObject *" and self.c_undefined_value(rtype) == "NULL": # The simplest case. self.emit_line(f"Py_CLEAR({target});") diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 27f00848b695..a32a55fbd9b5 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -91,7 +91,7 @@ def build_type_map( is_abstract=cdef.info.is_abstract, is_final_class=cdef.info.is_final, is_ext_class=is_extension_class(cdef), - is_value_type=is_value_type(cdef), + is_value_type=is_value_type(cdef) and options.experimental_value_types, ) if class_ir.is_value_type: diff --git a/mypyc/options.py b/mypyc/options.py index 24e68163bb11..4db72abfb0a9 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -38,3 +38,6 @@ def __init__( # will assume the return type of the method strictly, which can lead to # more optimization opportunities. self.strict_dunders_typing = strict_dunder_typing + # Enable value types for the generated code. This option is experimental until + # the feature reference get removed from INCOMPLETE_FEATURES. + self.experimental_value_types = True # overridden by the mypy command line option diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 99a43a7a841f..984c62c3a23d 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -15,7 +15,7 @@ from mypy import build from mypy.errors import CompileError -from mypy.options import Options +from mypy.options import MYPYC_VALUE_TYPES, Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase from mypy.test.helpers import assert_module_equivalence, perform_file_operations @@ -199,6 +199,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> options.preserve_asts = True options.allow_empty_bodies = True options.incremental = self.separate + options.enable_incomplete_feature.append(MYPYC_VALUE_TYPES) # Avoid checking modules/packages named 'unchecked', to provide a way # to test interacting with code we don't have types for. From 01a8b30d93ec6a5043d3a632997e1092d0e0a9ef Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Sun, 24 Nov 2024 11:53:00 -0500 Subject: [PATCH 18/21] Add new tests --- mypyc/test-data/irbuild-valuetype.test | 62 ++++++++++++++++++++++++++ mypyc/test-data/valuetype-errors.test | 35 +++++++++++++++ mypyc/test/test_irbuild.py | 2 + mypyc/test/test_valuetype.py | 38 ++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 mypyc/test-data/irbuild-valuetype.test create mode 100644 mypyc/test-data/valuetype-errors.test create mode 100644 mypyc/test/test_valuetype.py diff --git a/mypyc/test-data/irbuild-valuetype.test b/mypyc/test-data/irbuild-valuetype.test new file mode 100644 index 000000000000..7a41049f1073 --- /dev/null +++ b/mypyc/test-data/irbuild-valuetype.test @@ -0,0 +1,62 @@ +[case testValueTypeBasic] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32) -> None: + self.x: Final = x + +@final +class Vector2Ir: + def __init__(self, x: i32) -> None: + self.x: Final = x + +def test_rt() -> None: + l1 = [Vector2I(1), Vector2I(1)] # require boxing + l2 = [Vector2Ir(1), Vector2Ir(1)] # do NOT require boxing + +[out] +def Vector2I.__init__(self, x): + self :: __main__.Vector2I + x :: i32 +L0: + self.x = x + return 1 +def Vector2Ir.__init__(self, x): + self :: __main__.Vector2Ir + x :: i32 +L0: + self.x = x + return 1 +def test_rt(): + r0, r1 :: __main__.Vector2I + r2 :: list + r3, r4 :: object + r5 :: ptr + l1 :: list + r6, r7 :: __main__.Vector2Ir + r8 :: list + r9 :: ptr + l2 :: list +L0: + r0 = Vector2I(1) + r1 = Vector2I(1) + r2 = PyList_New(2) + r3 = box(__main__.Vector2I, r0) + r4 = box(__main__.Vector2I, r1) + r5 = list_items r2 + buf_init_item r5, 0, r3 + buf_init_item r5, 1, r4 + keep_alive r2 + l1 = r2 + r6 = Vector2Ir(1) + r7 = Vector2Ir(1) + r8 = PyList_New(2) + r9 = list_items r8 + buf_init_item r9, 0, r6 + buf_init_item r9, 1, r7 + keep_alive r8 + l2 = r8 + return 1 diff --git a/mypyc/test-data/valuetype-errors.test b/mypyc/test-data/valuetype-errors.test new file mode 100644 index 000000000000..651076b9c721 --- /dev/null +++ b/mypyc/test-data/valuetype-errors.test @@ -0,0 +1,35 @@ +[case testValueTypeNotFinal] +from mypy_extensions import mypyc_attr + +@mypyc_attr(value_type=True) +class A: + pass + +[out] +main:4: error: Value types must be immutable and final + +[case testValueTypeHasNotParent] +from mypy_extensions import mypyc_attr +from typing import final + +class A: pass + +@final +@mypyc_attr(value_type=True) +class B(A): pass + +[out] +main:8: error: Value types must not inherit from other types + +[case testValueTypeForbiddenMethods] +from typing import final +from mypy_extensions import mypyc_attr + +@final +@mypyc_attr(value_type=True) +class A: + def __iter__(self) -> None: + pass + +[out] +main:7: error: Value types must not define method '__iter__' diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 5b3f678d8f17..1f2400095fa7 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -44,6 +44,7 @@ "irbuild-i32.test", "irbuild-i16.test", "irbuild-u8.test", + "irbuild-valuetype.test", "irbuild-vectorcall.test", "irbuild-unreachable.test", "irbuild-isinstance.test", @@ -69,6 +70,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if options is None: # Skipped test case return + assert options.experimental_value_types with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): expected_output = remove_comment_lines(testcase.output) expected_output = replace_word_size(expected_output) diff --git a/mypyc/test/test_valuetype.py b/mypyc/test/test_valuetype.py new file mode 100644 index 000000000000..ba03246b740e --- /dev/null +++ b/mypyc/test/test_valuetype.py @@ -0,0 +1,38 @@ +"""Test runner for value type test cases.""" + +from __future__ import annotations + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.test.testutil import ( + MypycDataSuite, + build_ir_for_single_file, + infer_ir_build_options_from_test_name, +) + +files = ["valuetype-errors.test"] + + +class TestValueTypeCompileErrors(MypycDataSuite): + """Negative cases which emit error on compile.""" + + files = files + base_path = test_temp_dir + + def run_case(self, testcase: DataDrivenTestCase) -> None: + """Perform a runtime checking transformation test case.""" + options = infer_ir_build_options_from_test_name(testcase.name) + if options is None: + # Skipped test case + return + + assert options.experimental_value_types + try: + build_ir_for_single_file(testcase.input, options) + except CompileError as e: + actual = "\n".join(e.messages).strip() + expected = "\n".join(testcase.output).strip() + assert actual == expected + else: + assert False, "Expected CompileError" From 2ee12a032058719619c68433f6df91a37df2a3e2 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Mon, 25 Nov 2024 07:19:52 -0500 Subject: [PATCH 19/21] Add new tests with tuples --- mypyc/__main__.py | 9 ++- mypyc/build.py | 11 +++- mypyc/codegen/emit.py | 7 ++- mypyc/ir/rtypes.py | 3 + mypyc/test-data/irbuild-valuetype.test | 82 +++++++++++++++++++++++++- mypyc/test-data/run-valuetype.test | 65 ++++++++++++++++++++ 6 files changed, 169 insertions(+), 8 deletions(-) diff --git a/mypyc/__main__.py b/mypyc/__main__.py index 653199e0fb55..d72cb7fe1c4a 100644 --- a/mypyc/__main__.py +++ b/mypyc/__main__.py @@ -24,7 +24,7 @@ from mypyc.build import mypycify setup(name='mypyc_output', - ext_modules=mypycify({}, opt_level="{}", debug_level="{}", strict_dunder_typing={}), + ext_modules=mypycify({}, opt_level="{}", debug_level="{}", strict_dunder_typing={}, compiler_type="{}"), ) """ @@ -39,10 +39,15 @@ def main() -> None: opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") strict_dunder_typing = bool(int(os.getenv("MYPYC_STRICT_DUNDER_TYPING", "0"))) + compiler = os.getenv("MYPYC_COMPILER", "") setup_file = os.path.join(build_dir, "setup.py") with open(setup_file, "w") as f: - f.write(setup_format.format(sys.argv[1:], opt_level, debug_level, strict_dunder_typing)) + f.write( + setup_format.format( + sys.argv[1:], opt_level, debug_level, strict_dunder_typing, compiler + ) + ) # We don't use run_setup (like we do in the test suite) because it throws # away the error code from distutils, and we don't care about the slight diff --git a/mypyc/build.py b/mypyc/build.py index 9267c3d5877e..85ad3993d98e 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -476,6 +476,7 @@ def mypycify( target_dir: str | None = None, include_runtime_files: bool | None = None, strict_dunder_typing: bool = False, + compiler_type: str | None = None, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -501,7 +502,7 @@ def mypycify( separate: Should compiled modules be placed in separate extension modules. If False, all modules are placed in a single shared library. If True, every module is placed in its own library. - Otherwise separate should be a list of + Otherwise, separate should be a list of (file name list, optional shared library name) pairs specifying groups of files that should be placed in the same shared library (while all other modules will be placed in its own library). @@ -518,6 +519,9 @@ def mypycify( strict_dunder_typing: If True, force dunder methods to have the return type of the method strictly, which can lead to more optimization opportunities. Defaults to False. + compiler_type: The distutils compiler. If None or empty, the default compiler + is used. Some possible values are 'msvc', 'mingw32', 'unix'. + Defaults to None. """ # Figure out our configuration @@ -546,17 +550,18 @@ def mypycify( # Create a compiler object so we can make decisions based on what # compiler is being used. typeshed is missing some attributes on the # compiler object so we give it type Any - compiler: Any = ccompiler.new_compiler() + compiler: Any = ccompiler.new_compiler(compiler=compiler_type or None, verbose=verbose) sysconfig.customize_compiler(compiler) build_dir = compiler_options.target_dir cflags: list[str] = [] - if compiler.compiler_type == "unix": + if compiler.compiler_type in ("mingw32", "unix"): cflags += [ f"-O{opt_level}", f"-g{debug_level}", "-Werror", + "-Wno-missing-braces", "-Wno-unused-function", "-Wno-unused-label", "-Wno-unreachable-code", diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index f04d7dd9129d..11a30ead3d60 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -466,6 +466,9 @@ def tuple_undefined_check_cond( return self.tuple_undefined_check_cond( item_type, tuple_expr_in_c + f".f{i}", c_type_compare_val, compare ) + elif isinstance(item_type, RInstanceValue): + check = f"{tuple_expr_in_c}.f{i}.vtable {compare} NULL" + return check else: check = f"{tuple_expr_in_c}.f{i} {compare} {c_type_compare_val(item_type)}" if rtuple.error_overlap and check_exception: @@ -1135,7 +1138,7 @@ def emit_gc_visit(self, target: str, rtype: RType) -> None: for i, item_type in enumerate(rtype.types): self.emit_gc_visit(f"{target}.f{i}", item_type) elif isinstance(rtype, RInstanceValue): - for i, (attr, attr_type) in enumerate(rtype.class_ir.all_attributes().items()): + for attr, attr_type in rtype.class_ir.all_attributes().items(): self.emit_gc_visit(f"{target}.{self.attr(attr)}", attr_type) elif self.ctype(rtype) == "PyObject *": # The simplest case. @@ -1162,7 +1165,7 @@ def emit_gc_clear(self, target: str, rtype: RType) -> None: for i, item_type in enumerate(rtype.types): self.emit_gc_clear(f"{target}.f{i}", item_type) elif isinstance(rtype, RInstanceValue): - for i, (attr, attr_type) in enumerate(rtype.class_ir.all_attributes().items()): + for attr, attr_type in rtype.class_ir.all_attributes().items(): self.emit_gc_clear(f"{target}.{self.attr(attr)}", attr_type) elif self.ctype(rtype) == "PyObject *" and self.c_undefined_value(rtype) == "NULL": # The simplest case. diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 1a3779ac2e37..3028661fccb6 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -887,6 +887,9 @@ class RInstanceValue(RInstance): These are used to represent unboxed values of RInstance which match the Value Type constraints. + + No error overlap happens in the Value Type because it is represented + with vtable == NULL. """ is_unboxed = True diff --git a/mypyc/test-data/irbuild-valuetype.test b/mypyc/test-data/irbuild-valuetype.test index 7a41049f1073..fa78e632971b 100644 --- a/mypyc/test-data/irbuild-valuetype.test +++ b/mypyc/test-data/irbuild-valuetype.test @@ -15,7 +15,7 @@ class Vector2Ir: def test_rt() -> None: l1 = [Vector2I(1), Vector2I(1)] # require boxing - l2 = [Vector2Ir(1), Vector2Ir(1)] # do NOT require boxing + l2 = [Vector2Ir(1), Vector2Ir(1)] # already boxed [out] def Vector2I.__init__(self, x): @@ -60,3 +60,83 @@ L0: keep_alive r8 l2 = r8 return 1 + +[case testValueTypeInTuple] +from typing import final, Final, Tuple +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class Vector2I: + def __init__(self, x: i32) -> None: + self.x: Final = x + +def return_tuple() -> Tuple[Vector2I, Vector2I]: + return Vector2I(11), Vector2I(10) # no boxing required due target type + +module_level = Vector2I(12) +final_module_level: Final = Vector2I(13) + +def fn_module_level() -> None: + assert module_level.x == 12 # unbox required + +def fn_module_level_final() -> None: + assert final_module_level.x == 13 # no boxing required + +[out] +def Vector2I.__init__(self, x): + self :: __main__.Vector2I + x :: i32 +L0: + self.x = x + return 1 +def return_tuple(): + r0, r1 :: __main__.Vector2I + r2 :: tuple[__main__.Vector2I, __main__.Vector2I] +L0: + r0 = Vector2I(11) + r1 = Vector2I(10) + r2 = (r0, r1) + return r2 +def fn_module_level(): + r0 :: dict + r1 :: str + r2 :: object + r3 :: __main__.Vector2I + r4 :: i32 + r5 :: bit + r6 :: bool +L0: + r0 = __main__.globals :: static + r1 = 'module_level' + r2 = CPyDict_GetItem(r0, r1) + r3 = unbox(__main__.Vector2I, r2) + r4 = r3.x + r5 = r4 == 12 + if r5 goto L2 else goto L1 :: bool +L1: + r6 = raise AssertionError + unreachable +L2: + return 1 +def fn_module_level_final(): + r0 :: __main__.Vector2I + r1 :: bool + r2 :: i32 + r3 :: bit + r4 :: bool +L0: + r0 = __main__.final_module_level :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "final_module_level" was not set') + unreachable +L2: + r2 = r0.x + r3 = r2 == 13 + if r3 goto L4 else goto L3 :: bool +L3: + r4 = raise AssertionError + unreachable +L4: + return 1 diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test index 0440d9d6efba..ecdcbae7e1fe 100644 --- a/mypyc/test-data/run-valuetype.test +++ b/mypyc/test-data/run-valuetype.test @@ -257,3 +257,68 @@ def f() -> B: def test_value_type_lifetime() -> None: assert f().a.n == "test" + +[case testFixedLengthTupleWithValueType] +from typing import final, Final +from mypy_extensions import mypyc_attr, i32 + +@final +@mypyc_attr(value_type=True) +class A: + def __init__(self, x: i32) -> None: + self.x: Final = x + +def test_fixed_length_tuple_with_value_type() -> None: + a = A(12) + t: tuple[A, A] = (a, a) + assert t[0].x == 12 + assert t[1].x == 12 + +def mtd_return_tuple() -> tuple[A, A]: + a = A(12) + return (a, a) + +def test_fixed_length_tuple_with_value_type_mtd_return_tuple() -> None: + t = mtd_return_tuple() + assert t[0].x == 12 + assert t[1].x == 12 + +def mtd_pass_tuple(a: tuple[A, A]) -> i32: + return a[0].x + a[1].x + +def test_fixed_length_tuple_with_value_type_mtd_pass_tuple() -> None: + t: tuple[A, A] = (A(12), A(2)) + assert mtd_pass_tuple(t) == 14 + +def mtd_return_tuple_but_raise() -> tuple[A, A]: + raise ValueError("test") + +def test_raise_in_tuple() -> None: + try: + mtd_return_tuple_but_raise() + except ValueError as e: + assert str(e) == "test" + +def mtd_return_value_type_raise() -> A: + raise ValueError("test") + +def test_raise_in_value_type() -> None: + try: + mtd_return_tuple_but_raise() + except ValueError as e: + assert str(e) == "test" + +module_level = A(12) +final_module_level: Final = A(13) + +def test_module_level() -> None: + assert module_level.x == 12 + assert final_module_level.x == 13 + +def mtd_with_default_value_type(a: A = A(12)) -> i32: + return a.x + +def test_default_value_type() -> None: + assert mtd_with_default_value_type() == 12 + assert mtd_with_default_value_type(A(13)) == 13 + assert mtd_with_default_value_type(a=A(13)) == 13 From ca4a989ec3445cb2cd9016c9fe8a225c150f878d Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Mon, 25 Nov 2024 07:41:23 -0500 Subject: [PATCH 20/21] Fix tuple typing syntax for python 3.8 --- mypyc/test-data/run-valuetype.test | 38 +++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/mypyc/test-data/run-valuetype.test b/mypyc/test-data/run-valuetype.test index ecdcbae7e1fe..d84a7b9ac9bb 100644 --- a/mypyc/test-data/run-valuetype.test +++ b/mypyc/test-data/run-valuetype.test @@ -238,11 +238,12 @@ def test_cmp() -> None: assert 2 not in a [case testValueTypeLifetime] -from typing import final, Final, NamedTuple +from typing import final, Final from mypy_extensions import mypyc_attr -class A(NamedTuple): - n: str +class A: + def __init__(self, n: str) -> None: + self.n = n @final @mypyc_attr(value_type=True) @@ -258,8 +259,27 @@ def f() -> B: def test_value_type_lifetime() -> None: assert f().a.n == "test" + +def f2(a: A) -> B: + # A instance is passed as an argument + # should be keep alive after the function returns + return B(a) + +def test_value_type_lifetime2() -> None: + assert f2(A("test")).a.n == "test" + +def f3(a: A) -> B: + # A instance is passed as an argument and discarded here + # should be keep alive after the function returns + return B(A("other")) + +def test_value_type_lifetime3() -> None: + a = A("test") + assert f3(a).a.n == "other" + assert a.n == "test", "a instance should be keep alive" + [case testFixedLengthTupleWithValueType] -from typing import final, Final +from typing import final, Final, Tuple from mypy_extensions import mypyc_attr, i32 @final @@ -270,11 +290,11 @@ class A: def test_fixed_length_tuple_with_value_type() -> None: a = A(12) - t: tuple[A, A] = (a, a) + t: Tuple[A, A] = (a, a) assert t[0].x == 12 assert t[1].x == 12 -def mtd_return_tuple() -> tuple[A, A]: +def mtd_return_tuple() -> Tuple[A, A]: a = A(12) return (a, a) @@ -283,14 +303,14 @@ def test_fixed_length_tuple_with_value_type_mtd_return_tuple() -> None: assert t[0].x == 12 assert t[1].x == 12 -def mtd_pass_tuple(a: tuple[A, A]) -> i32: +def mtd_pass_tuple(a: Tuple[A, A]) -> i32: return a[0].x + a[1].x def test_fixed_length_tuple_with_value_type_mtd_pass_tuple() -> None: - t: tuple[A, A] = (A(12), A(2)) + t: Tuple[A, A] = (A(12), A(2)) assert mtd_pass_tuple(t) == 14 -def mtd_return_tuple_but_raise() -> tuple[A, A]: +def mtd_return_tuple_but_raise() -> Tuple[A, A]: raise ValueError("test") def test_raise_in_tuple() -> None: From f32f983ddeaaef918db91678259934a7982d0fe4 Mon Sep 17 00:00:00 2001 From: Jairo Velasco <1904410+jairov4@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:26:37 -0500 Subject: [PATCH 21/21] Add dec ref for value types --- mypyc/codegen/emit.py | 9 +++++++-- mypyc/irbuild/prepare.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 11a30ead3d60..028db891c341 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -530,8 +530,9 @@ def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None: for i, item_type in enumerate(rtype.types): self.emit_inc_ref(f"{dest}.f{i}", item_type) elif isinstance(rtype, RInstanceValue): - for i, (attr, attr_type) in enumerate(rtype.class_ir.all_attributes().items()): - self.emit_inc_ref(f"{dest}.{self.attr(attr)}", attr_type) + if rtype.is_refcounted: + for attr, attr_type in rtype.class_ir.all_attributes().items(): + self.emit_inc_ref(f"{dest}.{self.attr(attr)}", attr_type) elif not rtype.is_unboxed: # Always inline, since this is a simple op self.emit_line("CPy_INCREF(%s);" % dest) @@ -557,6 +558,10 @@ def emit_dec_ref( elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_dec_ref(f"{dest}.f{i}", item_type, is_xdec=is_xdec, rare=rare) + elif isinstance(rtype, RInstanceValue): + if rtype.is_refcounted: + for attr, attr_type in rtype.class_ir.all_attributes().items(): + self.emit_dec_ref(f"{dest}.{self.attr(attr)}", attr_type, is_xdec=is_xdec) elif not rtype.is_unboxed: if rare: self.emit_line(f"CPy_{x}DecRef({dest});") diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index a32a55fbd9b5..616b4d6d615e 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -175,6 +175,7 @@ def check_value_type( "__getitem__", "__setitem__", "__delitem__", + "__del__", ): mtd = cdef.info.get_method(mtd_name) if mtd is not None: