[mypyc] Support new syntax for generic functions and classes (PEP 695) (#17357)

Generate an implicit `Generic` base class for new-style generic classes.
For this to work, also create C statics that can be used to access type
variable objects (e.g. `T` or `Ts`) at runtime. These are needed when
evaluating base classes. Import `TypeVar` and friends from the `_typing`
C extension instead of `typing`, since the latter is pretty slow to
import, and we don't want to add a hidden new runtime dependency in case
the full `typing` module isn't needed.

Generic functions don't need any changes, since they don't support
indexing with a type, and type variable types aren't valid in runtime
contexts. Type erasure seems sufficient, especially considering that
mypyc doesn't support classes nested within functions. (I'm not 100%
sure about this though, and we might need to put function type variables
into statics eventually.)

Update builtins test fixtures used in mypyc tests to not defined type
variables such as `T`, since these leak into tests and can produce
unexpected or unrealistic results.

Ignore upper bounds and value restrictions. These are only used for type
checking. This should only affect introspection of type variables, which
isn't properly supported in compiled code anyway.

New type alias syntax is not supported in this PR.

Author: JukkaL

mypy/nodes.py

@@ -2535,8 +2535,9 @@ def __init__(
         default: mypy.types.Type,
         variance: int = INVARIANT,
         is_new_style: bool = False,
+        line: int = -1,
     ) -> None:
-        super().__init__()
+        super().__init__(line=line)
         self._name = name
         self._fullname = fullname
         self.upper_bound = upper_bound
@@ -2582,8 +2583,9 @@ def __init__(
         default: mypy.types.Type,
         variance: int = INVARIANT,
         is_new_style: bool = False,
+        line: int = -1,
     ) -> None:
-        super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
+        super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
         self.values = values
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
@@ -2661,8 +2663,9 @@ def __init__(
         default: mypy.types.Type,
         variance: int = INVARIANT,
         is_new_style: bool = False,
+        line: int = -1,
     ) -> None:
-        super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
+        super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
         self.tuple_fallback = tuple_fallback
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:

mypy/semanal.py

@@ -1709,7 +1709,7 @@ def push_type_args(
         self.scope_stack.append(SCOPE_ANNOTATION)
         tvs: list[tuple[str, TypeVarLikeExpr]] = []
         for p in type_args:
-            tv = self.analyze_type_param(p)
+            tv = self.analyze_type_param(p, context)
             if tv is None:
                 return None
             tvs.append((p.name, tv))
@@ -1732,7 +1732,9 @@ def is_defined_type_param(self, name: str) -> bool:
                     return True
         return False
 
-    def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
+    def analyze_type_param(
+        self, type_param: TypeParam, context: Context
+    ) -> TypeVarLikeExpr | None:
         fullname = self.qualified_name(type_param.name)
         if type_param.upper_bound:
             upper_bound = self.anal_type(type_param.upper_bound)
@@ -1757,6 +1759,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
                 default=default,
                 variance=VARIANCE_NOT_READY,
                 is_new_style=True,
+                line=context.line,
             )
         elif type_param.kind == PARAM_SPEC_KIND:
             return ParamSpecExpr(
@@ -1765,6 +1768,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
                 upper_bound=upper_bound,
                 default=default,
                 is_new_style=True,
+                line=context.line,
             )
         else:
             assert type_param.kind == TYPE_VAR_TUPLE_KIND
@@ -1777,6 +1781,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
                 tuple_fallback=tuple_fallback,
                 default=default,
                 is_new_style=True,
+                line=context.line,
             )
 
     def pop_type_args(self, type_args: list[TypeParam] | None) -> None:

mypyc/codegen/emitfunc.py

@@ -6,14 +6,22 @@
 
 from mypyc.analysis.blockfreq import frequently_executed_blocks
 from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer
-from mypyc.common import MODULE_PREFIX, NATIVE_PREFIX, REG_PREFIX, STATIC_PREFIX, TYPE_PREFIX
+from mypyc.common import (
+    MODULE_PREFIX,
+    NATIVE_PREFIX,
+    REG_PREFIX,
+    STATIC_PREFIX,
+    TYPE_PREFIX,
+    TYPE_VAR_PREFIX,
+)
 from mypyc.ir.class_ir import ClassIR
 from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values
 from mypyc.ir.ops import (
     ERR_FALSE,
     NAMESPACE_MODULE,
     NAMESPACE_STATIC,
     NAMESPACE_TYPE,
+    NAMESPACE_TYPE_VAR,
     Assign,
     AssignMulti,
     BasicBlock,
@@ -477,6 +485,7 @@ def visit_set_attr(self, op: SetAttr) -> None:
         NAMESPACE_STATIC: STATIC_PREFIX,
         NAMESPACE_TYPE: TYPE_PREFIX,
         NAMESPACE_MODULE: MODULE_PREFIX,
+        NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX,
     }
 
     def visit_load_static(self, op: LoadStatic) -> None:

mypyc/codegen/emitmodule.py

@@ -41,6 +41,7 @@
     PREFIX,
     RUNTIME_C_FILES,
     TOP_LEVEL_NAME,
+    TYPE_VAR_PREFIX,
     shared_lib_name,
     short_id_from_name,
     use_vectorcall,
@@ -590,6 +591,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
             self.declare_finals(module_name, module.final_names, declarations)
             for cl in module.classes:
                 generate_class_type_decl(cl, emitter, ext_declarations, declarations)
+            self.declare_type_vars(module_name, module.type_var_names, declarations)
             for fn in module.functions:
                 generate_function_declaration(fn, declarations)
 
@@ -1063,6 +1065,15 @@ def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None:
         symbol = emitter.static_name(identifier, None)
         self.declare_global("PyObject *", symbol)
 
+    def declare_type_vars(self, module: str, type_var_names: list[str], emitter: Emitter) -> None:
+        for name in type_var_names:
+            static_name = emitter.static_name(name, module, prefix=TYPE_VAR_PREFIX)
+            emitter.context.declarations[static_name] = HeaderDeclaration(
+                f"PyObject *{static_name};",
+                [f"PyObject *{static_name} = NULL;"],
+                needs_export=False,
+            )
+
 
 def sort_classes(classes: list[tuple[str, ClassIR]]) -> list[tuple[str, ClassIR]]:
     mod_name = {ir: name for name, ir in classes}

mypyc/common.py

@@ -13,6 +13,7 @@
 STATIC_PREFIX: Final = "CPyStatic_"  # Static variables (for literals etc.)
 TYPE_PREFIX: Final = "CPyType_"  # Type object struct
 MODULE_PREFIX: Final = "CPyModule_"  # Cached modules
+TYPE_VAR_PREFIX: Final = "CPyTypeVar_"  # Type variables when using new-style Python 3.12 syntax
 ATTR_PREFIX: Final = "_"  # Attributes
 
 ENV_ATTR_NAME: Final = "__mypyc_env__"

mypyc/ir/module_ir.py

@@ -21,12 +21,17 @@ def __init__(
         functions: list[FuncIR],
         classes: list[ClassIR],
         final_names: list[tuple[str, RType]],
+        type_var_names: list[str],
     ) -> None:
         self.fullname = fullname
         self.imports = imports.copy()
         self.functions = functions
         self.classes = classes
         self.final_names = final_names
+        # Names of C statics used for Python 3.12 type variable objects.
+        # These are only visible in the module that defined them, so no need
+        # to serialize.
+        self.type_var_names = type_var_names
 
     def serialize(self) -> JsonDict:
         return {
@@ -45,6 +50,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR:
             [ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]],
             [ClassIR.deserialize(c, ctx) for c in data["classes"]],
             [(k, deserialize_type(t, ctx)) for k, t in data["final_names"]],
+            [],
         )
 
 

mypyc/ir/ops.py

@@ -789,6 +789,9 @@ def accept(self, visitor: OpVisitor[T]) -> T:
 # Namespace for modules
 NAMESPACE_MODULE: Final = "module"
 
+# Namespace for Python 3.12 type variable objects (implicitly created TypeVar instances, etc.)
+NAMESPACE_TYPE_VAR: Final = "typevar"
+
 
 class LoadStatic(RegisterOp):
     """Load a static name (name :: static).

mypyc/irbuild/builder.py

@@ -69,6 +69,7 @@
 from mypyc.ir.func_ir import INVALID_FUNC_DEF, FuncDecl, FuncIR, FuncSignature, RuntimeArg
 from mypyc.ir.ops import (
     NAMESPACE_MODULE,
+    NAMESPACE_TYPE_VAR,
     Assign,
     BasicBlock,
     Branch,
@@ -179,6 +180,7 @@ def __init__(
         self.function_names: set[tuple[str | None, str]] = set()
         self.classes: list[ClassIR] = []
         self.final_names: list[tuple[str, RType]] = []
+        self.type_var_names: list[str] = []
         self.callable_class_names: set[str] = set()
         self.options = options
 
@@ -541,6 +543,21 @@ def load_final_static(
             error_msg=f'value for final name "{error_name}" was not set',
         )
 
+    def init_type_var(self, value: Value, name: str, line: int) -> None:
+        unique_name = name + "___" + str(line)
+        self.type_var_names.append(unique_name)
+        self.add(InitStatic(value, unique_name, self.module_name, namespace=NAMESPACE_TYPE_VAR))
+
+    def load_type_var(self, name: str, line: int) -> Value:
+        return self.add(
+            LoadStatic(
+                object_rprimitive,
+                name + "___" + str(line),
+                self.module_name,
+                namespace=NAMESPACE_TYPE_VAR,
+            )
+        )
+
     def load_literal_value(self, val: int | str | bytes | float | complex | bool) -> Value:
         """Load value of a final name, class-level attribute, or constant folded expression."""
         if isinstance(val, bool):

mypyc/irbuild/classdef.py

@@ -7,6 +7,9 @@
 from typing import Callable, Final
 
 from mypy.nodes import (
+    PARAM_SPEC_KIND,
+    TYPE_VAR_KIND,
+    TYPE_VAR_TUPLE_KIND,
     AssignmentStmt,
     CallExpr,
     ClassDef,
@@ -22,6 +25,7 @@
     StrExpr,
     TempNode,
     TypeInfo,
+    TypeParam,
     is_class_var,
 )
 from mypy.types import ENUM_REMOVED_PROPS, Instance, RawExpressionType, get_proper_type
@@ -63,9 +67,16 @@
 )
 from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator
 from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op
-from mypyc.primitives.generic_ops import py_hasattr_op, py_setattr_op
+from mypyc.primitives.generic_ops import (
+    iter_op,
+    next_op,
+    py_get_item_op,
+    py_hasattr_op,
+    py_setattr_op,
+)
 from mypyc.primitives.misc_ops import (
     dataclass_sleight_of_hand,
+    import_op,
     not_implemented_op,
     py_calc_meta_op,
     pytype_from_template_op,
@@ -405,8 +416,14 @@ def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None:
 def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
     # OK AND NOW THE FUN PART
     base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs
-    if base_exprs:
-        bases = [builder.accept(x) for x in base_exprs]
+    new_style_type_args = cdef.type_args
+    if new_style_type_args:
+        bases = [make_generic_base_class(builder, cdef.fullname, new_style_type_args, cdef.line)]
+    else:
+        bases = []
+
+    if base_exprs or new_style_type_args:
+        bases.extend([builder.accept(x) for x in base_exprs])
         tp_bases = builder.new_tuple(bases, cdef.line)
     else:
         tp_bases = builder.add(LoadErrorValue(object_rprimitive, is_borrowed=True))
@@ -453,6 +470,45 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
     return tp
 
 
+def make_generic_base_class(
+    builder: IRBuilder, fullname: str, type_args: list[TypeParam], line: int
+) -> Value:
+    """Construct Generic[...] base class object for a new-style generic class (Python 3.12)."""
+    mod = builder.call_c(import_op, [builder.load_str("_typing")], line)
+    tvs = []
+    type_var_imported: Value | None = None
+    for type_param in type_args:
+        unpack = False
+        if type_param.kind == TYPE_VAR_KIND:
+            if type_var_imported:
+                # Reuse previously imported value as a minor optimization
+                tvt = type_var_imported
+            else:
+                tvt = builder.py_get_attr(mod, "TypeVar", line)
+                type_var_imported = tvt
+        elif type_param.kind == TYPE_VAR_TUPLE_KIND:
+            tvt = builder.py_get_attr(mod, "TypeVarTuple", line)
+            unpack = True
+        else:
+            assert type_param.kind == PARAM_SPEC_KIND
+            tvt = builder.py_get_attr(mod, "ParamSpec", line)
+        tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line)
+        builder.init_type_var(tv, type_param.name, line)
+        if unpack:
+            # Evaluate *Ts for a TypeVarTuple
+            it = builder.call_c(iter_op, [tv], line)
+            tv = builder.call_c(next_op, [it], line)
+        tvs.append(tv)
+    gent = builder.py_get_attr(mod, "Generic", line)
+    if len(tvs) == 1:
+        arg = tvs[0]
+    else:
+        arg = builder.new_tuple(tvs, line)
+
+    base = builder.call_c(py_get_item_op, [gent, arg], line)
+    return base
+
+
 # Mypy uses these internally as base classes of TypedDict classes. These are
 # lies and don't have any runtime equivalent.
 MAGIC_TYPED_DICT_CLASSES: Final[tuple[str, ...]] = (

mypyc/irbuild/expression.py

@@ -44,6 +44,7 @@
     TupleExpr,
     TypeApplication,
     TypeInfo,
+    TypeVarLikeExpr,
     UnaryExpr,
     Var,
 )
@@ -106,6 +107,10 @@
 
 
 def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value:
+    if isinstance(expr.node, TypeVarLikeExpr) and expr.node.is_new_style:
+        # Reference to Python 3.12 implicit TypeVar/TupleVarTuple/... object.
+        # These are stored in C statics and not visible in Python namespaces.
+        return builder.load_type_var(expr.node.name, expr.node.line)
     if expr.node is None:
         builder.add(
             RaiseStandardError(