Experimental: allow inline/anonymous TypedDicts (#17457)

Fixes #9884

I was always a bit skeptical about this thing, since it feels more like
TypeScript than Python, but it is second most upvoted issue. Also (this
specific) implementation is like 60 lines of code plus tests, so why
not.

I know there is no PEP etc., but IMO this syntax is obvious and it just
works.
cc @JukkaL

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: ilevkivskyi

docs/source/command_line.rst

@@ -1055,6 +1055,27 @@ List of currently incomplete/experimental features:
          # Without PreciseTupleTypes: tuple[int, ...]
          # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int]
 
+* ``NewGenericSyntax``: this feature enables support for syntax defined
+  by :pep:`695`. For example:
+
+  .. code-block:: python
+
+     class Container[T]:  # defines a generic class
+         content: T
+
+     def first[T](items: list[T]) -> T:  # defines a generic function
+         return items[0]
+
+     type Items[T] = list[tuple[T, T]]  # defines a generic type alias
+
+* ``InlineTypedDict``: this feature enables non-standard syntax for inline
+  :ref:`TypedDicts <typeddict>`, for example:
+
+  .. code-block:: python
+
+     def test_values() -> {"int": int, "str": str}:
+         return {"int": 42, "str": "test"}
+
 
 Miscellaneous
 *************

docs/source/typed_dict.rst

@@ -248,3 +248,41 @@ section of the docs has a full description with an example, but in short, you wi
 need to give each TypedDict the same key where each value has a unique
 :ref:`Literal type <literal_types>`. Then, check that key to distinguish
 between your TypedDicts.
+
+Inline TypedDict types
+----------------------
+
+.. note::
+
+    This is an experimental (non-standard) feature. Use
+    ``--enable-incomplete-feature=InlineTypedDict`` to enable.
+
+Sometimes you may want to define a complex nested JSON schema, or annotate
+a one-off function that returns a TypedDict. In such cases it may be convenient
+to use inline TypedDict syntax. For example:
+
+.. code-block:: python
+
+    def test_values() -> {"int": int, "str": str}:
+        return {"int": 42, "str": "test"}
+
+    class Response(TypedDict):
+        status: int
+        msg: str
+        # Using inline syntax here avoids defining two additional TypedDicts.
+        content: {"items": list[{"key": str, "value": str}]}
+
+Inline TypedDicts can also by used as targets of type aliases, but due to
+ambiguity with a regular variables it is only allowed for (newer) explicit
+type alias forms:
+
+.. code-block:: python
+
+    from typing import TypeAlias
+
+    X = {"a": int, "b": int}  # creates a variable with type dict[str, type[int]]
+    Y: TypeAlias = {"a": int, "b": int}  # creates a type alias
+    type Z = {"a": int, "b": int}  # same as above (Python 3.12+ only)
+
+Also, due to incompatibility with runtime type-checking it is strongly recommended
+to *not* use inline syntax in union types.

mypy/checker.py

@@ -2971,7 +2971,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
             self.msg.annotation_in_unchecked_function(context=s)
 
     def check_type_alias_rvalue(self, s: AssignmentStmt) -> None:
-        alias_type = self.expr_checker.accept(s.rvalue)
+        with self.msg.filter_errors():
+            alias_type = self.expr_checker.accept(s.rvalue)
         self.store_type(s.lvalues[-1], alias_type)
 
     def check_assignment(
@@ -5311,7 +5312,8 @@ def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var,
                         del type_map[expr]
 
     def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None:
-        self.expr_checker.accept(o.value)
+        with self.msg.filter_errors():
+            self.expr_checker.accept(o.value)
 
     def make_fake_typeinfo(
         self,

mypy/exprtotype.py

@@ -4,9 +4,11 @@
 
 from mypy.fastparse import parse_type_string
 from mypy.nodes import (
+    MISSING_FALLBACK,
     BytesExpr,
     CallExpr,
     ComplexExpr,
+    DictExpr,
     EllipsisExpr,
     Expression,
     FloatExpr,
@@ -29,9 +31,11 @@
     AnyType,
     CallableArgument,
     EllipsisType,
+    Instance,
     ProperType,
     RawExpressionType,
     Type,
+    TypedDictType,
     TypeList,
     TypeOfAny,
     UnboundType,
@@ -55,7 +59,7 @@ def _extract_argument_name(expr: Expression) -> str | None:
 
 def expr_to_unanalyzed_type(
     expr: Expression,
-    options: Options | None = None,
+    options: Options,
     allow_new_syntax: bool = False,
     _parent: Expression | None = None,
     allow_unpack: bool = False,
@@ -67,6 +71,8 @@ def expr_to_unanalyzed_type(
 
     If allow_new_syntax is True, allow all type syntax independent of the target
     Python version (used in stubs).
+
+    # TODO: a lot of code here is duplicated in fastparse.py, refactor this.
     """
     # The `parent` parameter is used in recursive calls to provide context for
     # understanding whether an CallableArgument is ok.
@@ -116,7 +122,7 @@ def expr_to_unanalyzed_type(
     elif (
         isinstance(expr, OpExpr)
         and expr.op == "|"
-        and ((options and options.python_version >= (3, 10)) or allow_new_syntax)
+        and ((options.python_version >= (3, 10)) or allow_new_syntax)
     ):
         return UnionType(
             [
@@ -206,5 +212,26 @@ def expr_to_unanalyzed_type(
         return UnpackType(
             expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True
         )
+    elif isinstance(expr, DictExpr):
+        if not expr.items:
+            raise TypeTranslationError()
+        items: dict[str, Type] = {}
+        extra_items_from = []
+        for item_name, value in expr.items:
+            if not isinstance(item_name, StrExpr):
+                if item_name is None:
+                    extra_items_from.append(
+                        expr_to_unanalyzed_type(value, options, allow_new_syntax, expr)
+                    )
+                    continue
+                raise TypeTranslationError()
+            items[item_name.value] = expr_to_unanalyzed_type(
+                value, options, allow_new_syntax, expr
+            )
+        result = TypedDictType(
+            items, set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column
+        )
+        result.extra_items_from = extra_items_from
+        return result
     else:
         raise TypeTranslationError()

mypy/fastparse.py

@@ -17,6 +17,7 @@
     ARG_POS,
     ARG_STAR,
     ARG_STAR2,
+    MISSING_FALLBACK,
     PARAM_SPEC_KIND,
     TYPE_VAR_KIND,
     TYPE_VAR_TUPLE_KIND,
@@ -42,7 +43,6 @@
     EllipsisExpr,
     Expression,
     ExpressionStmt,
-    FakeInfo,
     FloatExpr,
     ForStmt,
     FuncDef,
@@ -116,6 +116,7 @@
     RawExpressionType,
     TupleType,
     Type,
+    TypedDictType,
     TypeList,
     TypeOfAny,
     UnboundType,
@@ -190,7 +191,6 @@ def ast3_parse(
 
 # There is no way to create reasonable fallbacks at this stage,
 # they must be patched later.
-MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal")
 _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1)
 
 TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)")
@@ -2106,6 +2106,22 @@ def visit_Tuple(self, n: ast3.Tuple) -> Type:
             column=self.convert_column(n.col_offset),
         )
 
+    def visit_Dict(self, n: ast3.Dict) -> Type:
+        if not n.keys:
+            return self.invalid_type(n)
+        items: dict[str, Type] = {}
+        extra_items_from = []
+        for item_name, value in zip(n.keys, n.values):
+            if not isinstance(item_name, ast3.Constant) or not isinstance(item_name.value, str):
+                if item_name is None:
+                    extra_items_from.append(self.visit(value))
+                    continue
+                return self.invalid_type(n)
+            items[item_name.value] = self.visit(value)
+        result = TypedDictType(items, set(), _dummy_fallback, n.lineno, n.col_offset)
+        result.extra_items_from = extra_items_from
+        return result
+
     # Attribute(expr value, identifier attr, expr_context ctx)
     def visit_Attribute(self, n: Attribute) -> Type:
         before_dot = self.visit(n.value)

mypy/message_registry.py

@@ -138,6 +138,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
 TYPEDDICT_KEY_MUST_BE_STRING_LITERAL: Final = ErrorMessage(
     "Expected TypedDict key to be string literal"
 )
+TYPEDDICT_OVERRIDE_MERGE: Final = 'Overwriting TypedDict field "{}" while merging'
 MALFORMED_ASSERT: Final = ErrorMessage("Assertion is always true, perhaps remove parentheses?")
 DUPLICATE_TYPE_SIGNATURES: Final = ErrorMessage("Function has duplicate type signatures")
 DESCRIPTOR_SET_NOT_CALLABLE: Final = ErrorMessage("{}.__set__ is not callable")

mypy/nodes.py

@@ -3480,6 +3480,7 @@ def __getattribute__(self, attr: str) -> type:
 VAR_NO_INFO: Final[TypeInfo] = FakeInfo("Var is lacking info")
 CLASSDEF_NO_INFO: Final[TypeInfo] = FakeInfo("ClassDef is lacking info")
 FUNC_NO_INFO: Final[TypeInfo] = FakeInfo("FuncBase for non-methods lack info")
+MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal")
 
 
 class TypeAlias(SymbolNode):

mypy/options.py

@@ -74,7 +74,8 @@ class BuildType:
 UNPACK: Final = "Unpack"
 PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes"
 NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax"
-INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX))
+INLINE_TYPEDDICT: Final = "InlineTypedDict"
+INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX, INLINE_TYPEDDICT))
 COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK))
 
 

mypy/semanal_typeddict.py

@@ -8,6 +8,7 @@
 from mypy.errorcodes import ErrorCode
 from mypy.expandtype import expand_type
 from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type
+from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE
 from mypy.messages import MessageBuilder
 from mypy.nodes import (
     ARG_NAMED,
@@ -216,7 +217,7 @@ def add_keys_and_types_from_base(
             valid_items = self.map_items_to_base(valid_items, tvars, base_args)
         for key in base_items:
             if key in keys:
-                self.fail(f'Overwriting TypedDict field "{key}" while merging', ctx)
+                self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx)
         keys.extend(valid_items.keys())
         types.extend(valid_items.values())
         required_keys.update(base_typed_dict.required_keys)
@@ -507,17 +508,7 @@ def parse_typeddict_fields_with_types(
                     field_type_expr, self.options, self.api.is_stub_file
                 )
             except TypeTranslationError:
-                if (
-                    isinstance(field_type_expr, CallExpr)
-                    and isinstance(field_type_expr.callee, RefExpr)
-                    and field_type_expr.callee.fullname in TPDICT_NAMES
-                ):
-                    self.fail_typeddict_arg(
-                        "Inline TypedDict types not supported; use assignment to define TypedDict",
-                        field_type_expr,
-                    )
-                else:
-                    self.fail_typeddict_arg("Invalid field type", field_type_expr)
+                self.fail_typeddict_arg("Use dict literal for nested TypedDict", field_type_expr)
                 return [], [], False
             analyzed = self.api.anal_type(
                 type,

mypy/typeanal.py

@@ -10,7 +10,11 @@
 from mypy import errorcodes as codes, message_registry, nodes
 from mypy.errorcodes import ErrorCode
 from mypy.expandtype import expand_type
-from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE
+from mypy.message_registry import (
+    INVALID_PARAM_SPEC_LOCATION,
+    INVALID_PARAM_SPEC_LOCATION_NOTE,
+    TYPEDDICT_OVERRIDE_MERGE,
+)
 from mypy.messages import (
     MessageBuilder,
     format_type,
@@ -25,6 +29,7 @@
     ARG_POS,
     ARG_STAR,
     ARG_STAR2,
+    MISSING_FALLBACK,
     SYMBOL_FUNCBASE_TYPES,
     ArgKind,
     Context,
@@ -43,7 +48,7 @@
     check_arg_names,
     get_nongen_builtins,
 )
-from mypy.options import Options
+from mypy.options import INLINE_TYPEDDICT, Options
 from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface
 from mypy.semanal_shared import (
     SemanticAnalyzerCoreInterface,
@@ -1220,10 +1225,45 @@ def visit_tuple_type(self, t: TupleType) -> Type:
         return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line)
 
     def visit_typeddict_type(self, t: TypedDictType) -> Type:
-        items = {
-            item_name: self.anal_type(item_type) for (item_name, item_type) in t.items.items()
-        }
-        return TypedDictType(items, set(t.required_keys), t.fallback)
+        req_keys = set()
+        items = {}
+        for item_name, item_type in t.items.items():
+            analyzed = self.anal_type(item_type, allow_required=True)
+            if isinstance(analyzed, RequiredType):
+                if analyzed.required:
+                    req_keys.add(item_name)
+                analyzed = analyzed.item
+            else:
+                # Keys are required by default.
+                req_keys.add(item_name)
+            items[item_name] = analyzed
+        if t.fallback.type is MISSING_FALLBACK:  # anonymous/inline TypedDict
+            if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature:
+                self.fail(
+                    "Inline TypedDict is experimental,"
+                    " must be enabled with --enable-incomplete-feature=InlineTypedDict",
+                    t,
+                )
+            required_keys = req_keys
+            fallback = self.named_type("typing._TypedDict")
+            for typ in t.extra_items_from:
+                analyzed = self.analyze_type(typ)
+                p_analyzed = get_proper_type(analyzed)
+                if not isinstance(p_analyzed, TypedDictType):
+                    if not isinstance(p_analyzed, (AnyType, PlaceholderType)):
+                        self.fail("Can only merge-in other TypedDict", t, code=codes.VALID_TYPE)
+                    continue
+                for sub_item_name, sub_item_type in p_analyzed.items.items():
+                    if sub_item_name in items:
+                        self.fail(TYPEDDICT_OVERRIDE_MERGE.format(sub_item_name), t)
+                        continue
+                    items[sub_item_name] = sub_item_type
+                    if sub_item_name in p_analyzed.required_keys:
+                        req_keys.add(sub_item_name)
+        else:
+            required_keys = t.required_keys
+            fallback = t.fallback
+        return TypedDictType(items, required_keys, fallback, t.line, t.column)
 
     def visit_raw_expression_type(self, t: RawExpressionType) -> Type:
         # We should never see a bare Literal. We synthesize these raw literals
@@ -1761,11 +1801,12 @@ def anal_type(
         allow_param_spec: bool = False,
         allow_unpack: bool = False,
         allow_ellipsis: bool = False,
+        allow_required: bool = False,
     ) -> Type:
         if nested:
             self.nesting_level += 1
         old_allow_required = self.allow_required
-        self.allow_required = False
+        self.allow_required = allow_required
         old_allow_ellipsis = self.allow_ellipsis
         self.allow_ellipsis = allow_ellipsis
         old_allow_unpack = self.allow_unpack