bpo-46417: Py_Finalize() clears static types (GH-30743)

Add _PyTypes_FiniTypes() best-effort function to clear static types:
don't deallocate a type if it still has subclasses.

remove_subclass() now sets tp_subclasses to NULL when removing the
last subclass.

Author: vstinner

Include/internal/pycore_typeobject.h

@@ -13,6 +13,7 @@ extern "C" {
 
 extern PyStatus _PyTypes_InitState(PyInterpreterState *);
 extern PyStatus _PyTypes_InitTypes(PyInterpreterState *);
+extern void _PyTypes_FiniTypes(PyInterpreterState *);
 extern void _PyTypes_Fini(PyInterpreterState *);
 
 

Objects/object.c

@@ -1837,6 +1837,94 @@ _PyTypes_InitState(PyInterpreterState *interp)
     return _PyStatus_OK();
 }
 
+
+static PyTypeObject* static_types[] = {
+    // base types
+    &PyAsyncGen_Type,
+    &PyBool_Type,
+    &PyByteArrayIter_Type,
+    &PyByteArray_Type,
+    &PyCFunction_Type,
+    &PyCallIter_Type,
+    &PyCapsule_Type,
+    &PyCell_Type,
+    &PyClassMethodDescr_Type,
+    &PyClassMethod_Type,
+    &PyCode_Type,
+    &PyComplex_Type,
+    &PyCoro_Type,
+    &PyDictItems_Type,
+    &PyDictIterItem_Type,
+    &PyDictIterKey_Type,
+    &PyDictIterValue_Type,
+    &PyDictKeys_Type,
+    &PyDictProxy_Type,
+    &PyDictRevIterItem_Type,
+    &PyDictRevIterKey_Type,
+    &PyDictRevIterValue_Type,
+    &PyDictValues_Type,
+    &PyDict_Type,
+    &PyEllipsis_Type,
+    &PyEnum_Type,
+    &PyFrame_Type,
+    &PyFrozenSet_Type,
+    &PyFunction_Type,
+    &PyGen_Type,
+    &PyGetSetDescr_Type,
+    &PyInstanceMethod_Type,
+    &PyListIter_Type,
+    &PyListRevIter_Type,
+    &PyList_Type,
+    &PyLongRangeIter_Type,
+    &PyMemberDescr_Type,
+    &PyMemoryView_Type,
+    &PyMethodDescr_Type,
+    &PyMethod_Type,
+    &PyModuleDef_Type,
+    &PyModule_Type,
+    &PyODictIter_Type,
+    &PyPickleBuffer_Type,
+    &PyProperty_Type,
+    &PyRangeIter_Type,
+    &PyRange_Type,
+    &PyReversed_Type,
+    &PySTEntry_Type,
+    &PySeqIter_Type,
+    &PySetIter_Type,
+    &PySet_Type,
+    &PySlice_Type,
+    &PyStaticMethod_Type,
+    &PyStdPrinter_Type,
+    &PySuper_Type,
+    &PyTraceBack_Type,
+    &PyWrapperDescr_Type,
+    &Py_GenericAliasType,
+    &_PyAnextAwaitable_Type,
+    &_PyAsyncGenASend_Type,
+    &_PyAsyncGenAThrow_Type,
+    &_PyAsyncGenWrappedValue_Type,
+    &_PyCoroWrapper_Type,
+    &_PyInterpreterID_Type,
+    &_PyManagedBuffer_Type,
+    &_PyMethodWrapper_Type,
+    &_PyNamespace_Type,
+    &_PyNone_Type,
+    &_PyNotImplemented_Type,
+    &_PyUnion_Type,
+    &_PyWeakref_CallableProxyType,
+    &_PyWeakref_ProxyType,
+    &_PyWeakref_RefType,
+
+    // subclasses: _PyTypes_FiniTypes() deallocates them before their base
+    // class
+    &PyCMethod_Type,      // base=&PyCFunction_Type
+    &PyODictItems_Type,   // base=&PyDictItems_Type
+    &PyODictKeys_Type,    // base=&PyDictKeys_Type
+    &PyODictValues_Type,  // base=&PyDictValues_Type
+    &PyODict_Type,        // base=&PyDict_Type
+};
+
+
 PyStatus
 _PyTypes_InitTypes(PyInterpreterState *interp)
 {
@@ -1858,91 +1946,44 @@ _PyTypes_InitTypes(PyInterpreterState *interp)
     assert(PyType_Type.tp_base == &PyBaseObject_Type);
 
     // All other static types (unless initialized elsewhere)
-    INIT_TYPE(PyAsyncGen_Type);
-    INIT_TYPE(PyBool_Type);
-    INIT_TYPE(PyByteArrayIter_Type);
-    INIT_TYPE(PyByteArray_Type);
-    INIT_TYPE(PyCFunction_Type);
-    INIT_TYPE(PyCMethod_Type);
-    INIT_TYPE(PyCallIter_Type);
-    INIT_TYPE(PyCapsule_Type);
-    INIT_TYPE(PyCell_Type);
-    INIT_TYPE(PyClassMethodDescr_Type);
-    INIT_TYPE(PyClassMethod_Type);
-    INIT_TYPE(PyCode_Type);
-    INIT_TYPE(PyComplex_Type);
-    INIT_TYPE(PyCoro_Type);
-    INIT_TYPE(PyDictItems_Type);
-    INIT_TYPE(PyDictIterItem_Type);
-    INIT_TYPE(PyDictIterKey_Type);
-    INIT_TYPE(PyDictIterValue_Type);
-    INIT_TYPE(PyDictKeys_Type);
-    INIT_TYPE(PyDictProxy_Type);
-    INIT_TYPE(PyDictRevIterItem_Type);
-    INIT_TYPE(PyDictRevIterKey_Type);
-    INIT_TYPE(PyDictRevIterValue_Type);
-    INIT_TYPE(PyDictValues_Type);
-    INIT_TYPE(PyDict_Type);
-    INIT_TYPE(PyEllipsis_Type);
-    INIT_TYPE(PyEnum_Type);
-    INIT_TYPE(PyFrame_Type);
-    INIT_TYPE(PyFrozenSet_Type);
-    INIT_TYPE(PyFunction_Type);
-    INIT_TYPE(PyGen_Type);
-    INIT_TYPE(PyGetSetDescr_Type);
-    INIT_TYPE(PyInstanceMethod_Type);
-    INIT_TYPE(PyListIter_Type);
-    INIT_TYPE(PyListRevIter_Type);
-    INIT_TYPE(PyList_Type);
-    INIT_TYPE(PyLongRangeIter_Type);
-    INIT_TYPE(PyMemberDescr_Type);
-    INIT_TYPE(PyMemoryView_Type);
-    INIT_TYPE(PyMethodDescr_Type);
-    INIT_TYPE(PyMethod_Type);
-    INIT_TYPE(PyModuleDef_Type);
-    INIT_TYPE(PyModule_Type);
-    INIT_TYPE(PyODictItems_Type);
-    INIT_TYPE(PyODictIter_Type);
-    INIT_TYPE(PyODictKeys_Type);
-    INIT_TYPE(PyODictValues_Type);
-    INIT_TYPE(PyODict_Type);
-    INIT_TYPE(PyPickleBuffer_Type);
-    INIT_TYPE(PyProperty_Type);
-    INIT_TYPE(PyRangeIter_Type);
-    INIT_TYPE(PyRange_Type);
-    INIT_TYPE(PyReversed_Type);
-    INIT_TYPE(PySTEntry_Type);
-    INIT_TYPE(PySeqIter_Type);
-    INIT_TYPE(PySetIter_Type);
-    INIT_TYPE(PySet_Type);
-    INIT_TYPE(PySlice_Type);
-    INIT_TYPE(PyStaticMethod_Type);
-    INIT_TYPE(PyStdPrinter_Type);
-    INIT_TYPE(PySuper_Type);
-    INIT_TYPE(PyTraceBack_Type);
-    INIT_TYPE(PyWrapperDescr_Type);
-    INIT_TYPE(Py_GenericAliasType);
-    INIT_TYPE(_PyAnextAwaitable_Type);
-    INIT_TYPE(_PyAsyncGenASend_Type);
-    INIT_TYPE(_PyAsyncGenAThrow_Type);
-    INIT_TYPE(_PyAsyncGenWrappedValue_Type);
-    INIT_TYPE(_PyCoroWrapper_Type);
-    INIT_TYPE(_PyInterpreterID_Type);
-    INIT_TYPE(_PyManagedBuffer_Type);
-    INIT_TYPE(_PyMethodWrapper_Type);
-    INIT_TYPE(_PyNamespace_Type);
-    INIT_TYPE(_PyNone_Type);
-    INIT_TYPE(_PyNotImplemented_Type);
-    INIT_TYPE(_PyWeakref_CallableProxyType);
-    INIT_TYPE(_PyWeakref_ProxyType);
-    INIT_TYPE(_PyWeakref_RefType);
-    INIT_TYPE(_PyUnion_Type);
+    for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) {
+        PyTypeObject *type = static_types[i];
+        if (PyType_Ready(type) < 0) {
+            return _PyStatus_ERR("Can't initialize types");
+        }
+    }
 
     return _PyStatus_OK();
 #undef INIT_TYPE
 }
 
 
+// Best-effort function clearing static types.
+//
+// Don't deallocate a type if it still has subclasses. If a Py_Finalize()
+// sub-function is interrupted by CTRL+C or fails with MemoryError, some
+// subclasses are not cleared properly. Leave the static type unchanged in this
+// case.
+void
+_PyTypes_FiniTypes(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return;
+    }
+
+    // Deallocate types in the reverse order to deallocate subclasses before
+    // their base classes.
+    for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) {
+        PyTypeObject *type = static_types[i];
+        // Cannot delete a type if it still has subclasses
+        if (type->tp_subclasses != NULL) {
+            continue;
+        }
+        _PyStaticType_Dealloc(type);
+    }
+}
+
+
 void
 _Py_NewReference(PyObject *op)
 {

Objects/typeobject.c

@@ -4071,6 +4071,18 @@ extern void
 _PyDictKeys_DecRef(PyDictKeysObject *keys);
 
 
+static void
+type_dealloc_common(PyTypeObject *type)
+{
+    PyObject *tp, *val, *tb;
+    PyErr_Fetch(&tp, &val, &tb);
+    remove_all_subclasses(type, type->tp_bases);
+    PyErr_Restore(tp, val, tb);
+
+    PyObject_ClearWeakRefs((PyObject *)type);
+}
+
+
 void
 _PyStaticType_Dealloc(PyTypeObject *type)
 {
@@ -4079,34 +4091,34 @@ _PyStaticType_Dealloc(PyTypeObject *type)
     // and a type must no longer be used once it's deallocated.
     assert(type->tp_subclasses == NULL);
 
+    type_dealloc_common(type);
+
     Py_CLEAR(type->tp_dict);
     Py_CLEAR(type->tp_bases);
     Py_CLEAR(type->tp_mro);
     Py_CLEAR(type->tp_cache);
     Py_CLEAR(type->tp_subclasses);
+
     type->tp_flags &= ~Py_TPFLAGS_READY;
 }
 
 
 static void
 type_dealloc(PyTypeObject *type)
 {
-    PyObject *tp, *val, *tb;
-
     /* Assert this is a heap-allocated type object */
     _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE);
     _PyObject_GC_UNTRACK(type);
-    PyErr_Fetch(&tp, &val, &tb);
-    remove_all_subclasses(type, type->tp_bases);
-    PyErr_Restore(tp, val, tb);
 
-    PyObject_ClearWeakRefs((PyObject *)type);
+    type_dealloc_common(type);
+
     Py_XDECREF(type->tp_base);
     Py_XDECREF(type->tp_dict);
     Py_XDECREF(type->tp_bases);
     Py_XDECREF(type->tp_mro);
     Py_XDECREF(type->tp_cache);
     Py_XDECREF(type->tp_subclasses);
+
     /* A type's tp_doc is heap allocated, unlike the tp_doc slots
      * of most other objects.  It's okay to cast it to char *.
      */
@@ -6541,6 +6553,10 @@ remove_subclass(PyTypeObject *base, PyTypeObject *type)
         PyErr_Clear();
     }
     Py_XDECREF(key);
+
+    if (PyDict_Size(dict) == 0) {
+        Py_CLEAR(base->tp_subclasses);
+    }
 }
 
 static void

Python/pylifecycle.c

@@ -1676,6 +1676,7 @@ finalize_interp_types(PyInterpreterState *interp)
     _PyThread_FiniType(interp);
     _PyErr_FiniTypes(interp);
     _PyTypes_Fini(interp);
+    _PyTypes_FiniTypes(interp);
 
     // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses
     // a dict internally.