From 5c34508231058796cbe2ffddf055386960f9a8e5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Jul 2017 19:20:17 +0200 Subject: [PATCH 01/93] Initial work on C implementation of ABCMeta --- Lib/abc.py | 5 +- Modules/_abc.c | 186 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 Modules/_abc.c diff --git a/Lib/abc.py b/Lib/abc.py index d13a0de89b4054..068e6dbead76a2 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -3,7 +3,7 @@ """Abstract Base Classes (ABCs) according to PEP 3119.""" -from _weakrefset import WeakSet +#from _weakrefset import WeakSet def abstractmethod(funcobj): @@ -231,6 +231,9 @@ def __subclasscheck__(cls, subclass): return False +from _abc import ABCMeta + + class ABC(metaclass=ABCMeta): """Helper class that provides a standard way to create an ABC using inheritance. diff --git a/Modules/_abc.c b/Modules/_abc.c new file mode 100644 index 00000000000000..56bce7703efd41 --- /dev/null +++ b/Modules/_abc.c @@ -0,0 +1,186 @@ +/* ABCMeta implementation */ + +#include "Python.h" +#include "structmember.h" + +PyDoc_STRVAR(_abc__doc__, +"_abc module contains (presumably faster) implementation of ABCMeta"); +#define DEFERRED_ADDRESS(ADDR) 0 + +typedef struct { + PyHeapTypeObject tp; + PyObject *abc_registry; + /*PyObject *abc_cache; + PyObject *abc_negative_cache; + int abc_negative_cache_version;*/ +} abc; + +static void +abcmeta_dealloc(abc *tp) +{ + Py_CLEAR(tp->abc_registry); + PyType_Type.tp_dealloc((PyObject *)tp); +} + +static int +abcmeta_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((abc *)self)->abc_registry); + return PyType_Type.tp_traverse(self, visit, arg); +} + +static int +abcmeta_clear(abc *tp) +{ + PySet_Clear(tp->abc_registry); + return PyType_Type.tp_clear((PyObject *)tp); +} + +static PyObject * +abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + abc *result = NULL; + result = (abc *)PyType_Type.tp_new(type, args, kwds); + if (!result) { + return NULL; + } + result->abc_registry = PySet_New(NULL); + return (PyObject *)result; +} + +static PyObject * +abcmeta_register(abc *self, PyObject *args) +{ + PyObject *subclass = NULL; + if (!PyArg_UnpackTuple(args, "register", 1, 1, &subclass)) { + return NULL; + } + if (!PyType_Check(subclass)) { + PyErr_SetString(PyExc_TypeError, "Can only register classes"); + return NULL; + } + PySet_Add(self->abc_registry, subclass); + Py_INCREF(subclass); + return subclass; +} + +static PyObject * +abcmeta_instancecheck(abc *self, PyObject *args) +{ + PyObject *instance = NULL; + if (!PyArg_UnpackTuple(args, "__isinstance__", 1, 1, &instance)) { + return NULL; + } + return Py_False; +} + +static PyObject * +abcmeta_subclasscheck(abc *self, PyObject *args) +{ + PyObject *subclass = NULL; + if (!PyArg_UnpackTuple(args, "__issubclass__", 1, 1, &subclass)) { + return NULL; + } + return Py_False; +} + +static PyObject * +abcmeta_dump(abc *self, PyObject *args) +{ + PyObject *file = NULL; + if (!PyArg_UnpackTuple(args, "file", 0, 1, &file)) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + +static PyMethodDef abcmeta_methods[] = { + {"register", (PyCFunction)abcmeta_register, METH_VARARGS, + PyDoc_STR("register subclass")}, + {"__instancecheck__", (PyCFunction)abcmeta_instancecheck, METH_VARARGS, + PyDoc_STR("check if instance is an instance of the class")}, + {"__subclasscheck__", (PyCFunction)abcmeta_subclasscheck, METH_VARARGS, + PyDoc_STR("check if subclass is a subclass of the class")}, + {"_dump_registry", (PyCFunction)abcmeta_dump, METH_VARARGS, + PyDoc_STR("dump registry (private)")}, + {NULL, NULL}, +}; + +PyTypeObject ABCMeta = { + PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) + "ABCMeta", /* tp_name */ + sizeof(abc), /* tp_basicsize */ + 0, /* tp_itemsize */ + abcmeta_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VERSION_TAG | + Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ + 0, /* tp_doc */ + abcmeta_traverse, /* tp_traverse */ + abcmeta_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + abcmeta_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + DEFERRED_ADDRESS(&PyType_Type), /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + abcmeta_new, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ +}; + + +static struct PyModuleDef _abcmodule = { + PyModuleDef_HEAD_INIT, + "_abc", + _abc__doc__, + -1, + NULL, + NULL, + NULL, + NULL, + NULL +}; + + +PyMODINIT_FUNC +PyInit__abc(void) +{ + PyObject *m; + + m = PyModule_Create(&_abcmodule); + if (m == NULL) + return NULL; + ABCMeta.tp_base = &PyType_Type; + if (PyType_Ready(&ABCMeta) < 0) { + return -1; + } + Py_INCREF(&ABCMeta); + if (PyModule_AddObject(m, "ABCMeta", + (PyObject *) &ABCMeta) < 0) { + return -1; + } + return m; +} From cb7ffcf60978781be4a35ee13378dad490ff88c9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 26 Jul 2017 00:02:37 +0200 Subject: [PATCH 02/93] Basic implementation of ABCMeta.__new__ --- Modules/_abc.c | 122 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 111 insertions(+), 11 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 56bce7703efd41..24e099e47389b5 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -7,18 +7,22 @@ PyDoc_STRVAR(_abc__doc__, "_abc module contains (presumably faster) implementation of ABCMeta"); #define DEFERRED_ADDRESS(ADDR) 0 +static Py_ssize_t abc_invalidation_counter = 0; + typedef struct { PyHeapTypeObject tp; - PyObject *abc_registry; - /*PyObject *abc_cache; - PyObject *abc_negative_cache; - int abc_negative_cache_version;*/ + PyObject *abc_registry; /* normal set of weakrefs without callback */ + PyObject *abc_cache; /* normal set of weakrefs with callback (we never iterate over it) */ + PyObject *abc_negative_cache; /* normal set of weakrefs with callback */ + Py_ssize_t abc_negative_cache_version; } abc; static void abcmeta_dealloc(abc *tp) { Py_CLEAR(tp->abc_registry); + Py_CLEAR(tp->abc_cache); + Py_CLEAR(tp->abc_negative_cache); PyType_Type.tp_dealloc((PyObject *)tp); } @@ -26,6 +30,8 @@ static int abcmeta_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(((abc *)self)->abc_registry); + Py_VISIT(((abc *)self)->abc_cache); + Py_VISIT(((abc *)self)->abc_negative_cache); return PyType_Type.tp_traverse(self, visit, arg); } @@ -33,6 +39,8 @@ static int abcmeta_clear(abc *tp) { PySet_Clear(tp->abc_registry); + PySet_Clear(tp->abc_cache); + PySet_Clear(tp->abc_negative_cache); return PyType_Type.tp_clear((PyObject *)tp); } @@ -40,11 +48,85 @@ static PyObject * abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { abc *result = NULL; + PyObject *ns, *bases, *items, *abstracts, *is_abstract, *base_abstracts; + PyObject *key, *value, *item, *iter; + Py_ssize_t pos = 0; + result = (abc *)PyType_Type.tp_new(type, args, kwds); if (!result) { return NULL; } - result->abc_registry = PySet_New(NULL); + result->abc_registry = PySet_New(NULL); /* TODO: Delay registry creation until it is actually needed */ + result->abc_cache = PySet_New(NULL); + result->abc_negative_cache = PySet_New(NULL); + if (!result->abc_registry || !result->abc_cache || + !result->abc_negative_cache) { + return NULL; + } + result->abc_negative_cache_version = abc_invalidation_counter; + abstracts = PyFrozenSet_New(NULL); + /* Stage 1: direct abstract methods */ + /* Safe to assume everything is fine since type.__new__ succeeded */ + ns = PyTuple_GET_ITEM(args, 2); + items = PyMapping_Items(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ + for (pos = 0; pos < PySequence_Size(items); pos++) { /* TODO: Check if it is a list or tuple? */ + item = PySequence_GetItem(items, pos); + key = PyTuple_GetItem(item, 0); + value = PyTuple_GetItem(item, 1); + is_abstract = PyObject_GetAttrString(value, "__isabstractmethod__"); + if (!is_abstract) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + continue; + } + if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { + return NULL; + } + } + /* Stage 2: inherited abstract methods */ + bases = PyTuple_GET_ITEM(args, 1); + for (pos = 0; pos < PyTuple_Size(bases); pos++) { + item = PyTuple_GetItem(bases, pos); + base_abstracts = PyObject_GetAttrString(item, "__abstractmethods__"); + if (!base_abstracts) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + continue; + } + if (!(iter = PyObject_GetIter(base_abstracts))) { + return NULL; + } + while (key = PyIter_Next(iter)) { + value = PyObject_GetAttr(result, key); + if (!value) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + continue; + } + is_abstract = PyObject_GetAttrString(value, "__isabstractmethod__"); + if (!is_abstract) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + continue; + } + if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { + return NULL; + } + Py_DECREF(key); + } + Py_DECREF(iter); + } + if (PyObject_SetAttrString((PyObject *)result, "__abstractmethods__", abstracts) < 0) { + return NULL; + } return (PyObject *)result; } @@ -59,8 +141,11 @@ abcmeta_register(abc *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "Can only register classes"); return NULL; } - PySet_Add(self->abc_registry, subclass); + if (PySet_Add(self->abc_registry, subclass) < 0) { + return NULL; + } Py_INCREF(subclass); + abc_invalidation_counter++; return subclass; } @@ -71,6 +156,7 @@ abcmeta_instancecheck(abc *self, PyObject *args) if (!PyArg_UnpackTuple(args, "__isinstance__", 1, 1, &instance)) { return NULL; } + Py_INCREF(Py_False); return Py_False; } @@ -81,6 +167,11 @@ abcmeta_subclasscheck(abc *self, PyObject *args) if (!PyArg_UnpackTuple(args, "__issubclass__", 1, 1, &subclass)) { return NULL; } + /* TODO: clear the registry from dead refs from time to time + on iteration here (have a counter for this) */ + /* TODO: Reset caches every n-th succes/failure correspondingly + so that they don't grow too large */ + Py_INCREF(Py_False); return Py_False; } @@ -88,7 +179,16 @@ static PyObject * abcmeta_dump(abc *self, PyObject *args) { PyObject *file = NULL; - if (!PyArg_UnpackTuple(args, "file", 0, 1, &file)) { + PyObject* version = PyLong_FromSsize_t(self->abc_negative_cache_version); + int rv; + if (!PyArg_UnpackTuple(args, "_dump_registry", 0, 1, &file)) { + return NULL; + } + if (!version) { + return NULL; + } + rv = PyObject_Print(version, stdout, 0); + if (rv < 0) { return NULL; } Py_INCREF(Py_None); @@ -112,7 +212,7 @@ PyTypeObject ABCMeta = { "ABCMeta", /* tp_name */ sizeof(abc), /* tp_basicsize */ 0, /* tp_itemsize */ - abcmeta_dealloc, /* tp_dealloc */ + (destructor)abcmeta_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -131,7 +231,7 @@ PyTypeObject ABCMeta = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ 0, /* tp_doc */ abcmeta_traverse, /* tp_traverse */ - abcmeta_clear, /* tp_clear */ + (inquiry)abcmeta_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -175,12 +275,12 @@ PyInit__abc(void) return NULL; ABCMeta.tp_base = &PyType_Type; if (PyType_Ready(&ABCMeta) < 0) { - return -1; + return NULL; } Py_INCREF(&ABCMeta); if (PyModule_AddObject(m, "ABCMeta", (PyObject *) &ABCMeta) < 0) { - return -1; + return NULL; } return m; } From b83ee80bf9d58f7e207e31c8c5f28d7b3bbd1ac5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 26 Jul 2017 01:31:03 +0200 Subject: [PATCH 03/93] Bare-bone implementation of register and subclass checks --- Lib/abc.py | 2 ++ Modules/_abc.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 068e6dbead76a2..a23612ece3a520 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -249,3 +249,5 @@ def get_cache_token(): with every call to ``register()`` on any ABC. """ return ABCMeta._abc_invalidation_counter + +from _abc import get_cache_token diff --git a/Modules/_abc.c b/Modules/_abc.c index 24e099e47389b5..ab3bfaa020bf77 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,5 +1,9 @@ /* ABCMeta implementation */ +/* TODO: Global check where checks are needed, and where I made objects myself */ +/* In particular use capitals like PyList_GET_SIZE */ +/* Think (ask) about inlining some calls, like __subclasses__ */ + #include "Python.h" #include "structmember.h" @@ -141,6 +145,14 @@ abcmeta_register(abc *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "Can only register classes"); return NULL; } + if (PyObject_IsSubclass(subclass, (PyObject *)self)) { /* TODO: Check for error here */ + Py_INCREF(subclass); + return subclass; + } + if (PyObject_IsSubclass((PyObject *)self, subclass)) { /* TODO: Check for error here */ + PyErr_SetString(PyExc_RuntimeError, "Refusing to create an inheritance cycle"); + return NULL; + } if (PySet_Add(self->abc_registry, subclass) < 0) { return NULL; } @@ -149,21 +161,27 @@ abcmeta_register(abc *self, PyObject *args) return subclass; } +static PyObject * +abcmeta_subclasscheck(abc *self, PyObject *args); /* Forward */ + static PyObject * abcmeta_instancecheck(abc *self, PyObject *args) { - PyObject *instance = NULL; + PyObject *subclass, *instance = NULL; if (!PyArg_UnpackTuple(args, "__isinstance__", 1, 1, &instance)) { return NULL; } - Py_INCREF(Py_False); - return Py_False; + subclass = Py_TYPE(instance); + /* TODO: Use cache */ + return abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); /* TODO: Refactor to avoid packing */ } static PyObject * abcmeta_subclasscheck(abc *self, PyObject *args) { - PyObject *subclass = NULL; + PyObject *subclasses, *subclass = NULL; + PyObject *ok, *mro, *iter, *key; + Py_ssize_t pos; if (!PyArg_UnpackTuple(args, "__issubclass__", 1, 1, &subclass)) { return NULL; } @@ -171,6 +189,38 @@ abcmeta_subclasscheck(abc *self, PyObject *args) on iteration here (have a counter for this) */ /* TODO: Reset caches every n-th succes/failure correspondingly so that they don't grow too large */ + ok = PyObject_CallMethod(self, "__subclasshook__", "O", subclass); + if (ok == Py_True) { + Py_INCREF(Py_True); + return Py_True; + } + if (ok == Py_False) { + Py_INCREF(Py_False); + return Py_False; + } + mro = ((PyTypeObject *)subclass)->tp_mro; + for (pos = 0; pos < PyTuple_Size(mro); pos++) { + if (self == PyTuple_GetItem(mro, pos)) { + Py_INCREF(Py_True); + return Py_True; + } + } + iter = PyObject_GetIter(self->abc_registry); + while (key = PyIter_Next(iter)) { + if (PyObject_IsSubclass(subclass, key)) { + Py_INCREF(Py_True); + return Py_True; + } + Py_DECREF(key); + } + Py_DECREF(iter); + subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); + for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { + if (PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos))) { + Py_INCREF(Py_True); + return Py_True; + } + } Py_INCREF(Py_False); return Py_False; } @@ -251,13 +301,30 @@ PyTypeObject ABCMeta = { 0, /* tp_is_gc */ }; +PyDoc_STRVAR(_cache_token_doc, +"Returns the current ABC cache token.\n\ +\n\ +The token is an opaque object (supporting equality testing) identifying the\n\ +current version of the ABC cache for virtual subclasses. The token changes\n\ +with every call to ``register()`` on any ABC."); + +static PyObject * +get_cache_token(void) +{ + return PyLong_FromSsize_t(abc_invalidation_counter); +} + +static struct PyMethodDef module_functions[] = { + {"get_cache_token", get_cache_token, METH_NOARGS, _cache_token_doc}, + {NULL, NULL} /* sentinel */ +}; static struct PyModuleDef _abcmodule = { PyModuleDef_HEAD_INIT, "_abc", _abc__doc__, -1, - NULL, + module_functions, NULL, NULL, NULL, From 181e83f712e5f2d400368de6d08813c606a126cc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 26 Jul 2017 19:34:36 +0200 Subject: [PATCH 04/93] Fix mock failure and silence compiler warnings --- Modules/_abc.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index ab3bfaa020bf77..1ab39b6baa5e14 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -3,6 +3,7 @@ /* TODO: Global check where checks are needed, and where I made objects myself */ /* In particular use capitals like PyList_GET_SIZE */ /* Think (ask) about inlining some calls, like __subclasses__ */ +/* Use PyId instead of string attrs */ #include "Python.h" #include "structmember.h" @@ -104,8 +105,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!(iter = PyObject_GetIter(base_abstracts))) { return NULL; } - while (key = PyIter_Next(iter)) { - value = PyObject_GetAttr(result, key); + while ((key = PyIter_Next(iter))) { + value = PyObject_GetAttr((PyObject *)result, key); if (!value) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; @@ -171,9 +172,13 @@ abcmeta_instancecheck(abc *self, PyObject *args) if (!PyArg_UnpackTuple(args, "__isinstance__", 1, 1, &instance)) { return NULL; } - subclass = Py_TYPE(instance); + subclass = (PyObject *)Py_TYPE(instance); /* TODO: Use cache */ - return abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); /* TODO: Refactor to avoid packing */ + if (abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)) == Py_True) { /* TODO: Refactor to avoid packing */ + return Py_True; + } + subclass = PyObject_GetAttrString(instance, "__class__"); + return abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); } static PyObject * @@ -189,7 +194,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) on iteration here (have a counter for this) */ /* TODO: Reset caches every n-th succes/failure correspondingly so that they don't grow too large */ - ok = PyObject_CallMethod(self, "__subclasshook__", "O", subclass); + ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); if (ok == Py_True) { Py_INCREF(Py_True); return Py_True; @@ -200,13 +205,13 @@ abcmeta_subclasscheck(abc *self, PyObject *args) } mro = ((PyTypeObject *)subclass)->tp_mro; for (pos = 0; pos < PyTuple_Size(mro); pos++) { - if (self == PyTuple_GetItem(mro, pos)) { + if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { Py_INCREF(Py_True); return Py_True; } } iter = PyObject_GetIter(self->abc_registry); - while (key = PyIter_Next(iter)) { + while ((key = PyIter_Next(iter))) { if (PyObject_IsSubclass(subclass, key)) { Py_INCREF(Py_True); return Py_True; @@ -214,7 +219,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) Py_DECREF(key); } Py_DECREF(iter); - subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); + subclasses = PyObject_CallMethod((PyObject *)self, "__subclasses__", NULL); for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { if (PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos))) { Py_INCREF(Py_True); From c084a7fd3c26ea17569ac400264adca843b6c10b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 26 Jul 2017 22:25:04 +0200 Subject: [PATCH 05/93] Provide nicer dump of registry --- Modules/_abc.c | 83 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 1ab39b6baa5e14..df94282937a800 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -12,6 +12,8 @@ PyDoc_STRVAR(_abc__doc__, "_abc module contains (presumably faster) implementation of ABCMeta"); #define DEFERRED_ADDRESS(ADDR) 0 +_Py_IDENTIFIER(stdout); + static Py_ssize_t abc_invalidation_counter = 0; typedef struct { @@ -230,20 +232,93 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return Py_False; } +int +_print_message(PyObject *file, const char* message) +{ + PyObject *mo = PyUnicode_FromString(message); + if (!mo) { + return -1; + } + if (PyFile_WriteObject(mo, file, Py_PRINT_RAW)) { + return -1; + } + return 0; +} + static PyObject * abcmeta_dump(abc *self, PyObject *args) { PyObject *file = NULL; - PyObject* version = PyLong_FromSsize_t(self->abc_negative_cache_version); - int rv; + PyObject *sizeo, *version = PyLong_FromSsize_t(self->abc_negative_cache_version); + Py_ssize_t size; if (!PyArg_UnpackTuple(args, "_dump_registry", 0, 1, &file)) { return NULL; } if (!version) { return NULL; } - rv = PyObject_Print(version, stdout, 0); - if (rv < 0) { + if (file == NULL || file == Py_None) { + file = _PySys_GetObjectId(&PyId_stdout); + if (file == NULL) { + PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); + return NULL; + } + } + /* Header */ + if (_print_message(file, "Class: ")) { + return NULL; + } + if (_print_message(file, ((PyTypeObject *)self)->tp_name)) { + return NULL; + } + if (_print_message(file, "\n")) { + return NULL; + } + /* Registry */ + if (_print_message(file, "Registry: ")) { + return NULL; + } + if (PyFile_WriteObject(self->abc_registry, file, Py_PRINT_RAW)) { + return NULL; + } + if (_print_message(file, "\n")) { + return NULL; + } + /* Postive cahce */ + if (_print_message(file, "Positive cache: ")) { + return NULL; + } + size = PySet_GET_SIZE(self->abc_cache); + if (!(sizeo = PyLong_FromSsize_t(size))) { + return NULL; + } + if (PyFile_WriteObject(sizeo, file, Py_PRINT_RAW)) { + return NULL; + } + if (_print_message(file, " items\n")) { + return NULL; + } + /* Negative cahce */ + if (_print_message(file, "Negative cache: ")) { + return NULL; + } + size = PySet_GET_SIZE(self->abc_cache); + if (!(sizeo = PyLong_FromSsize_t(size))) { + return NULL; + } + if (PyFile_WriteObject(sizeo, file, Py_PRINT_RAW)) { + return NULL; + } + if (_print_message(file, " items\n")) { + return NULL; + } + if (_print_message(file, "Negative cache version: ")) { + return NULL; + } + if (PyFile_WriteObject(version, file, Py_PRINT_RAW)) { + return NULL; + } + if (_print_message(file, "\n")) { return NULL; } Py_INCREF(Py_None); From a192d5d6186987ea2706478fe76f500a9106fe24 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 26 Jul 2017 22:41:32 +0200 Subject: [PATCH 06/93] Add better docstrings --- Modules/_abc.c | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index df94282937a800..68946ca1854b55 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -9,7 +9,7 @@ #include "structmember.h" PyDoc_STRVAR(_abc__doc__, -"_abc module contains (presumably faster) implementation of ABCMeta"); +"Module contains faster C implementation of abc.ABCMeta"); #define DEFERRED_ADDRESS(ADDR) 0 _Py_IDENTIFIER(stdout); @@ -325,18 +325,36 @@ abcmeta_dump(abc *self, PyObject *args) return Py_None; } +PyDoc_STRVAR(_register_doc, +"Register a virtual subclass of an ABC.\n\ +\n\ +Returns the subclass, to allow usage as a class decorator."); + static PyMethodDef abcmeta_methods[] = { {"register", (PyCFunction)abcmeta_register, METH_VARARGS, - PyDoc_STR("register subclass")}, + _register_doc}, {"__instancecheck__", (PyCFunction)abcmeta_instancecheck, METH_VARARGS, - PyDoc_STR("check if instance is an instance of the class")}, + PyDoc_STR("Override for isinstance(instance, cls).")}, {"__subclasscheck__", (PyCFunction)abcmeta_subclasscheck, METH_VARARGS, - PyDoc_STR("check if subclass is a subclass of the class")}, + PyDoc_STR("Override for issubclass(subclass, cls).")}, {"_dump_registry", (PyCFunction)abcmeta_dump, METH_VARARGS, - PyDoc_STR("dump registry (private)")}, + PyDoc_STR("Debug helper to print the ABC registry.")}, {NULL, NULL}, }; +PyDoc_STRVAR(abcmeta_doc, + "Metaclass for defining Abstract Base Classes (ABCs).\n\ +\n\ +Use this metaclass to create an ABC. An ABC can be subclassed\n\ +directly, and then acts as a mix-in class. You can also register\n\ +unrelated concrete classes (even built-in classes) and unrelated\n\ +ABCs as 'virtual subclasses' -- these and their descendants will\n\ +be considered subclasses of the registering ABC by the built-in\n\ +issubclass() function, but the registering ABC won't show up in\n\ +their MRO (Method Resolution Order) nor will method\n\ +implementations defined by the registering ABC be callable (not\n\ +even via super())."); + PyTypeObject ABCMeta = { PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) "ABCMeta", /* tp_name */ @@ -359,7 +377,7 @@ PyTypeObject ABCMeta = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VERSION_TAG | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ - 0, /* tp_doc */ + abcmeta_doc, /* tp_doc */ abcmeta_traverse, /* tp_traverse */ (inquiry)abcmeta_clear, /* tp_clear */ 0, /* tp_richcompare */ From 35a24728420b6d3ca04d1f9323d87d0f34de5896 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 27 Jul 2017 11:39:22 +0200 Subject: [PATCH 07/93] Expose the internal cavhes and registry (backward compatibility) --- Modules/_abc.c | 71 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 68946ca1854b55..02650dc1f7f61f 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -4,6 +4,9 @@ /* In particular use capitals like PyList_GET_SIZE */ /* Think (ask) about inlining some calls, like __subclasses__ */ /* Use PyId instead of string attrs */ +/* Work on INCREF/DECREF */ +/* Be sure to return NULL where exception possible */ +/* Use macros */ #include "Python.h" #include "structmember.h" @@ -45,9 +48,15 @@ abcmeta_traverse(PyObject *self, visitproc visit, void *arg) static int abcmeta_clear(abc *tp) { - PySet_Clear(tp->abc_registry); - PySet_Clear(tp->abc_cache); - PySet_Clear(tp->abc_negative_cache); + if (tp->abc_registry) { /* This may be because _abc_registry is writeable (and was shared by typing) */ + PySet_Clear(tp->abc_registry); + } + if (tp->abc_cache) { + PySet_Clear(tp->abc_cache); + } + if (tp->abc_negative_cache) { + PySet_Clear(tp->abc_negative_cache); + } return PyType_Type.tp_clear((PyObject *)tp); } @@ -58,7 +67,6 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *ns, *bases, *items, *abstracts, *is_abstract, *base_abstracts; PyObject *key, *value, *item, *iter; Py_ssize_t pos = 0; - result = (abc *)PyType_Type.tp_new(type, args, kwds); if (!result) { return NULL; @@ -141,6 +149,7 @@ static PyObject * abcmeta_register(abc *self, PyObject *args) { PyObject *subclass = NULL; + int result; if (!PyArg_UnpackTuple(args, "register", 1, 1, &subclass)) { return NULL; } @@ -148,14 +157,22 @@ abcmeta_register(abc *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "Can only register classes"); return NULL; } - if (PyObject_IsSubclass(subclass, (PyObject *)self)) { /* TODO: Check for error here */ + result = PyObject_IsSubclass(subclass, (PyObject *)self); + if (result > 0) { Py_INCREF(subclass); return subclass; } - if (PyObject_IsSubclass((PyObject *)self, subclass)) { /* TODO: Check for error here */ + if (result < 0) { + return NULL; + } + result = PyObject_IsSubclass((PyObject *)self, subclass); + if (result > 0) { PyErr_SetString(PyExc_RuntimeError, "Refusing to create an inheritance cycle"); return NULL; } + if (result < 0) { + return NULL; + } if (PySet_Add(self->abc_registry, subclass) < 0) { return NULL; } @@ -170,13 +187,17 @@ abcmeta_subclasscheck(abc *self, PyObject *args); /* Forward */ static PyObject * abcmeta_instancecheck(abc *self, PyObject *args) { - PyObject *subclass, *instance = NULL; - if (!PyArg_UnpackTuple(args, "__isinstance__", 1, 1, &instance)) { + PyObject *result, *subclass, *instance = NULL; + if (!PyArg_UnpackTuple(args, "__instancecheck__", 1, 1, &instance)) { return NULL; } subclass = (PyObject *)Py_TYPE(instance); /* TODO: Use cache */ - if (abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)) == Py_True) { /* TODO: Refactor to avoid packing */ + result = abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); + if (!result) { + return NULL; + } + if (result == Py_True) { /* TODO: Refactor to avoid packing */ return Py_True; } subclass = PyObject_GetAttrString(instance, "__class__"); @@ -189,7 +210,8 @@ abcmeta_subclasscheck(abc *self, PyObject *args) PyObject *subclasses, *subclass = NULL; PyObject *ok, *mro, *iter, *key; Py_ssize_t pos; - if (!PyArg_UnpackTuple(args, "__issubclass__", 1, 1, &subclass)) { + int result; + if (!PyArg_UnpackTuple(args, "__subclasscheck__", 1, 1, &subclass)) { return NULL; } /* TODO: clear the registry from dead refs from time to time @@ -197,6 +219,9 @@ abcmeta_subclasscheck(abc *self, PyObject *args) /* TODO: Reset caches every n-th succes/failure correspondingly so that they don't grow too large */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); + if (!ok) { + return NULL; + } if (ok == Py_True) { Py_INCREF(Py_True); return Py_True; @@ -214,19 +239,27 @@ abcmeta_subclasscheck(abc *self, PyObject *args) } iter = PyObject_GetIter(self->abc_registry); while ((key = PyIter_Next(iter))) { - if (PyObject_IsSubclass(subclass, key)) { + result = PyObject_IsSubclass(subclass, key); + if (result > 0) { Py_INCREF(Py_True); return Py_True; } + if (result < 0) { + return NULL; + } Py_DECREF(key); } Py_DECREF(iter); subclasses = PyObject_CallMethod((PyObject *)self, "__subclasses__", NULL); for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { - if (PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos))) { + result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); + if (result > 0) { Py_INCREF(Py_True); return Py_True; } + if (result < 0) { + return NULL; + } } Py_INCREF(Py_False); return Py_False; @@ -342,6 +375,18 @@ static PyMethodDef abcmeta_methods[] = { {NULL, NULL}, }; +static PyMemberDef abc_members[] = { + {"_abc_registry", T_OBJECT, offsetof(abc, abc_registry), 0, /* Maybe make these READONLY */ + PyDoc_STR("ABC registry (private).")}, + {"_abc_cache", T_OBJECT, offsetof(abc, abc_cache), 0, + PyDoc_STR("ABC positive cache (private).")}, + {"_abc_negative_cache", T_OBJECT, offsetof(abc, abc_negative_cache), 0, + PyDoc_STR("ABC negative cache (private).")}, + {"_abc_negative_cache_version", T_OBJECT, offsetof(abc, abc_negative_cache), 0, + PyDoc_STR("ABC negative cache version (private).")}, + {NULL} +}; + PyDoc_STRVAR(abcmeta_doc, "Metaclass for defining Abstract Base Classes (ABCs).\n\ \n\ @@ -385,7 +430,7 @@ PyTypeObject ABCMeta = { 0, /* tp_iter */ 0, /* tp_iternext */ abcmeta_methods, /* tp_methods */ - 0, /* tp_members */ + abc_members, /* tp_members */ 0, /* tp_getset */ DEFERRED_ADDRESS(&PyType_Type), /* tp_base */ 0, /* tp_dict */ From bbee578440218a46252ddcd405d09e53471324b0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 20 Jan 2018 12:29:53 +0000 Subject: [PATCH 08/93] Add _abc to Setup.dist --- Modules/Setup.dist | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 1f2d56c06555ba..35f3673dd658d2 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -118,6 +118,7 @@ _weakref _weakref.c # weak references _functools _functoolsmodule.c # Tools for working with functions and callable objects _operator _operator.c # operator.add() and similar goodies _collections _collectionsmodule.c # Container types +_abc _abcmodule.c # Abstract base classes itertools itertoolsmodule.c # Functions creating iterators for efficient looping atexit atexitmodule.c # Register functions to be run at interpreter-shutdown _signal signalmodule.c From a3464fdce0d6e0b6b1ba7f92005da38348f40e2d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 20 Jan 2018 12:32:44 +0000 Subject: [PATCH 09/93] Fix _abc in Setup.dist --- Modules/Setup.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 35f3673dd658d2..39f8a0bd787ace 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -118,7 +118,7 @@ _weakref _weakref.c # weak references _functools _functoolsmodule.c # Tools for working with functions and callable objects _operator _operator.c # operator.add() and similar goodies _collections _collectionsmodule.c # Container types -_abc _abcmodule.c # Abstract base classes +_abc _abc.c # Abstract base classes itertools itertoolsmodule.c # Functions creating iterators for efficient looping atexit atexitmodule.c # Register functions to be run at interpreter-shutdown _signal signalmodule.c From 947bf7d8043da974794962851f1fff36da040c0f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 20 Jan 2018 12:59:59 +0000 Subject: [PATCH 10/93] Update a comment --- Modules/_abc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 02650dc1f7f61f..0cc0e3a2ce6653 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -215,7 +215,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } /* TODO: clear the registry from dead refs from time to time - on iteration here (have a counter for this) */ + on iteration here (have a counter for this) or maybe better durin a .register() call */ /* TODO: Reset caches every n-th succes/failure correspondingly so that they don't grow too large */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); From 7ffc59ecd7defbbc6bdec5b4a7e69bc684050fde Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 20 Jan 2018 13:41:24 +0000 Subject: [PATCH 11/93] Settle the .py version --- Lib/abc.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 5f970d8d650c2c..2e13fa8020123c 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -3,8 +3,6 @@ """Abstract Base Classes (ABCs) according to PEP 3119.""" -#from _weakrefset import WeakSet - def abstractmethod(funcobj): """A decorator indicating abstract methods. @@ -233,16 +231,6 @@ def __subclasscheck__(cls, subclass): return False -from _abc import ABCMeta - - -class ABC(metaclass=ABCMeta): - """Helper class that provides a standard way to create an ABC using - inheritance. - """ - __slots__ = () - - def get_cache_token(): """Returns the current ABC cache token. @@ -252,4 +240,16 @@ def get_cache_token(): """ return ABCMeta._abc_invalidation_counter -from _abc import get_cache_token + +try: + from _abc import ABCMeta, get_cache_token +except ImportError: + # We postpone this import to speed-up Python start-up time + from _weakrefset import WeakSet + + +class ABC(metaclass=ABCMeta): + """Helper class that provides a standard way to create an ABC using + inheritance. + """ + __slots__ = () From 41287a7ac3f236dda35c83830a5d651ee3f84ece Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 21 Jan 2018 21:25:49 +0000 Subject: [PATCH 12/93] Fix some TODOs and refleaks --- Lib/test/libregrtest/refleak.py | 4 +- Modules/_abc.c | 67 +++++++++++++++++++++++---------- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 2ca9aa87644c0d..43a7103a5923e5 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -113,7 +113,6 @@ def check_fd_deltas(deltas): def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg import collections.abc - from weakref import WeakSet # Restore some original values. warnings.filters[:] = fs @@ -137,7 +136,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: - obj._abc_registry = abcs.get(obj, WeakSet()).copy() + for el in abcs.get(obj, set()): + obj._abc_registry.add(el) obj._abc_cache.clear() obj._abc_negative_cache.clear() diff --git a/Modules/_abc.c b/Modules/_abc.c index 0cc0e3a2ce6653..0f7ea82cbf0a00 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -3,7 +3,6 @@ /* TODO: Global check where checks are needed, and where I made objects myself */ /* In particular use capitals like PyList_GET_SIZE */ /* Think (ask) about inlining some calls, like __subclasses__ */ -/* Use PyId instead of string attrs */ /* Work on INCREF/DECREF */ /* Be sure to return NULL where exception possible */ /* Use macros */ @@ -16,14 +15,17 @@ PyDoc_STRVAR(_abc__doc__, #define DEFERRED_ADDRESS(ADDR) 0 _Py_IDENTIFIER(stdout); +_Py_IDENTIFIER(__isabstractmethod__); +_Py_IDENTIFIER(__abstractmethods__); +_Py_IDENTIFIER(__class__); static Py_ssize_t abc_invalidation_counter = 0; typedef struct { PyHeapTypeObject tp; - PyObject *abc_registry; /* normal set of weakrefs without callback */ - PyObject *abc_cache; /* normal set of weakrefs with callback (we never iterate over it) */ - PyObject *abc_negative_cache; /* normal set of weakrefs with callback */ + PyObject *abc_registry; /* these three are normal set of weakrefs without callback */ + PyObject *abc_cache; + PyObject *abc_negative_cache; Py_ssize_t abc_negative_cache_version; } abc; @@ -33,7 +35,7 @@ abcmeta_dealloc(abc *tp) Py_CLEAR(tp->abc_registry); Py_CLEAR(tp->abc_cache); Py_CLEAR(tp->abc_negative_cache); - PyType_Type.tp_dealloc((PyObject *)tp); + Py_TYPE(tp)->tp_free((PyObject *)tp); } static int @@ -48,7 +50,7 @@ abcmeta_traverse(PyObject *self, visitproc visit, void *arg) static int abcmeta_clear(abc *tp) { - if (tp->abc_registry) { /* This may be because _abc_registry is writeable (and was shared by typing) */ + if (tp->abc_registry) { PySet_Clear(tp->abc_registry); } if (tp->abc_cache) { @@ -88,23 +90,30 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) item = PySequence_GetItem(items, pos); key = PyTuple_GetItem(item, 0); value = PyTuple_GetItem(item, 1); - is_abstract = PyObject_GetAttrString(value, "__isabstractmethod__"); + is_abstract = _PyObject_GetAttrId(value, &PyId___isabstractmethod__); if (!is_abstract) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(item); + Py_DECREF(items); return NULL; } PyErr_Clear(); + Py_DECREF(item); continue; } if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { + Py_DECREF(item); + Py_DECREF(items); return NULL; } + Py_DECREF(item); } + Py_DECREF(items); /* Stage 2: inherited abstract methods */ bases = PyTuple_GET_ITEM(args, 1); for (pos = 0; pos < PyTuple_Size(bases); pos++) { - item = PyTuple_GetItem(bases, pos); - base_abstracts = PyObject_GetAttrString(item, "__abstractmethods__"); + item = PyTuple_GET_ITEM(bases, pos); + base_abstracts = _PyObject_GetAttrId(item, &PyId___abstractmethods__); if (!base_abstracts) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; @@ -119,27 +128,41 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) value = PyObject_GetAttr((PyObject *)result, key); if (!value) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); return NULL; } PyErr_Clear(); + Py_DECREF(key); + Py_DECREF(value); continue; } - is_abstract = PyObject_GetAttrString(value, "__isabstractmethod__"); + is_abstract = _PyObject_GetAttrId(value, &PyId___isabstractmethod__); if (!is_abstract) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); return NULL; } PyErr_Clear(); + Py_DECREF(key); + Py_DECREF(value); continue; } if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); return NULL; } Py_DECREF(key); + Py_DECREF(value); } Py_DECREF(iter); } - if (PyObject_SetAttrString((PyObject *)result, "__abstractmethods__", abstracts) < 0) { + if (_PyObject_SetAttrId((PyObject *)result, &PyId___abstractmethods__, abstracts) < 0) { return NULL; } return (PyObject *)result; @@ -200,7 +223,7 @@ abcmeta_instancecheck(abc *self, PyObject *args) if (result == Py_True) { /* TODO: Refactor to avoid packing */ return Py_True; } - subclass = PyObject_GetAttrString(instance, "__class__"); + subclass = _PyObject_GetAttrId(instance, &PyId___class__); return abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); } @@ -215,7 +238,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } /* TODO: clear the registry from dead refs from time to time - on iteration here (have a counter for this) or maybe better durin a .register() call */ + on iteration here (have a counter for this) or maybe better during a .register() call */ /* TODO: Reset caches every n-th succes/failure correspondingly so that they don't grow too large */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); @@ -223,13 +246,12 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } if (ok == Py_True) { - Py_INCREF(Py_True); return Py_True; } if (ok == Py_False) { - Py_INCREF(Py_False); return Py_False; } + Py_DECREF(ok); mro = ((PyTypeObject *)subclass)->tp_mro; for (pos = 0; pos < PyTuple_Size(mro); pos++) { if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { @@ -242,9 +264,13 @@ abcmeta_subclasscheck(abc *self, PyObject *args) result = PyObject_IsSubclass(subclass, key); if (result > 0) { Py_INCREF(Py_True); + Py_DECREF(key); + Py_DECREF(iter); return Py_True; } if (result < 0) { + Py_DECREF(key); + Py_DECREF(iter); return NULL; } Py_DECREF(key); @@ -255,13 +281,16 @@ abcmeta_subclasscheck(abc *self, PyObject *args) result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (result > 0) { Py_INCREF(Py_True); + Py_DECREF(subclasses); return Py_True; } if (result < 0) { + Py_DECREF(subclasses); return NULL; } } Py_INCREF(Py_False); + Py_DECREF(subclasses); return Py_False; } @@ -376,13 +405,13 @@ static PyMethodDef abcmeta_methods[] = { }; static PyMemberDef abc_members[] = { - {"_abc_registry", T_OBJECT, offsetof(abc, abc_registry), 0, /* Maybe make these READONLY */ + {"_abc_registry", T_OBJECT, offsetof(abc, abc_registry), READONLY, PyDoc_STR("ABC registry (private).")}, - {"_abc_cache", T_OBJECT, offsetof(abc, abc_cache), 0, + {"_abc_cache", T_OBJECT, offsetof(abc, abc_cache), READONLY, PyDoc_STR("ABC positive cache (private).")}, - {"_abc_negative_cache", T_OBJECT, offsetof(abc, abc_negative_cache), 0, + {"_abc_negative_cache", T_OBJECT, offsetof(abc, abc_negative_cache), READONLY, PyDoc_STR("ABC negative cache (private).")}, - {"_abc_negative_cache_version", T_OBJECT, offsetof(abc, abc_negative_cache), 0, + {"_abc_negative_cache_version", T_OBJECT, offsetof(abc, abc_negative_cache), READONLY, PyDoc_STR("ABC negative cache version (private).")}, {NULL} }; From 569cc447c370031aac5839e8b3896799c476cb8f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 21 Jan 2018 22:09:03 +0000 Subject: [PATCH 13/93] Fix some more refleaks; use weak refs in registry --- Modules/_abc.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 0f7ea82cbf0a00..030285d3fa17bc 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -78,10 +78,14 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) result->abc_negative_cache = PySet_New(NULL); if (!result->abc_registry || !result->abc_cache || !result->abc_negative_cache) { + Py_DECREF(result); return NULL; } result->abc_negative_cache_version = abc_invalidation_counter; - abstracts = PyFrozenSet_New(NULL); + if (!(abstracts = PyFrozenSet_New(NULL))) { + Py_DECREF(result); + return NULL; + } /* Stage 1: direct abstract methods */ /* Safe to assume everything is fine since type.__new__ succeeded */ ns = PyTuple_GET_ITEM(args, 2); @@ -95,6 +99,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { Py_DECREF(item); Py_DECREF(items); + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } PyErr_Clear(); @@ -104,6 +110,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { Py_DECREF(item); Py_DECREF(items); + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } Py_DECREF(item); @@ -116,12 +124,16 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) base_abstracts = _PyObject_GetAttrId(item, &PyId___abstractmethods__); if (!base_abstracts) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } PyErr_Clear(); continue; } if (!(iter = PyObject_GetIter(base_abstracts))) { + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } while ((key = PyIter_Next(iter))) { @@ -131,6 +143,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } PyErr_Clear(); @@ -144,6 +158,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } PyErr_Clear(); @@ -155,6 +171,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } Py_DECREF(key); @@ -163,6 +181,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(iter); } if (_PyObject_SetAttrId((PyObject *)result, &PyId___abstractmethods__, abstracts) < 0) { + Py_DECREF(result); + Py_DECREF(abstracts); return NULL; } return (PyObject *)result; @@ -196,7 +216,7 @@ abcmeta_register(abc *self, PyObject *args) if (result < 0) { return NULL; } - if (PySet_Add(self->abc_registry, subclass) < 0) { + if (PySet_Add(self->abc_registry, PyWeakref_NewRef(subclass, NULL)) < 0) { return NULL; } Py_INCREF(subclass); @@ -231,7 +251,7 @@ static PyObject * abcmeta_subclasscheck(abc *self, PyObject *args) { PyObject *subclasses, *subclass = NULL; - PyObject *ok, *mro, *iter, *key; + PyObject *ok, *mro, *iter, *key, *rkey; Py_ssize_t pos; int result; if (!PyArg_UnpackTuple(args, "__subclasscheck__", 1, 1, &subclass)) { @@ -261,7 +281,11 @@ abcmeta_subclasscheck(abc *self, PyObject *args) } iter = PyObject_GetIter(self->abc_registry); while ((key = PyIter_Next(iter))) { - result = PyObject_IsSubclass(subclass, key); + rkey = PyWeakref_GetObject(key); + if (rkey == Py_None) { + continue; + } + result = PyObject_IsSubclass(subclass, rkey); if (result > 0) { Py_INCREF(Py_True); Py_DECREF(key); @@ -302,8 +326,10 @@ _print_message(PyObject *file, const char* message) return -1; } if (PyFile_WriteObject(mo, file, Py_PRINT_RAW)) { + Py_DECREF(mo); return -1; } + Py_DECREF(mo); return 0; } @@ -314,6 +340,7 @@ abcmeta_dump(abc *self, PyObject *args) PyObject *sizeo, *version = PyLong_FromSsize_t(self->abc_negative_cache_version); Py_ssize_t size; if (!PyArg_UnpackTuple(args, "_dump_registry", 0, 1, &file)) { + Py_XDECREF(version); return NULL; } if (!version) { @@ -323,66 +350,95 @@ abcmeta_dump(abc *self, PyObject *args) file = _PySys_GetObjectId(&PyId_stdout); if (file == NULL) { PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); + Py_DECREF(version); return NULL; } } /* Header */ if (_print_message(file, "Class: ")) { + Py_DECREF(version); return NULL; } if (_print_message(file, ((PyTypeObject *)self)->tp_name)) { + Py_DECREF(version); return NULL; } if (_print_message(file, "\n")) { + Py_DECREF(version); return NULL; } /* Registry */ if (_print_message(file, "Registry: ")) { + Py_DECREF(version); return NULL; } if (PyFile_WriteObject(self->abc_registry, file, Py_PRINT_RAW)) { + Py_DECREF(version); return NULL; } if (_print_message(file, "\n")) { + Py_DECREF(version); return NULL; } /* Postive cahce */ if (_print_message(file, "Positive cache: ")) { + Py_DECREF(version); return NULL; } size = PySet_GET_SIZE(self->abc_cache); if (!(sizeo = PyLong_FromSsize_t(size))) { + Py_DECREF(version); return NULL; } if (PyFile_WriteObject(sizeo, file, Py_PRINT_RAW)) { + Py_DECREF(version); + Py_DECREF(sizeo); return NULL; } if (_print_message(file, " items\n")) { + Py_DECREF(version); + Py_DECREF(sizeo); return NULL; } /* Negative cahce */ if (_print_message(file, "Negative cache: ")) { + Py_DECREF(version); + Py_DECREF(sizeo); return NULL; } + Py_DECREF(sizeo); size = PySet_GET_SIZE(self->abc_cache); if (!(sizeo = PyLong_FromSsize_t(size))) { + Py_DECREF(version); return NULL; } if (PyFile_WriteObject(sizeo, file, Py_PRINT_RAW)) { + Py_DECREF(sizeo); + Py_DECREF(version); return NULL; } if (_print_message(file, " items\n")) { + Py_DECREF(sizeo); + Py_DECREF(version); return NULL; } if (_print_message(file, "Negative cache version: ")) { + Py_DECREF(sizeo); + Py_DECREF(version); return NULL; } if (PyFile_WriteObject(version, file, Py_PRINT_RAW)) { + Py_DECREF(sizeo); + Py_DECREF(version); return NULL; } if (_print_message(file, "\n")) { + Py_DECREF(sizeo); + Py_DECREF(version); return NULL; } + Py_DECREF(sizeo); + Py_DECREF(version); Py_INCREF(Py_None); return Py_None; } From 34665a8ef47b042444169c1b11b799be81cb11aa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 21 Jan 2018 22:32:21 +0000 Subject: [PATCH 14/93] Some more fixes; add some caching --- Modules/_abc.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 030285d3fa17bc..67fc9ade2f72fa 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -230,21 +230,29 @@ abcmeta_subclasscheck(abc *self, PyObject *args); /* Forward */ static PyObject * abcmeta_instancecheck(abc *self, PyObject *args) { - PyObject *result, *subclass, *instance = NULL; + PyObject *result, *subclass, *subtype, *instance = NULL; if (!PyArg_UnpackTuple(args, "__instancecheck__", 1, 1, &instance)) { return NULL; } - subclass = (PyObject *)Py_TYPE(instance); - /* TODO: Use cache */ - result = abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); + subclass = _PyObject_GetAttrId(instance, &PyId___class__); + if (PySet_Contains(self->abc_cache, PyWeakref_NewRef(subclass, NULL)) > 0) { + Py_INCREF(Py_True); + return Py_True; + } + subtype = (PyObject *)Py_TYPE(instance); + if (subtype == subclass) { + 1 == 1; + } + + result = abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); /* TODO: Refactor to avoid packing + here and below. */ if (!result) { return NULL; } - if (result == Py_True) { /* TODO: Refactor to avoid packing */ + if (result == Py_True) { return Py_True; } - subclass = _PyObject_GetAttrId(instance, &PyId___class__); - return abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); + return abcmeta_subclasscheck(self, PyTuple_Pack(1, subtype)); } static PyObject * From 576acac44dc7bc228b820f92b0fd3abb20382de8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 21 Jan 2018 23:09:46 +0000 Subject: [PATCH 15/93] Fix a crash due to erroneous DECREF --- Modules/_abc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 67fc9ade2f72fa..80611ca83ac067 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -141,7 +141,6 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!value) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { Py_DECREF(key); - Py_DECREF(value); Py_DECREF(iter); Py_DECREF(result); Py_DECREF(abstracts); @@ -149,7 +148,6 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } PyErr_Clear(); Py_DECREF(key); - Py_DECREF(value); continue; } is_abstract = _PyObject_GetAttrId(value, &PyId___isabstractmethod__); From 30098b4d7f9d0671bfd5e0e340d4713fdf2d08fc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 22 Jan 2018 00:18:02 +0000 Subject: [PATCH 16/93] Simplify some code; reorganize TODOs --- Modules/_abc.c | 139 +++++++++++++++---------------------------------- 1 file changed, 43 insertions(+), 96 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 80611ca83ac067..a8049f1672c032 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,11 +1,11 @@ /* ABCMeta implementation */ -/* TODO: Global check where checks are needed, and where I made objects myself */ -/* In particular use capitals like PyList_GET_SIZE */ -/* Think (ask) about inlining some calls, like __subclasses__ */ -/* Work on INCREF/DECREF */ -/* Be sure to return NULL where exception possible */ -/* Use macros */ +/* TODO: + + Use fast paths where possible. + In particular use capitals like PyList_GET_SIZE. + Think (ask) about inlining some calls, like __subclasses__. + Fix refleaks. */ #include "Python.h" #include "structmember.h" @@ -73,7 +73,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!result) { return NULL; } - result->abc_registry = PySet_New(NULL); /* TODO: Delay registry creation until it is actually needed */ + result->abc_registry = PySet_New(NULL); result->abc_cache = PySet_New(NULL); result->abc_negative_cache = PySet_New(NULL); if (!result->abc_registry || !result->abc_cache || @@ -90,7 +90,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Safe to assume everything is fine since type.__new__ succeeded */ ns = PyTuple_GET_ITEM(args, 2); items = PyMapping_Items(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ - for (pos = 0; pos < PySequence_Size(items); pos++) { /* TODO: Check if it is a list or tuple? */ + for (pos = 0; pos < PySequence_Size(items); pos++) { item = PySequence_GetItem(items, pos); key = PyTuple_GetItem(item, 0); value = PyTuple_GetItem(item, 1); @@ -99,9 +99,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { Py_DECREF(item); Py_DECREF(items); - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } PyErr_Clear(); Py_DECREF(item); @@ -110,9 +108,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { Py_DECREF(item); Py_DECREF(items); - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } Py_DECREF(item); } @@ -124,17 +120,13 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) base_abstracts = _PyObject_GetAttrId(item, &PyId___abstractmethods__); if (!base_abstracts) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } PyErr_Clear(); continue; } if (!(iter = PyObject_GetIter(base_abstracts))) { - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } while ((key = PyIter_Next(iter))) { value = PyObject_GetAttr((PyObject *)result, key); @@ -142,9 +134,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { Py_DECREF(key); Py_DECREF(iter); - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } PyErr_Clear(); Py_DECREF(key); @@ -156,9 +146,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } PyErr_Clear(); Py_DECREF(key); @@ -169,9 +157,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } Py_DECREF(key); Py_DECREF(value); @@ -179,11 +165,13 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(iter); } if (_PyObject_SetAttrId((PyObject *)result, &PyId___abstractmethods__, abstracts) < 0) { - Py_DECREF(result); - Py_DECREF(abstracts); - return NULL; + goto error; } return (PyObject *)result; +error: + Py_DECREF(result); + Py_DECREF(abstracts); + return NULL; } static PyObject * @@ -243,7 +231,7 @@ abcmeta_instancecheck(abc *self, PyObject *args) } result = abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); /* TODO: Refactor to avoid packing - here and below. */ + here and below. */ if (!result) { return NULL; } @@ -264,7 +252,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } /* TODO: clear the registry from dead refs from time to time - on iteration here (have a counter for this) or maybe better during a .register() call */ + on iteration here (have a counter for this) or maybe during a .register() call */ /* TODO: Reset caches every n-th succes/failure correspondingly so that they don't grow too large */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); @@ -339,111 +327,70 @@ _print_message(PyObject *file, const char* message) return 0; } +int +_dump_attr(PyObject *file, PyObject *obj, char *msg) +{ + if (_print_message(file, msg)) { + return 0; + } + if (PyFile_WriteObject(obj, file, Py_PRINT_RAW)) { + return 0; + } + if (_print_message(file, "\n")) { + return 0; + } + return 1; +} + static PyObject * abcmeta_dump(abc *self, PyObject *args) { PyObject *file = NULL; - PyObject *sizeo, *version = PyLong_FromSsize_t(self->abc_negative_cache_version); - Py_ssize_t size; + PyObject *version; if (!PyArg_UnpackTuple(args, "_dump_registry", 0, 1, &file)) { - Py_XDECREF(version); - return NULL; - } - if (!version) { return NULL; } if (file == NULL || file == Py_None) { file = _PySys_GetObjectId(&PyId_stdout); if (file == NULL) { PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); - Py_DECREF(version); return NULL; } } - /* Header */ if (_print_message(file, "Class: ")) { - Py_DECREF(version); return NULL; } if (_print_message(file, ((PyTypeObject *)self)->tp_name)) { - Py_DECREF(version); - return NULL; - } - if (_print_message(file, "\n")) { - Py_DECREF(version); - return NULL; - } - /* Registry */ - if (_print_message(file, "Registry: ")) { - Py_DECREF(version); - return NULL; - } - if (PyFile_WriteObject(self->abc_registry, file, Py_PRINT_RAW)) { - Py_DECREF(version); return NULL; } if (_print_message(file, "\n")) { - Py_DECREF(version); return NULL; } - /* Postive cahce */ - if (_print_message(file, "Positive cache: ")) { - Py_DECREF(version); - return NULL; - } - size = PySet_GET_SIZE(self->abc_cache); - if (!(sizeo = PyLong_FromSsize_t(size))) { - Py_DECREF(version); + if (!_dump_attr(file, self->abc_registry, "Registry: ")) { return NULL; } - if (PyFile_WriteObject(sizeo, file, Py_PRINT_RAW)) { - Py_DECREF(version); - Py_DECREF(sizeo); + if (!_dump_attr(file, self->abc_cache, "Positive cache: ")) { return NULL; } - if (_print_message(file, " items\n")) { - Py_DECREF(version); - Py_DECREF(sizeo); + if (!_dump_attr(file, self->abc_negative_cache, "Negative cache: ")) { return NULL; } - /* Negative cahce */ - if (_print_message(file, "Negative cache: ")) { - Py_DECREF(version); - Py_DECREF(sizeo); - return NULL; - } - Py_DECREF(sizeo); - size = PySet_GET_SIZE(self->abc_cache); - if (!(sizeo = PyLong_FromSsize_t(size))) { - Py_DECREF(version); - return NULL; - } - if (PyFile_WriteObject(sizeo, file, Py_PRINT_RAW)) { - Py_DECREF(sizeo); - Py_DECREF(version); - return NULL; - } - if (_print_message(file, " items\n")) { - Py_DECREF(sizeo); - Py_DECREF(version); + version = PyLong_FromSsize_t(self->abc_negative_cache_version); + if (!version) { return NULL; } if (_print_message(file, "Negative cache version: ")) { - Py_DECREF(sizeo); Py_DECREF(version); return NULL; } if (PyFile_WriteObject(version, file, Py_PRINT_RAW)) { - Py_DECREF(sizeo); Py_DECREF(version); return NULL; } if (_print_message(file, "\n")) { - Py_DECREF(sizeo); Py_DECREF(version); return NULL; } - Py_DECREF(sizeo); Py_DECREF(version); Py_INCREF(Py_None); return Py_None; From 51ede5dcd0797c52c81ccc5036a1d23ded0a9453 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 22 Jan 2018 01:22:24 +0000 Subject: [PATCH 17/93] Finish caches; add more comments --- Modules/_abc.c | 141 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 18 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index a8049f1672c032..db3d1e117e797a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -19,6 +19,12 @@ _Py_IDENTIFIER(__isabstractmethod__); _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); + +/* A global counter that is incremented each time a class is + registered as a virtual subclass of anything. It forces the + negative cache to be cleared before its next use. + Note: this counter is private. Use `abc.get_cache_token()` for + external code. */ static Py_ssize_t abc_invalidation_counter = 0; typedef struct { @@ -73,11 +79,11 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!result) { return NULL; } + /* Set up inheritance registry. */ result->abc_registry = PySet_New(NULL); result->abc_cache = PySet_New(NULL); result->abc_negative_cache = PySet_New(NULL); - if (!result->abc_registry || !result->abc_cache || - !result->abc_negative_cache) { + if (!result->abc_registry || !result->abc_cache || !result->abc_negative_cache) { Py_DECREF(result); return NULL; } @@ -86,8 +92,9 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(result); return NULL; } - /* Stage 1: direct abstract methods */ - /* Safe to assume everything is fine since type.__new__ succeeded */ + /* Compute set of abstract method names in two stages: + Stage 1: direct abstract methods. + (It is safe to assume everything is fine since type.__new__ succeeded.) */ ns = PyTuple_GET_ITEM(args, 2); items = PyMapping_Items(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ for (pos = 0; pos < PySequence_Size(items); pos++) { @@ -113,7 +120,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(item); } Py_DECREF(items); - /* Stage 2: inherited abstract methods */ + /* Stage 2: inherited abstract methods. */ bases = PyTuple_GET_ITEM(args, 1); for (pos = 0; pos < PyTuple_Size(bases); pos++) { item = PyTuple_GET_ITEM(bases, pos); @@ -174,6 +181,37 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } +int +_in_weak_set(PyObject *set, PyObject *obj) +{ + PyObject *ref; + int res; + ref = PyWeakref_NewRef(obj, NULL); + if (!ref) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + return 0; + } + return -1; + } + res = PySet_Contains(set, ref); + Py_DECREF(ref); + return res; +} + +int +_add_weak_set(PyObject *set, PyObject *obj) +{ + PyObject *ref; + ref = PyWeakref_NewRef(obj, NULL); + if (!ref) { + return 0; + } + if (PySet_Add(set, ref) < 0) { + return 0; + } + return 1; +} + static PyObject * abcmeta_register(abc *self, PyObject *args) { @@ -189,24 +227,27 @@ abcmeta_register(abc *self, PyObject *args) result = PyObject_IsSubclass(subclass, (PyObject *)self); if (result > 0) { Py_INCREF(subclass); - return subclass; + return subclass; /* Already a subclass. */ } if (result < 0) { return NULL; } + /* Subtle: test for cycles *after* testing for "already a subclass"; + this means we allow X.register(X) and interpret it as a no-op. */ result = PyObject_IsSubclass((PyObject *)self, subclass); if (result > 0) { + /* This would create a cycle, which is bad for the algorithm below. */ PyErr_SetString(PyExc_RuntimeError, "Refusing to create an inheritance cycle"); return NULL; } if (result < 0) { return NULL; } - if (PySet_Add(self->abc_registry, PyWeakref_NewRef(subclass, NULL)) < 0) { + if (!_add_weak_set(self->abc_registry, subclass)) { return NULL; } Py_INCREF(subclass); - abc_invalidation_counter++; + abc_invalidation_counter++; /* Invalidate negative cache */ return subclass; } @@ -217,28 +258,41 @@ static PyObject * abcmeta_instancecheck(abc *self, PyObject *args) { PyObject *result, *subclass, *subtype, *instance = NULL; + int incache; if (!PyArg_UnpackTuple(args, "__instancecheck__", 1, 1, &instance)) { return NULL; } subclass = _PyObject_GetAttrId(instance, &PyId___class__); - if (PySet_Contains(self->abc_cache, PyWeakref_NewRef(subclass, NULL)) > 0) { + /* Inline the cache checking. */ + incache = _in_weak_set(self->abc_cache, subclass); + if (incache < 0) { + return NULL; + } + if (incache > 0) { Py_INCREF(Py_True); return Py_True; } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { - 1 == 1; + incache = _in_weak_set(self->abc_negative_cache, subclass); + if (incache < 0) { + return NULL; + } + if (self->abc_negative_cache_version == abc_invalidation_counter && incache) { + Py_INCREF(Py_False); + return Py_False; + } + /* Fall back to the subclass check. */ + return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subclass); } - - result = abcmeta_subclasscheck(self, PyTuple_Pack(1, subclass)); /* TODO: Refactor to avoid packing - here and below. */ + result = PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subclass); if (!result) { return NULL; } if (result == Py_True) { return Py_True; } - return abcmeta_subclasscheck(self, PyTuple_Pack(1, subtype)); + return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subtype); } static PyObject * @@ -247,7 +301,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) PyObject *subclasses, *subclass = NULL; PyObject *ok, *mro, *iter, *key, *rkey; Py_ssize_t pos; - int result; + int incache, result; if (!PyArg_UnpackTuple(args, "__subclasscheck__", 1, 1, &subclass)) { return NULL; } @@ -255,24 +309,64 @@ abcmeta_subclasscheck(abc *self, PyObject *args) on iteration here (have a counter for this) or maybe during a .register() call */ /* TODO: Reset caches every n-th succes/failure correspondingly so that they don't grow too large */ + + /* 1. Check cache. */ + incache = _in_weak_set(self->abc_cache, subclass); + if (incache < 0) { + return NULL; + } + if (incache > 0) { + Py_INCREF(Py_True); + return Py_True; + } + /* 2. Check negative cache; may have to invalidate. */ + incache = _in_weak_set(self->abc_negative_cache, subclass); + if (incache < 0) { + return NULL; + } + if (self->abc_negative_cache_version < abc_invalidation_counter) { + /* Invalidate the negative cache. */ + self->abc_negative_cache = PySet_New(NULL); + if (!self->abc_negative_cache) { + return NULL; + } + self->abc_negative_cache_version = abc_invalidation_counter; + } else if (incache) { + Py_INCREF(Py_False); + return Py_False; + } + /* 3. Check the subclass hook. */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); if (!ok) { return NULL; } if (ok == Py_True) { + if (!_add_weak_set(self->abc_cache, subclass)) { + Py_DECREF(ok); + return NULL; + } return Py_True; } if (ok == Py_False) { + if (!_add_weak_set(self->abc_negative_cache, subclass)) { + Py_DECREF(ok); + return NULL; + } return Py_False; } Py_DECREF(ok); + /* 4. Check if it's a direct subclass. */ mro = ((PyTypeObject *)subclass)->tp_mro; for (pos = 0; pos < PyTuple_Size(mro); pos++) { if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { Py_INCREF(Py_True); + if (!_add_weak_set(self->abc_cache, subclass)) { + return NULL; + } return Py_True; } } + /* 5. Check if it's a subclass of a registered class (recursive). */ iter = PyObject_GetIter(self->abc_registry); while ((key = PyIter_Next(iter))) { rkey = PyWeakref_GetObject(key); @@ -281,9 +375,12 @@ abcmeta_subclasscheck(abc *self, PyObject *args) } result = PyObject_IsSubclass(subclass, rkey); if (result > 0) { - Py_INCREF(Py_True); Py_DECREF(key); Py_DECREF(iter); + if (!_add_weak_set(self->abc_cache, subclass)) { + return NULL; + } + Py_INCREF(Py_True); return Py_True; } if (result < 0) { @@ -294,12 +391,16 @@ abcmeta_subclasscheck(abc *self, PyObject *args) Py_DECREF(key); } Py_DECREF(iter); + /* 6. Check if it's a subclass of a subclass (recursive). */ subclasses = PyObject_CallMethod((PyObject *)self, "__subclasses__", NULL); for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (result > 0) { - Py_INCREF(Py_True); + if (!_add_weak_set(self->abc_cache, subclass)) { + return NULL; + } Py_DECREF(subclasses); + Py_INCREF(Py_True); return Py_True; } if (result < 0) { @@ -307,8 +408,12 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } } - Py_INCREF(Py_False); Py_DECREF(subclasses); + /* No dice; update negative cache. */ + if (!_add_weak_set(self->abc_negative_cache, subclass)) { + return NULL; + } + Py_INCREF(Py_False); return Py_False; } From 5263e1a3cea8d12a795e685731a8386004129f30 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 24 Jan 2018 17:53:07 +0900 Subject: [PATCH 18/93] Use Py_RETURN_TRUE/FALSE and fix refleak --- Modules/_abc.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index db3d1e117e797a..df69cda1084a3d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -269,8 +269,7 @@ abcmeta_instancecheck(abc *self, PyObject *args) return NULL; } if (incache > 0) { - Py_INCREF(Py_True); - return Py_True; + Py_RETURN_TRUE; } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { @@ -279,8 +278,7 @@ abcmeta_instancecheck(abc *self, PyObject *args) return NULL; } if (self->abc_negative_cache_version == abc_invalidation_counter && incache) { - Py_INCREF(Py_False); - return Py_False; + Py_RETURN_FALSE; } /* Fall back to the subclass check. */ return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subclass); @@ -292,6 +290,7 @@ abcmeta_instancecheck(abc *self, PyObject *args) if (result == Py_True) { return Py_True; } + Py_DECREF(result); return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subtype); } @@ -316,8 +315,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } if (incache > 0) { - Py_INCREF(Py_True); - return Py_True; + Py_RETURN_TRUE; } /* 2. Check negative cache; may have to invalidate. */ incache = _in_weak_set(self->abc_negative_cache, subclass); @@ -332,8 +330,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) } self->abc_negative_cache_version = abc_invalidation_counter; } else if (incache) { - Py_INCREF(Py_False); - return Py_False; + Py_RETURN_FALSE; } /* 3. Check the subclass hook. */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); @@ -359,11 +356,10 @@ abcmeta_subclasscheck(abc *self, PyObject *args) mro = ((PyTypeObject *)subclass)->tp_mro; for (pos = 0; pos < PyTuple_Size(mro); pos++) { if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { - Py_INCREF(Py_True); if (!_add_weak_set(self->abc_cache, subclass)) { return NULL; } - return Py_True; + Py_RETURN_TRUE; } } /* 5. Check if it's a subclass of a registered class (recursive). */ @@ -380,8 +376,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) if (!_add_weak_set(self->abc_cache, subclass)) { return NULL; } - Py_INCREF(Py_True); - return Py_True; + Py_RETURN_TRUE; } if (result < 0) { Py_DECREF(key); @@ -400,8 +395,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } Py_DECREF(subclasses); - Py_INCREF(Py_True); - return Py_True; + Py_RETURN_TRUE; } if (result < 0) { Py_DECREF(subclasses); @@ -413,8 +407,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) if (!_add_weak_set(self->abc_negative_cache, subclass)) { return NULL; } - Py_INCREF(Py_False); - return Py_False; + Py_RETURN_FALSE; } int From 1f7aee9aa16d3d26b68ff81968b86c3767530dde Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 24 Jan 2018 18:05:55 +0900 Subject: [PATCH 19/93] Use Py_RETURN_NONE --- Modules/_abc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index df69cda1084a3d..e72f7530a99a55 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -490,8 +490,7 @@ abcmeta_dump(abc *self, PyObject *args) return NULL; } Py_DECREF(version); - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } PyDoc_STRVAR(_register_doc, From 11fea706fe72848b58032c57b7c92481c7c0ee65 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 24 Jan 2018 18:50:38 +0900 Subject: [PATCH 20/93] Use _PyObject_IsAbstract() --- Modules/_abc.c | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index e72f7530a99a55..834974a1a9d339 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -15,7 +15,6 @@ PyDoc_STRVAR(_abc__doc__, #define DEFERRED_ADDRESS(ADDR) 0 _Py_IDENTIFIER(stdout); -_Py_IDENTIFIER(__isabstractmethod__); _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); @@ -72,7 +71,7 @@ static PyObject * abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { abc *result = NULL; - PyObject *ns, *bases, *items, *abstracts, *is_abstract, *base_abstracts; + PyObject *ns, *bases, *items, *abstracts, *base_abstracts; PyObject *key, *value, *item, *iter; Py_ssize_t pos = 0; result = (abc *)PyType_Type.tp_new(type, args, kwds); @@ -92,6 +91,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(result); return NULL; } + /* Compute set of abstract method names in two stages: Stage 1: direct abstract methods. (It is safe to assume everything is fine since type.__new__ succeeded.) */ @@ -101,18 +101,13 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) item = PySequence_GetItem(items, pos); key = PyTuple_GetItem(item, 0); value = PyTuple_GetItem(item, 1); - is_abstract = _PyObject_GetAttrId(value, &PyId___isabstractmethod__); - if (!is_abstract) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - Py_DECREF(item); - Py_DECREF(items); - goto error; - } - PyErr_Clear(); + int is_abstract = _PyObject_IsAbstract(value); + if (is_abstract < 0) { Py_DECREF(item); - continue; + Py_DECREF(items); + goto error; } - if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { + if (is_abstract && PySet_Add(abstracts, key) < 0) { Py_DECREF(item); Py_DECREF(items); goto error; @@ -120,6 +115,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(item); } Py_DECREF(items); + /* Stage 2: inherited abstract methods. */ bases = PyTuple_GET_ITEM(args, 1); for (pos = 0; pos < PyTuple_Size(bases); pos++) { @@ -147,27 +143,19 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(key); continue; } - is_abstract = _PyObject_GetAttrId(value, &PyId___isabstractmethod__); - if (!is_abstract) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - Py_DECREF(key); - Py_DECREF(value); - Py_DECREF(iter); - goto error; - } - PyErr_Clear(); + int is_abstract = _PyObject_IsAbstract(value); + Py_DECREF(value); + if (is_abstract < 0) { Py_DECREF(key); - Py_DECREF(value); - continue; + Py_DECREF(iter); + goto error; } - if (is_abstract == Py_True && PySet_Add(abstracts, key) < 0) { + if (is_abstract && PySet_Add(abstracts, key) < 0) { Py_DECREF(key); - Py_DECREF(value); Py_DECREF(iter); goto error; } Py_DECREF(key); - Py_DECREF(value); } Py_DECREF(iter); } From 7ff3fbb61305fce399ff8321fe2d8cf9708812d4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 24 Jan 2018 19:17:34 +0900 Subject: [PATCH 21/93] Use _PySet_NextEntry --- Modules/_abc.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 834974a1a9d339..c466d88a1bb6aa 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -286,7 +286,7 @@ static PyObject * abcmeta_subclasscheck(abc *self, PyObject *args) { PyObject *subclasses, *subclass = NULL; - PyObject *ok, *mro, *iter, *key, *rkey; + PyObject *ok, *mro, *key, *rkey; Py_ssize_t pos; int incache, result; if (!PyArg_UnpackTuple(args, "__subclasscheck__", 1, 1, &subclass)) { @@ -350,30 +350,27 @@ abcmeta_subclasscheck(abc *self, PyObject *args) Py_RETURN_TRUE; } } + /* 5. Check if it's a subclass of a registered class (recursive). */ - iter = PyObject_GetIter(self->abc_registry); - while ((key = PyIter_Next(iter))) { + pos = 0; + Py_hash_t hash; + while (_PySet_NextEntry(self->abc_registry, &pos, &key, &hash)) { rkey = PyWeakref_GetObject(key); if (rkey == Py_None) { continue; } result = PyObject_IsSubclass(subclass, rkey); + if (result < 0) { + return NULL; + } if (result > 0) { - Py_DECREF(key); - Py_DECREF(iter); if (!_add_weak_set(self->abc_cache, subclass)) { return NULL; } Py_RETURN_TRUE; } - if (result < 0) { - Py_DECREF(key); - Py_DECREF(iter); - return NULL; - } - Py_DECREF(key); } - Py_DECREF(iter); + /* 6. Check if it's a subclass of a subclass (recursive). */ subclasses = PyObject_CallMethod((PyObject *)self, "__subclasses__", NULL); for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { @@ -581,7 +578,8 @@ get_cache_token(void) } static struct PyMethodDef module_functions[] = { - {"get_cache_token", get_cache_token, METH_NOARGS, _cache_token_doc}, + {"get_cache_token", (PyCFunction)get_cache_token, METH_NOARGS, + _cache_token_doc}, {NULL, NULL} /* sentinel */ }; From 9b4eb2f39dbbe1312394cc59e30b214127e7c132 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 Jan 2018 01:03:13 +0000 Subject: [PATCH 22/93] Minor review comments --- Modules/_abc.c | 11 +++++++++++ setup.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/Modules/_abc.c b/Modules/_abc.c index c466d88a1bb6aa..5a2b0bbbbdb175 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -97,8 +97,16 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) (It is safe to assume everything is fine since type.__new__ succeeded.) */ ns = PyTuple_GET_ITEM(args, 2); items = PyMapping_Items(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ + if (!items) { + Py_DECREF(result); + return NULL; + } for (pos = 0; pos < PySequence_Size(items); pos++) { item = PySequence_GetItem(items, pos); + if (!PyTuple_Check(item) || (PyTuple_GET_SIZE(item) != 2)) { + PyErr_SetString(PyExc_TypeError, "Iteration over class namespace must" + " yield length-two tuples"); + } key = PyTuple_GetItem(item, 0); value = PyTuple_GetItem(item, 1); int is_abstract = _PyObject_IsAbstract(value); @@ -158,6 +166,9 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(key); } Py_DECREF(iter); + if (PyErr_Occurred()) { + goto error; + } } if (_PyObject_SetAttrId((PyObject *)result, &PyId___abstractmethods__, abstracts) < 0) { goto error; diff --git a/setup.py b/setup.py index 8e98fdcb7ef3ca..8d41e192eb3079 100644 --- a/setup.py +++ b/setup.py @@ -699,6 +699,8 @@ def detect_modules(self): exts.append( Extension('_opcode', ['_opcode.c']) ) # asyncio speedups exts.append( Extension("_asyncio", ["_asynciomodule.c"]) ) + # _abc speedups + exts.append( Extension("_abc", ["_abc.c"]) ) # _queue module exts.append( Extension("_queue", ["_queuemodule.c"]) ) From ab20a334cceff4133149f51a49280e21987c89c2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 Jan 2018 10:28:07 +0000 Subject: [PATCH 23/93] Sketch the new API --- Modules/_abc.c | 331 ++++++++++++++----------------------------------- 1 file changed, 90 insertions(+), 241 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 5a2b0bbbbdb175..296a3897b384ee 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -26,46 +26,6 @@ _Py_IDENTIFIER(__class__); external code. */ static Py_ssize_t abc_invalidation_counter = 0; -typedef struct { - PyHeapTypeObject tp; - PyObject *abc_registry; /* these three are normal set of weakrefs without callback */ - PyObject *abc_cache; - PyObject *abc_negative_cache; - Py_ssize_t abc_negative_cache_version; -} abc; - -static void -abcmeta_dealloc(abc *tp) -{ - Py_CLEAR(tp->abc_registry); - Py_CLEAR(tp->abc_cache); - Py_CLEAR(tp->abc_negative_cache); - Py_TYPE(tp)->tp_free((PyObject *)tp); -} - -static int -abcmeta_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(((abc *)self)->abc_registry); - Py_VISIT(((abc *)self)->abc_cache); - Py_VISIT(((abc *)self)->abc_negative_cache); - return PyType_Type.tp_traverse(self, visit, arg); -} - -static int -abcmeta_clear(abc *tp) -{ - if (tp->abc_registry) { - PySet_Clear(tp->abc_registry); - } - if (tp->abc_cache) { - PySet_Clear(tp->abc_cache); - } - if (tp->abc_negative_cache) { - PySet_Clear(tp->abc_negative_cache); - } - return PyType_Type.tp_clear((PyObject *)tp); -} static PyObject * abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -242,7 +202,7 @@ abcmeta_register(abc *self, PyObject *args) if (result < 0) { return NULL; } - if (!_add_weak_set(self->abc_registry, subclass)) { + if (!_add_to_registry(self, subclass)) { return NULL; } Py_INCREF(subclass); @@ -263,7 +223,7 @@ abcmeta_instancecheck(abc *self, PyObject *args) } subclass = _PyObject_GetAttrId(instance, &PyId___class__); /* Inline the cache checking. */ - incache = _in_weak_set(self->abc_cache, subclass); + incache = _in_cache(self, subclass); if (incache < 0) { return NULL; } @@ -272,11 +232,11 @@ abcmeta_instancecheck(abc *self, PyObject *args) } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { - incache = _in_weak_set(self->abc_negative_cache, subclass); + incache = _in_negative_cache(self, subclass); if (incache < 0) { return NULL; } - if (self->abc_negative_cache_version == abc_invalidation_counter && incache) { + if (_get_negative_cache_version(self) == abc_invalidation_counter && incache) { Py_RETURN_FALSE; } /* Fall back to the subclass check. */ @@ -293,8 +253,80 @@ abcmeta_instancecheck(abc *self, PyObject *args) return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subtype); } +int +_add_to_cache(PyObject *self, PyObject *cls) +{ + return 0; +} + +int +_add_to_negative_cache(PyObject *self, PyObject *cls) +{ + return 0; +} + +int +_in_cache(PyObject *self, PyObject *cls) +{ + return 0; +} + +int +_in_negative_cache(PyObject *self, PyObject *cls) +{ + return 0; +} + +int +_get_registry(PyObject *self) +{ + return 0; +} + +int +_add_to_registry(PyObject *self, PyObject *cls) +{ + return 0; +} + +int +_reset_registry(PyObject *self) +{ + return 0; +} + +int +_reset_cache(PyObject *self) +{ + return 0; +} + +int +_reset_negative_cache(PyObject *self) +{ + return 0; +} + +int +_get_negative_cache_version(PyObject *self) +{ + return 0; +} + +int +_set_negative_cache_version(PyObject *self, Py_ssize_t version) +{ + return 0; +} + +PyObject * +_get_dump() +{ + return NULL; +} + static PyObject * -abcmeta_subclasscheck(abc *self, PyObject *args) +abcmeta_subclasscheck(PyObject *self, PyObject *args) { PyObject *subclasses, *subclass = NULL; PyObject *ok, *mro, *key, *rkey; @@ -309,7 +341,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) so that they don't grow too large */ /* 1. Check cache. */ - incache = _in_weak_set(self->abc_cache, subclass); + incache = _in_cache(self, subclass); if (incache < 0) { return NULL; } @@ -317,17 +349,16 @@ abcmeta_subclasscheck(abc *self, PyObject *args) Py_RETURN_TRUE; } /* 2. Check negative cache; may have to invalidate. */ - incache = _in_weak_set(self->abc_negative_cache, subclass); + incache = _in_negative_cache(self, subclass); if (incache < 0) { return NULL; } - if (self->abc_negative_cache_version < abc_invalidation_counter) { + if (_get_negative_cache_version(self) < abc_invalidation_counter) { /* Invalidate the negative cache. */ - self->abc_negative_cache = PySet_New(NULL); - if (!self->abc_negative_cache) { + if (!_reset_negative_cache(self)) { return NULL; } - self->abc_negative_cache_version = abc_invalidation_counter; + _set_negative_cache_version(self, abc_invalidation_counter); } else if (incache) { Py_RETURN_FALSE; } @@ -337,14 +368,14 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } if (ok == Py_True) { - if (!_add_weak_set(self->abc_cache, subclass)) { + if (!_add_to_cache(self, subclass)) { Py_DECREF(ok); return NULL; } return Py_True; } if (ok == Py_False) { - if (!_add_weak_set(self->abc_negative_cache, subclass)) { + if (!_add_to_negative_cache(self, subclass)) { Py_DECREF(ok); return NULL; } @@ -355,7 +386,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) mro = ((PyTypeObject *)subclass)->tp_mro; for (pos = 0; pos < PyTuple_Size(mro); pos++) { if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { - if (!_add_weak_set(self->abc_cache, subclass)) { + if (!_add_to_cache(self, subclass)) { return NULL; } Py_RETURN_TRUE; @@ -365,7 +396,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) /* 5. Check if it's a subclass of a registered class (recursive). */ pos = 0; Py_hash_t hash; - while (_PySet_NextEntry(self->abc_registry, &pos, &key, &hash)) { + while (_PySet_NextEntry(_get_registry(self), &pos, &key, &hash)) { rkey = PyWeakref_GetObject(key); if (rkey == Py_None) { continue; @@ -375,7 +406,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) return NULL; } if (result > 0) { - if (!_add_weak_set(self->abc_cache, subclass)) { + if (!_add_to_cache(self, subclass)) { return NULL; } Py_RETURN_TRUE; @@ -387,7 +418,7 @@ abcmeta_subclasscheck(abc *self, PyObject *args) for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (result > 0) { - if (!_add_weak_set(self->abc_cache, subclass)) { + if (!_add_to_cache(self, subclass)) { return NULL; } Py_DECREF(subclasses); @@ -400,180 +431,12 @@ abcmeta_subclasscheck(abc *self, PyObject *args) } Py_DECREF(subclasses); /* No dice; update negative cache. */ - if (!_add_weak_set(self->abc_negative_cache, subclass)) { + if (!_add_to_negative_cache(self, subclass)) { return NULL; } Py_RETURN_FALSE; } -int -_print_message(PyObject *file, const char* message) -{ - PyObject *mo = PyUnicode_FromString(message); - if (!mo) { - return -1; - } - if (PyFile_WriteObject(mo, file, Py_PRINT_RAW)) { - Py_DECREF(mo); - return -1; - } - Py_DECREF(mo); - return 0; -} - -int -_dump_attr(PyObject *file, PyObject *obj, char *msg) -{ - if (_print_message(file, msg)) { - return 0; - } - if (PyFile_WriteObject(obj, file, Py_PRINT_RAW)) { - return 0; - } - if (_print_message(file, "\n")) { - return 0; - } - return 1; -} - -static PyObject * -abcmeta_dump(abc *self, PyObject *args) -{ - PyObject *file = NULL; - PyObject *version; - if (!PyArg_UnpackTuple(args, "_dump_registry", 0, 1, &file)) { - return NULL; - } - if (file == NULL || file == Py_None) { - file = _PySys_GetObjectId(&PyId_stdout); - if (file == NULL) { - PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); - return NULL; - } - } - if (_print_message(file, "Class: ")) { - return NULL; - } - if (_print_message(file, ((PyTypeObject *)self)->tp_name)) { - return NULL; - } - if (_print_message(file, "\n")) { - return NULL; - } - if (!_dump_attr(file, self->abc_registry, "Registry: ")) { - return NULL; - } - if (!_dump_attr(file, self->abc_cache, "Positive cache: ")) { - return NULL; - } - if (!_dump_attr(file, self->abc_negative_cache, "Negative cache: ")) { - return NULL; - } - version = PyLong_FromSsize_t(self->abc_negative_cache_version); - if (!version) { - return NULL; - } - if (_print_message(file, "Negative cache version: ")) { - Py_DECREF(version); - return NULL; - } - if (PyFile_WriteObject(version, file, Py_PRINT_RAW)) { - Py_DECREF(version); - return NULL; - } - if (_print_message(file, "\n")) { - Py_DECREF(version); - return NULL; - } - Py_DECREF(version); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(_register_doc, -"Register a virtual subclass of an ABC.\n\ -\n\ -Returns the subclass, to allow usage as a class decorator."); - -static PyMethodDef abcmeta_methods[] = { - {"register", (PyCFunction)abcmeta_register, METH_VARARGS, - _register_doc}, - {"__instancecheck__", (PyCFunction)abcmeta_instancecheck, METH_VARARGS, - PyDoc_STR("Override for isinstance(instance, cls).")}, - {"__subclasscheck__", (PyCFunction)abcmeta_subclasscheck, METH_VARARGS, - PyDoc_STR("Override for issubclass(subclass, cls).")}, - {"_dump_registry", (PyCFunction)abcmeta_dump, METH_VARARGS, - PyDoc_STR("Debug helper to print the ABC registry.")}, - {NULL, NULL}, -}; - -static PyMemberDef abc_members[] = { - {"_abc_registry", T_OBJECT, offsetof(abc, abc_registry), READONLY, - PyDoc_STR("ABC registry (private).")}, - {"_abc_cache", T_OBJECT, offsetof(abc, abc_cache), READONLY, - PyDoc_STR("ABC positive cache (private).")}, - {"_abc_negative_cache", T_OBJECT, offsetof(abc, abc_negative_cache), READONLY, - PyDoc_STR("ABC negative cache (private).")}, - {"_abc_negative_cache_version", T_OBJECT, offsetof(abc, abc_negative_cache), READONLY, - PyDoc_STR("ABC negative cache version (private).")}, - {NULL} -}; - -PyDoc_STRVAR(abcmeta_doc, - "Metaclass for defining Abstract Base Classes (ABCs).\n\ -\n\ -Use this metaclass to create an ABC. An ABC can be subclassed\n\ -directly, and then acts as a mix-in class. You can also register\n\ -unrelated concrete classes (even built-in classes) and unrelated\n\ -ABCs as 'virtual subclasses' -- these and their descendants will\n\ -be considered subclasses of the registering ABC by the built-in\n\ -issubclass() function, but the registering ABC won't show up in\n\ -their MRO (Method Resolution Order) nor will method\n\ -implementations defined by the registering ABC be callable (not\n\ -even via super())."); - -PyTypeObject ABCMeta = { - PyVarObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type), 0) - "ABCMeta", /* tp_name */ - sizeof(abc), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)abcmeta_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VERSION_TAG | - Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, /* tp_flags */ - abcmeta_doc, /* tp_doc */ - abcmeta_traverse, /* tp_traverse */ - (inquiry)abcmeta_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - abcmeta_methods, /* tp_methods */ - abc_members, /* tp_members */ - 0, /* tp_getset */ - DEFERRED_ADDRESS(&PyType_Type), /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - abcmeta_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ -}; PyDoc_STRVAR(_cache_token_doc, "Returns the current ABC cache token.\n\ @@ -610,19 +473,5 @@ static struct PyModuleDef _abcmodule = { PyMODINIT_FUNC PyInit__abc(void) { - PyObject *m; - - m = PyModule_Create(&_abcmodule); - if (m == NULL) - return NULL; - ABCMeta.tp_base = &PyType_Type; - if (PyType_Ready(&ABCMeta) < 0) { - return NULL; - } - Py_INCREF(&ABCMeta); - if (PyModule_AddObject(m, "ABCMeta", - (PyObject *) &ABCMeta) < 0) { - return NULL; - } - return m; + return PyModule_Create(&_abcmodule); } From 493d0ec432a4df066c66a851518221f46c678dea Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 Jan 2018 10:35:29 +0000 Subject: [PATCH 24/93] Use _PyObject_LookupAttr --- Modules/_abc.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 296a3897b384ee..a3cb20f39078ca 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -34,6 +34,7 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *ns, *bases, *items, *abstracts, *base_abstracts; PyObject *key, *value, *item, *iter; Py_ssize_t pos = 0; + int ret; result = (abc *)PyType_Type.tp_new(type, args, kwds); if (!result) { return NULL; @@ -88,26 +89,22 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) bases = PyTuple_GET_ITEM(args, 1); for (pos = 0; pos < PyTuple_Size(bases); pos++) { item = PyTuple_GET_ITEM(bases, pos); - base_abstracts = _PyObject_GetAttrId(item, &PyId___abstractmethods__); - if (!base_abstracts) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - goto error; - } - PyErr_Clear(); + ret = _PyObject_LookupAttrId(item, &PyId___abstractmethods__, &base_abstracts); + if (ret < 0) { + goto error; + } else if (!ret) { continue; } if (!(iter = PyObject_GetIter(base_abstracts))) { goto error; } while ((key = PyIter_Next(iter))) { - value = PyObject_GetAttr((PyObject *)result, key); - if (!value) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - Py_DECREF(key); - Py_DECREF(iter); - goto error; - } - PyErr_Clear(); + ret = _PyObject_LookupAttr((PyObject *)result, key, &value); + if (ret < 0) { + Py_DECREF(key); + Py_DECREF(iter); + goto error; + } else if (!ret) { Py_DECREF(key); continue; } From 2fe2c547c15e0626ab05212bbd9eb6384a567e1d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 26 Jan 2018 01:42:58 +0000 Subject: [PATCH 25/93] Some more progress --- Lib/abc.py | 76 ++++++---- Modules/_abc.c | 403 +++++++++++++++++++++++++++++++------------------ 2 files changed, 307 insertions(+), 172 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 2e13fa8020123c..b83cb70abe2cb9 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -4,6 +4,29 @@ """Abstract Base Classes (ABCs) according to PEP 3119.""" +def get_cache_token(): + """Returns the current ABC cache token. + + The token is an opaque object (supporting equality testing) identifying the + current version of the ABC cache for virtual subclasses. The token changes + with every call to ``register()`` on any ABC. + """ + return ABCMeta._abc_invalidation_counter + + +_C_speedup = False + +try: + from _abc import (get_cache_token, _abc_init, _abc_register, + _abc_instancecheck, _abc_subclasscheck, _get_dump, + _reset_registry, _reset_caches) +except ImportError: + # We postpone this import to speed-up Python start-up time + from _weakrefset import WeakSet +else: + _C_speedup = True + + def abstractmethod(funcobj): """A decorator indicating abstract methods. @@ -25,8 +48,7 @@ def my_abstract_method(self, ...): class abstractclassmethod(classmethod): - """ - A decorator indicating abstract classmethods. + """A decorator indicating abstract classmethods. Similar to abstractmethod. @@ -49,8 +71,7 @@ def __init__(self, callable): class abstractstaticmethod(staticmethod): - """ - A decorator indicating abstract staticmethods. + """A decorator indicating abstract staticmethods. Similar to abstractmethod. @@ -73,8 +94,7 @@ def __init__(self, callable): class abstractproperty(property): - """ - A decorator indicating abstract properties. + """A decorator indicating abstract properties. Requires that the metaclass is ABCMeta or derived from it. A class that has a metaclass derived from ABCMeta cannot be @@ -105,7 +125,6 @@ def setx(self, value): ... class ABCMeta(type): - """Metaclass for defining Abstract Base Classes (ABCs). Use this metaclass to create an ABC. An ABC can be subclassed @@ -117,7 +136,6 @@ class ABCMeta(type): their MRO (Method Resolution Order) nor will method implementations defined by the registering ABC be callable (not even via super()). - """ # A global counter that is incremented each time a class is @@ -129,6 +147,9 @@ class ABCMeta(type): def __new__(mcls, name, bases, namespace, **kwargs): cls = super().__new__(mcls, name, bases, namespace, **kwargs) + if _C_speedup: + _abc_init(cls) + return cls # Compute set of abstract method names abstracts = {name for name, value in namespace.items() @@ -151,6 +172,9 @@ def register(cls, subclass): Returns the subclass, to allow usage as a class decorator. """ + if _C_speedup: + _abc_register(cls, subclass) + return subclass if not isinstance(subclass, type): raise TypeError("Can only register classes") if issubclass(subclass, cls): @@ -166,17 +190,28 @@ def register(cls, subclass): def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" - print("Class: %s.%s" % (cls.__module__, cls.__qualname__), file=file) - print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file) + print(f"Class: {cls.__module__}.{cls.__qualname__}"), file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + if _C_speedup: + (_abc_registry, _abc_cache, _abc_negative_cache, + _abc_negative_cache_version) = _get_dump(cls) + print(f"_abc_registry: {_abc_registry!r}", file=file) + print(f"_abc_cache: {_abc_cache!r}", file=file) + print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) + print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", + file=file) + return for name in cls.__dict__: if name.startswith("_abc_"): value = getattr(cls, name) if isinstance(value, WeakSet): value = set(value) - print("%s: %r" % (name, value), file=file) + print(f"{name}: {value!r}", file=file) def __instancecheck__(cls, instance): """Override for isinstance(instance, cls).""" + if _C_speedup: + return _abc_instancecheck(cls, instance) # Inline the cache checking subclass = instance.__class__ if subclass in cls._abc_cache: @@ -193,6 +228,8 @@ def __instancecheck__(cls, instance): def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" + if _C_speedup: + return _abc_subclasscheck(cls, subclass) # Check cache if subclass in cls._abc_cache: return True @@ -231,23 +268,6 @@ def __subclasscheck__(cls, subclass): return False -def get_cache_token(): - """Returns the current ABC cache token. - - The token is an opaque object (supporting equality testing) identifying the - current version of the ABC cache for virtual subclasses. The token changes - with every call to ``register()`` on any ABC. - """ - return ABCMeta._abc_invalidation_counter - - -try: - from _abc import ABCMeta, get_cache_token -except ImportError: - # We postpone this import to speed-up Python start-up time - from _weakrefset import WeakSet - - class ABC(metaclass=ABCMeta): """Helper class that provides a standard way to create an ABC using inheritance. diff --git a/Modules/_abc.c b/Modules/_abc.c index a3cb20f39078ca..0b9f86c7b99138 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,11 +1,16 @@ /* ABCMeta implementation */ -/* TODO: - - Use fast paths where possible. - In particular use capitals like PyList_GET_SIZE. - Think (ask) about inlining some calls, like __subclasses__. - Fix refleaks. */ +/* TODO: Expose public functions */ +/* TODO: Add callbacks */ +/* TODO: Get it compiled */ +/* TODO: Get tests passing */ +/* TODO: Add iteration guard */ +/* TODO: DECREF all WeakRefs! */ +/* TODO: Fix refleaks! */ +/* TODO: Think (ask) about thread-safety. */ +/* TODO: Add checks only to those that can fail and use GET_SIZE etc. */ +/* TODO: Think (ask) about inlining some calls (like __subclasses__) and/or macros */ +/* TODO: Use separate branches with "fast paths" */ #include "Python.h" #include "structmember.h" @@ -17,6 +22,7 @@ PyDoc_STRVAR(_abc__doc__, _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); +_Py_IDENTIFIER(__dict__); /* A global counter that is incremented each time a class is @@ -24,29 +30,235 @@ _Py_IDENTIFIER(__class__); negative cache to be cleared before its next use. Note: this counter is private. Use `abc.get_cache_token()` for external code. */ -static Py_ssize_t abc_invalidation_counter = 0; +static PyObject *abc_invalidation_counter = PyLong_FromLong(0); + +static PyObject *_the_registry = PyDict_New(NULL); +static PyObject *_the_cache = PyDict_New(NULL); +static PyObject *_the_negative_cache = PyDict_New(NULL); +static PyObject *_the_cache_version = PyDict_New(NULL); + +int +_in_weak_set(PyObject *set, PyObject *obj) +{ + PyObject *ref; + int res; + ref = PyWeakref_NewRef(obj, NULL); + if (!ref) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + return 0; + } + return -1; + } + res = PySet_Contains(set, ref); + Py_DECREF(ref); + return res; +} + +int +_in_cache(PyObject *self, PyObject *cls) +{ + PyObject *key, *cache; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + cache = PyObject_GetItem(_the_cache, key); + if (!cache) { + return 0; + } + return _in_weak_set(cache, cls); +} + +int +_in_negative_cache(PyObject *self, PyObject *cls) +{ + PyObject *key, *cache; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + cache = PyObject_GetItem(_the_negative_cache, key); + if (!cache) { + return 0; + } + return _in_weak_set(cache, cls); +} + +int +_add_to_weak_set(PyObject *set, PyObject *obj) +{ + PyObject *ref; + ref = PyWeakref_NewRef(obj, NULL); + if (!ref) { + return 0; + } + if (PySet_Add(set, ref) < 0) { + return 0; + } + return 1; +} +PyObject * +_get_registry(PyObject *self) +{ + PyObject *key; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + return PyObject_GetItem(_the_registry, key); +} + +int +_add_to_registry(PyObject *self, PyObject *cls) +{ + PyObject *registry = _get_registry(self); + if (!registry) { + return 0; + } + return _add_to_weak_set(registry, cls); +} + +int +_add_to_cache(PyObject *self, PyObject *cls) +{ + PyObject *key, *cache; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + cache = PyObject_GetItem(_the_cache, key); + if (!cache) { + return 0; + } + return _add_to_weak_set(cache, cls); +} + +int +_add_to_negative_cache(PyObject *self, PyObject *cls) +{ + PyObject *key, *cache; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + cache = PyObject_GetItem(_the_negative_cache, key); + if (!cache) { + return 0; + } + return _add_to_weak_set(cache, cls); +} + +int +_reset_registry(PyObject *self) +{ + PyObject *registry = _get_registry(self); + if (!registry) { + return 0; + } + return PySet_Clear(registry); +} + +int +_reset_caches(PyObject *self) +{ + PyObject *cache, *key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + cache = PyObject_GetItem(_the_cache, key); + if (!cache) { + return 0; + } + if (!PySet_Clear(cache)) { + return 0; + } + /* also the second cache */ + cache = PyObject_GetItem(_the_negative_cache, key); + if (!cache) { + return 0; + } + return PySet_Clear(cache); +} + +int +_get_negative_cache_version(PyObject *self) +{ + PyObject *key; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + return PyObject_GetItem(_the_cache_version, key); +} + +int +_set_negative_cache_version(PyObject *self, PyObject *version) +{ + PyObject *key; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + return PyObject_SetItem(_the_cache_version, key, version); +} + +PyObject * +_get_dump(PyObject *self) +{ + PyObject *key, *registry, *cache, *negative_cache, *cache_version, *res = PyTuple_New(4); + registry = _get_registry(self); + if (!registry) { + return NULL; + } + registry = PyObject_CallMethod(registry, "copy"); + if (!registry) { + return NULL; + } + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return NULL; + } + cache = PyObject_GetItem(_the_cache, key); + if (!cache) { + return NULL; + } + cache = PyObject_CallMethod(cache, "copy"); + if (!cache) { + return NULL; + } + negative_cache = PyObject_GetItem(_the_negative_cache, key); + if (!negative_cache) { + return NULL; + } + negative_cache = PyObject_CallMethod(negative_cache, "copy"); + if (!negative_cache) { + return NULL; + } + cache_version = _get_negative_cache_version(self); + if (!cache_version) { + return NULL; + } + PyTuple_SetItem(res, 0, registry); + PyTuple_SetItem(res, 1, cache); + PyTuple_SetItem(res, 2, negative_cache); + PyTuple_SetItem(res, 3, cache_version); + return res; +} static PyObject * -abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +_abc_init(PyObject *self) { abc *result = NULL; PyObject *ns, *bases, *items, *abstracts, *base_abstracts; PyObject *key, *value, *item, *iter; Py_ssize_t pos = 0; int ret; - result = (abc *)PyType_Type.tp_new(type, args, kwds); - if (!result) { - return NULL; - } /* Set up inheritance registry. */ - result->abc_registry = PySet_New(NULL); - result->abc_cache = PySet_New(NULL); - result->abc_negative_cache = PySet_New(NULL); - if (!result->abc_registry || !result->abc_cache || !result->abc_negative_cache) { - Py_DECREF(result); - return NULL; - } + _ABC_SETUP_SET(self, _the_registry); + _ABC_SETUP_SET(self, _the_cache); + _ABC_SETUP_SET(self, _the_negative_cache); + result->abc_negative_cache_version = abc_invalidation_counter; if (!(abstracts = PyFrozenSet_New(NULL))) { Py_DECREF(result); @@ -56,34 +268,41 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Compute set of abstract method names in two stages: Stage 1: direct abstract methods. (It is safe to assume everything is fine since type.__new__ succeeded.) */ - ns = PyTuple_GET_ITEM(args, 2); - items = PyMapping_Items(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ - if (!items) { + ns = _PyObject_GetAttrId(self, &PyId___dict__); + keys = PyMapping_Keys(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ + if (!keys) { Py_DECREF(result); return NULL; } - for (pos = 0; pos < PySequence_Size(items); pos++) { - item = PySequence_GetItem(items, pos); - if (!PyTuple_Check(item) || (PyTuple_GET_SIZE(item) != 2)) { - PyErr_SetString(PyExc_TypeError, "Iteration over class namespace must" - " yield length-two tuples"); + for (pos = 0; pos < PySequence_Size(keys); pos++) { + key = PySequence_GetItem(keys, pos); + if () { + Py_DECREF(keys); + goto error; + } + value = PyObject_GetItem(ns, key); + if (!value) { + Py_DECREF(keys); + Py_DECREF(key); + goto error; } - key = PyTuple_GetItem(item, 0); - value = PyTuple_GetItem(item, 1); int is_abstract = _PyObject_IsAbstract(value); if (is_abstract < 0) { - Py_DECREF(item); - Py_DECREF(items); + Py_DECREF(keys); + Py_DECREF(key); + Py_DECREF(value); goto error; } if (is_abstract && PySet_Add(abstracts, key) < 0) { - Py_DECREF(item); - Py_DECREF(items); + Py_DECREF(keys); + Py_DECREF(key); + Py_DECREF(value); goto error; } - Py_DECREF(item); + Py_DECREF(key); + Py_DECREF(value); } - Py_DECREF(items); + Py_DECREF(keys); /* Stage 2: inherited abstract methods. */ bases = PyTuple_GET_ITEM(args, 1); @@ -137,39 +356,8 @@ abcmeta_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } -int -_in_weak_set(PyObject *set, PyObject *obj) -{ - PyObject *ref; - int res; - ref = PyWeakref_NewRef(obj, NULL); - if (!ref) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - return 0; - } - return -1; - } - res = PySet_Contains(set, ref); - Py_DECREF(ref); - return res; -} - -int -_add_weak_set(PyObject *set, PyObject *obj) -{ - PyObject *ref; - ref = PyWeakref_NewRef(obj, NULL); - if (!ref) { - return 0; - } - if (PySet_Add(set, ref) < 0) { - return 0; - } - return 1; -} - static PyObject * -abcmeta_register(abc *self, PyObject *args) +_abc_register(abc *self, PyObject *args) { PyObject *subclass = NULL; int result; @@ -208,10 +396,7 @@ abcmeta_register(abc *self, PyObject *args) } static PyObject * -abcmeta_subclasscheck(abc *self, PyObject *args); /* Forward */ - -static PyObject * -abcmeta_instancecheck(abc *self, PyObject *args) +_abc_instancecheck(abc *self, PyObject *args) { PyObject *result, *subclass, *subtype, *instance = NULL; int incache; @@ -250,80 +435,9 @@ abcmeta_instancecheck(abc *self, PyObject *args) return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subtype); } -int -_add_to_cache(PyObject *self, PyObject *cls) -{ - return 0; -} - -int -_add_to_negative_cache(PyObject *self, PyObject *cls) -{ - return 0; -} - -int -_in_cache(PyObject *self, PyObject *cls) -{ - return 0; -} - -int -_in_negative_cache(PyObject *self, PyObject *cls) -{ - return 0; -} - -int -_get_registry(PyObject *self) -{ - return 0; -} - -int -_add_to_registry(PyObject *self, PyObject *cls) -{ - return 0; -} - -int -_reset_registry(PyObject *self) -{ - return 0; -} - -int -_reset_cache(PyObject *self) -{ - return 0; -} - -int -_reset_negative_cache(PyObject *self) -{ - return 0; -} - -int -_get_negative_cache_version(PyObject *self) -{ - return 0; -} - -int -_set_negative_cache_version(PyObject *self, Py_ssize_t version) -{ - return 0; -} - -PyObject * -_get_dump() -{ - return NULL; -} static PyObject * -abcmeta_subclasscheck(PyObject *self, PyObject *args) +_abc_subclasscheck(PyObject *self, PyObject *args) { PyObject *subclasses, *subclass = NULL; PyObject *ok, *mro, *key, *rkey; @@ -445,7 +559,8 @@ with every call to ``register()`` on any ABC."); static PyObject * get_cache_token(void) { - return PyLong_FromSsize_t(abc_invalidation_counter); + Py_INCREF(abc_invalidation_counter); + return abc_invalidation_counter; } static struct PyMethodDef module_functions[] = { From 9476af63402bb25ec13ee9f98594a5bce7fd4548 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 26 Jan 2018 22:55:40 +0000 Subject: [PATCH 26/93] Implement weakref callbacks and guarded iteration --- Modules/_abc.c | 300 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 266 insertions(+), 34 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 0b9f86c7b99138..f9a26eebdf1708 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,15 +1,12 @@ /* ABCMeta implementation */ -/* TODO: Expose public functions */ -/* TODO: Add callbacks */ /* TODO: Get it compiled */ /* TODO: Get tests passing */ -/* TODO: Add iteration guard */ /* TODO: DECREF all WeakRefs! */ /* TODO: Fix refleaks! */ /* TODO: Think (ask) about thread-safety. */ -/* TODO: Add checks only to those that can fail and use GET_SIZE etc. */ -/* TODO: Think (ask) about inlining some calls (like __subclasses__) and/or macros */ +/* TODO: Add checks only to those calls that can fail and use _GET_SIZE etc. */ +/* TODO: Think about inlining some calls (like __subclasses__) and/or using macros */ /* TODO: Use separate branches with "fast paths" */ #include "Python.h" @@ -17,14 +14,12 @@ PyDoc_STRVAR(_abc__doc__, "Module contains faster C implementation of abc.ABCMeta"); -#define DEFERRED_ADDRESS(ADDR) 0 _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); - /* A global counter that is incremented each time a class is registered as a virtual subclass of anything. It forces the negative cache to be cleared before its next use. @@ -37,6 +32,85 @@ static PyObject *_the_cache = PyDict_New(NULL); static PyObject *_the_negative_cache = PyDict_New(NULL); static PyObject *_the_cache_version = PyDict_New(NULL); +typedef struct { + PyObject_HEAD + unsigned long iterating; + PyObject *data; + PyObject *pending; + PyObject *in_weakreflist; +} _guarded_set; + +static void +gset_dealloc(_guarded_set *self) +{ + if (self->data != NULL) { + PySet_Clear(self->data); + } + if (self->pending != NULL) { + PyList_Type->tp_dealloc(self->pending); + } + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *) self); + } + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +gset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + _guarded_set *self; + + self = (_guarded_set *) type->tp_alloc(type, 0); + if (self != NULL) { + self->iterating = 0; + self->data = PySet_New(NULL); + self->pending = PyList_New(NULL); + self->in_weakreflist = NULL; + } + return (PyObject *) self; +} + +static PyTypeObject _guarded_set_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "_guarded_set", /*tp_name*/ + sizeof(_guarded_set), /*tp_size*/ + 0, /*tp_itemsize*/ + (destructor)gset_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + offsetof(_guarded_set, in_weakreflist), /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + gset_new /* tp_new */ + }; + int _in_weak_set(PyObject *set, PyObject *obj) { @@ -84,11 +158,64 @@ _in_negative_cache(PyObject *self, PyObject *cls) return _in_weak_set(cache, cls); } +static PyObject * +_destroy(PyObject *setweakref, PyObject *objweakref) +{ + PyObject *set; + set = PyWeakref_GET_OBJECT(setweakref); + if (set == Py_None) + Py_RETURN_NONE; + Py_INCREF(set); + if (PySet_Discard(set, objweakref) < 0) { + return NULL; + } + Py_DECREF(set); + Py_RETURN_NONE; +} + +static PyMethodDef _destroy_def = { + "_destroy", (PyCFunction) _destroy, METH_O +}; + +static PyObject * +_destroy_guarded(PyObject *setweakref, PyObject *objweakref) +{ + PyObject *set; + set = PyWeakref_GET_OBJECT(setref); + if (set == Py_None) { + Py_RETURN_NONE; + } + Py_INCREF(set); + if (set->iterating) { + PyList_Append(set->pending, objweakref); + } else { + if (PySet_Discard(set, objweakref) < 0) { + return NULL; + } + } + Py_DECREF(set); + Py_RETURN_NONE; +} + +static PyMethodDef _destroy_guarded_def = { + "_destroy_guarded", (PyCFunction) _destroy_guarded, METH_O +}; + int -_add_to_weak_set(PyObject *set, PyObject *obj) +_add_to_weak_set(PyObject *set, PyObject *obj, int guarded) { - PyObject *ref; - ref = PyWeakref_NewRef(obj, NULL); + PyObject *ref, *wr; + PyObject *destroy_cb; + wr = PyWeakref_NewRef(set, NULL); + if (!wr) { + return 0; + } + if (guarded) { + destroy_cb = PyCFunction_NewEx(&_destroy_guarded_def, wr, NULL); + } else { + destroy_cb = PyCFunction_NewEx(&_destroy_def, wr, NULL); + } + ref = PyWeakref_NewRef(obj, destroy_cb); if (!ref) { return 0; } @@ -106,7 +233,41 @@ _get_registry(PyObject *self) if (!key) { return 0; } - return PyObject_GetItem(_the_registry, key); + return PyObject_GetItem(_the_registry, key)->data; +} + +int +_enter_iter(PyObject *self) +{ + PyObject *key, *registry; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + registry = PyObject_GetItem(_the_registry, key); + registry->iterating++; + return 1; +} + +int +_exit_iter(PyObject *self) +{ + PyObject *key, *registry, *ref; + int pos; + key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + registry->iterating--; + if (registry->iterating) { + return 1; + } + registry = PyObject_GetItem(_the_registry, key); + for (pos = 0; pos < PyList_GET_SIZE(registry->pending); pos++) { + ref = PyObject_CallMethod(registry->pending, "pop", NULL); + PySet_Discard(registry->data, ref); + } + return 1; } int @@ -116,7 +277,7 @@ _add_to_registry(PyObject *self, PyObject *cls) if (!registry) { return 0; } - return _add_to_weak_set(registry, cls); + return _add_to_weak_set(registry, cls, 1); } int @@ -131,7 +292,7 @@ _add_to_cache(PyObject *self, PyObject *cls) if (!cache) { return 0; } - return _add_to_weak_set(cache, cls); + return _add_to_weak_set(cache, cls, 0); } int @@ -146,9 +307,14 @@ _add_to_negative_cache(PyObject *self, PyObject *cls) if (!cache) { return 0; } - return _add_to_weak_set(cache, cls); + return _add_to_weak_set(cache, cls, 0); } +PyDoc_STRVAR(_reset_caches_doc, +"Internal ABC helper to reset registry of a given class.\n\ +\n\ +Should be only used by refleak.py"); + int _reset_registry(PyObject *self) { @@ -156,9 +322,14 @@ _reset_registry(PyObject *self) if (!registry) { return 0; } - return PySet_Clear(registry); + return PySet_Clear(registry->data); } +PyDoc_STRVAR(_reset_caches_doc, +"Internal ABC helper to reset both caches of a given class.\n\ +\n\ +Should be only used by refleak.py"); + int _reset_caches(PyObject *self) { @@ -203,6 +374,13 @@ _set_negative_cache_version(PyObject *self, PyObject *version) return PyObject_SetItem(_the_cache_version, key, version); } +PyDoc_STRVAR(_get_dump_doc, +"Internal ABC helper for cache and registry debugging.\n\ +\n\ +Return shallow copies of registry, of both caches, and\n\ +negative cache version.\n\ Don't call this function directly,\n\ +instead use ABC._dump_registry() for a nice repr."); + PyObject * _get_dump(PyObject *self) { @@ -211,7 +389,7 @@ _get_dump(PyObject *self) if (!registry) { return NULL; } - registry = PyObject_CallMethod(registry, "copy"); + registry = PyObject_CallMethod(registry->data, "copy"); if (!registry) { return NULL; } @@ -246,25 +424,49 @@ _get_dump(PyObject *self) return res; } +PyDoc_STRVAR(_abc_init_doc, +"Internal ABC helper for class set-up. Should be never used outside abc module"); + static PyObject * _abc_init(PyObject *self) { abc *result = NULL; PyObject *ns, *bases, *items, *abstracts, *base_abstracts; - PyObject *key, *value, *item, *iter; + PyObject *key, *value, *item, *iter, *registry, *cache; Py_ssize_t pos = 0; int ret; /* Set up inheritance registry. */ - _ABC_SETUP_SET(self, _the_registry); - _ABC_SETUP_SET(self, _the_cache); - _ABC_SETUP_SET(self, _the_negative_cache); - - result->abc_negative_cache_version = abc_invalidation_counter; + ref = PyWeakref_NewRef(self, NULL); + if (!ref) { + return NULL; + } + registry = gset_new(&_guarded_set_type, NULL, NULL); + if (!registry) { + return NULL; + } + if (PyDict_SetItem(_the_registry, ref, registry) < 0) { + return NULL; + } + cache = PySet_New(NULL); + if (!cache) { + return NULL; + } + if (PyDict_SetItem(_the_cache, ref, cache) < 0) { + return NULL; + } + cache = PySet_New(NULL); + if (!cache) { + return NULL; + } + if (PyDict_SetItem(_the_negative_cache, ref, cache) < 0) { + return NULL; + } + if (PyDict_SetItem(_the_cache_version, ref, abc_invalidation_counter) < 0) { + return NULL; + } if (!(abstracts = PyFrozenSet_New(NULL))) { - Py_DECREF(result); return NULL; } - /* Compute set of abstract method names in two stages: Stage 1: direct abstract methods. (It is safe to assume everything is fine since type.__new__ succeeded.) */ @@ -349,13 +551,15 @@ _abc_init(PyObject *self) if (_PyObject_SetAttrId((PyObject *)result, &PyId___abstractmethods__, abstracts) < 0) { goto error; } - return (PyObject *)result; + Py_RETURN_NONE; error: - Py_DECREF(result); Py_DECREF(abstracts); return NULL; } +PyDoc_STRVAR(_abc_subclasscheck_doc, +"Internal ABC helper for subclasss registration. Should be never used outside abc module"); + static PyObject * _abc_register(abc *self, PyObject *args) { @@ -391,12 +595,16 @@ _abc_register(abc *self, PyObject *args) return NULL; } Py_INCREF(subclass); - abc_invalidation_counter++; /* Invalidate negative cache */ + /* Invalidate negative cache */ + abc_invalidation_counter = PyNumber_Add(abc_invalidation_counter, PyLong_FromLong(1)); return subclass; } +PyDoc_STRVAR(_abc_instancecheck_doc, +"Internal ABC helper for instance checks. Should be never used outside abc module"); + static PyObject * -_abc_instancecheck(abc *self, PyObject *args) +_abc_instancecheck(PyObject *self, PyObject *args) { PyObject *result, *subclass, *subtype, *instance = NULL; int incache; @@ -418,13 +626,14 @@ _abc_instancecheck(abc *self, PyObject *args) if (incache < 0) { return NULL; } - if (_get_negative_cache_version(self) == abc_invalidation_counter && incache) { + if ((PyObject_RichCompareBool(_get_negative_cache_version(self), + abc_invalidation_counter, Py_EQ) > 0) && incache) { Py_RETURN_FALSE; } /* Fall back to the subclass check. */ - return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subclass); + return PyObject_CallMethod((self, "__subclasscheck__", "O", subclass); } - result = PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subclass); + result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); if (!result) { return NULL; } @@ -432,10 +641,13 @@ _abc_instancecheck(abc *self, PyObject *args) return Py_True; } Py_DECREF(result); - return PyObject_CallMethod((PyObject *)self, "__subclasscheck__", "O", subtype); + return PyObject_CallMethod(self, "__subclasscheck__", "O", subtype); } +PyDoc_STRVAR(_abc_subclasscheck_doc, +"Internal ABC helper for subclasss checks. Should be never used outside abc module"); + static PyObject * _abc_subclasscheck(PyObject *self, PyObject *args) { @@ -464,7 +676,8 @@ _abc_subclasscheck(PyObject *self, PyObject *args) if (incache < 0) { return NULL; } - if (_get_negative_cache_version(self) < abc_invalidation_counter) { + if (PyObject_RichCompareBool(_get_negative_cache_version(self), + abc_invalidation_counter, Py_LT) > 0) { /* Invalidate the negative cache. */ if (!_reset_negative_cache(self)) { return NULL; @@ -507,6 +720,7 @@ _abc_subclasscheck(PyObject *self, PyObject *args) /* 5. Check if it's a subclass of a registered class (recursive). */ pos = 0; Py_hash_t hash; + _enter_iter(self); while (_PySet_NextEntry(_get_registry(self), &pos, &key, &hash)) { rkey = PyWeakref_GetObject(key); if (rkey == Py_None) { @@ -514,18 +728,22 @@ _abc_subclasscheck(PyObject *self, PyObject *args) } result = PyObject_IsSubclass(subclass, rkey); if (result < 0) { + _exit_iter(self); return NULL; } if (result > 0) { if (!_add_to_cache(self, subclass)) { + _exit_iter(self); return NULL; } + _exit_iter(self); Py_RETURN_TRUE; } } + _exit_iter(self); /* 6. Check if it's a subclass of a subclass (recursive). */ - subclasses = PyObject_CallMethod((PyObject *)self, "__subclasses__", NULL); + subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (result > 0) { @@ -566,6 +784,20 @@ get_cache_token(void) static struct PyMethodDef module_functions[] = { {"get_cache_token", (PyCFunction)get_cache_token, METH_NOARGS, _cache_token_doc}, + {"_abc_init", (PyCFunction)_abc_init, METH_O, + _abc_init_doc}, + {"_reset_registry", (PyCFunction)_reset_registry, METH_O, + _abc_init_doc}, + {"_reset_caches", (PyCFunction)_reset_caches, METH_O, + _reset_caches_doc}, + {"_get_dump", (PyCFunction)_get_dump, METH_O, + _get_dump_doc}, + {"_abc_register", (PyCFunction)_abc_register, METH_VARARGS, + _abc_resiter_doc}, + {"_abc_instancecheck", (PyCFunction)_abc_instancecheck, METH_VARARGS, + _abc_instancecheck_doc}, + {"_abc_subclasscheck", (PyCFunction)_abc_subclasscheck, METH_VARARGS, + _abc_subclasscheck_doc}, {NULL, NULL} /* sentinel */ }; From ab68cdb0dead1d6de3074bc5928a7fb10e2ded97 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 26 Jan 2018 23:51:16 +0000 Subject: [PATCH 27/93] Fix some errors --- Lib/abc.py | 2 +- Modules/_abc.c | 124 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 82 insertions(+), 44 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index b83cb70abe2cb9..565ed707c076da 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -190,7 +190,7 @@ def register(cls, subclass): def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" - print(f"Class: {cls.__module__}.{cls.__qualname__}"), file=file) + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) print(f"Inv. counter: {get_cache_token()}", file=file) if _C_speedup: (_abc_registry, _abc_cache, _abc_negative_cache, diff --git a/Modules/_abc.c b/Modules/_abc.c index f9a26eebdf1708..9850026ed0c8ad 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -19,18 +19,19 @@ _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); +_Py_IDENTIFIER(__bases__); /* A global counter that is incremented each time a class is registered as a virtual subclass of anything. It forces the negative cache to be cleared before its next use. Note: this counter is private. Use `abc.get_cache_token()` for external code. */ -static PyObject *abc_invalidation_counter = PyLong_FromLong(0); +static PyObject *abc_invalidation_counter; -static PyObject *_the_registry = PyDict_New(NULL); -static PyObject *_the_cache = PyDict_New(NULL); -static PyObject *_the_negative_cache = PyDict_New(NULL); -static PyObject *_the_cache_version = PyDict_New(NULL); +static PyObject *_the_registry; +static PyObject *_the_cache; +static PyObject *_the_negative_cache; +static PyObject *_the_cache_version; typedef struct { PyObject_HEAD @@ -47,7 +48,7 @@ gset_dealloc(_guarded_set *self) PySet_Clear(self->data); } if (self->pending != NULL) { - PyList_Type->tp_dealloc(self->pending); + PyList_Type.tp_dealloc(self->pending); } if (self->in_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) self); @@ -64,7 +65,7 @@ gset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (self != NULL) { self->iterating = 0; self->data = PySet_New(NULL); - self->pending = PyList_New(NULL); + self->pending = PyList_New(0); self->in_weakreflist = NULL; } return (PyObject *) self; @@ -181,15 +182,17 @@ static PyObject * _destroy_guarded(PyObject *setweakref, PyObject *objweakref) { PyObject *set; - set = PyWeakref_GET_OBJECT(setref); + _guarded_set *gset; + set = PyWeakref_GET_OBJECT(setweakref); if (set == Py_None) { Py_RETURN_NONE; } Py_INCREF(set); - if (set->iterating) { - PyList_Append(set->pending, objweakref); + gset = (_guarded_set *)set; + if (gset->iterating) { + PyList_Append(gset->pending, objweakref); } else { - if (PySet_Discard(set, objweakref) < 0) { + if (PySet_Discard(gset->data, objweakref) < 0) { return NULL; } } @@ -219,6 +222,9 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) if (!ref) { return 0; } + if (guarded) { + set = ((_guarded_set *) set)->data; + } if (PySet_Add(set, ref) < 0) { return 0; } @@ -229,22 +235,31 @@ PyObject * _get_registry(PyObject *self) { PyObject *key; + _guarded_set *registry; key = PyWeakref_NewRef(self, NULL); if (!key) { return 0; } - return PyObject_GetItem(_the_registry, key)->data; + registry = (_guarded_set *)PyObject_GetItem(_the_registry, key); + if (!registry) { + return NULL; + } + return registry->data; } int _enter_iter(PyObject *self) { - PyObject *key, *registry; + PyObject *key; + _guarded_set *registry; key = PyWeakref_NewRef(self, NULL); if (!key) { return 0; } - registry = PyObject_GetItem(_the_registry, key); + registry = (_guarded_set *)PyObject_GetItem(_the_registry, key); + if (!registry) { + return 0; + } registry->iterating++; return 1; } @@ -252,17 +267,21 @@ _enter_iter(PyObject *self) int _exit_iter(PyObject *self) { - PyObject *key, *registry, *ref; + PyObject *key, *ref; + _guarded_set *registry; int pos; key = PyWeakref_NewRef(self, NULL); if (!key) { return 0; } + registry = (_guarded_set *) PyObject_GetItem(_the_registry, key); + if (!registry) { + return 0; + } registry->iterating--; if (registry->iterating) { return 1; } - registry = PyObject_GetItem(_the_registry, key); for (pos = 0; pos < PyList_GET_SIZE(registry->pending); pos++) { ref = PyObject_CallMethod(registry->pending, "pop", NULL); PySet_Discard(registry->data, ref); @@ -273,10 +292,12 @@ _exit_iter(PyObject *self) int _add_to_registry(PyObject *self, PyObject *cls) { - PyObject *registry = _get_registry(self); - if (!registry) { + PyObject *key, *registry; + key = PyWeakref_NewRef(self, NULL); + if (!key) { return 0; } + registry = PyObject_GetItem(_the_registry, key); return _add_to_weak_set(registry, cls, 1); } @@ -310,7 +331,7 @@ _add_to_negative_cache(PyObject *self, PyObject *cls) return _add_to_weak_set(cache, cls, 0); } -PyDoc_STRVAR(_reset_caches_doc, +PyDoc_STRVAR(_reset_registry_doc, "Internal ABC helper to reset registry of a given class.\n\ \n\ Should be only used by refleak.py"); @@ -322,7 +343,25 @@ _reset_registry(PyObject *self) if (!registry) { return 0; } - return PySet_Clear(registry->data); + return PySet_Clear(registry); +} + + +int +_reset_negative_cache(PyObject *self) +{ + PyObject *cache, *key = PyWeakref_NewRef(self, NULL); + if (!key) { + return 0; + } + cache = PyObject_GetItem(_the_negative_cache, key); + if (!cache) { + return 0; + } + if (!PySet_Clear(cache)) { + return 0; + } + return 1; } PyDoc_STRVAR(_reset_caches_doc, @@ -345,20 +384,16 @@ _reset_caches(PyObject *self) return 0; } /* also the second cache */ - cache = PyObject_GetItem(_the_negative_cache, key); - if (!cache) { - return 0; - } - return PySet_Clear(cache); + return _reset_negative_cache(self); } -int +PyObject * _get_negative_cache_version(PyObject *self) { PyObject *key; key = PyWeakref_NewRef(self, NULL); if (!key) { - return 0; + return NULL; } return PyObject_GetItem(_the_cache_version, key); } @@ -378,7 +413,7 @@ PyDoc_STRVAR(_get_dump_doc, "Internal ABC helper for cache and registry debugging.\n\ \n\ Return shallow copies of registry, of both caches, and\n\ -negative cache version.\n\ Don't call this function directly,\n\ +negative cache version. Don't call this function directly,\n\ instead use ABC._dump_registry() for a nice repr."); PyObject * @@ -389,7 +424,7 @@ _get_dump(PyObject *self) if (!registry) { return NULL; } - registry = PyObject_CallMethod(registry->data, "copy"); + registry = PyObject_CallMethod(registry, "copy", NULL); if (!registry) { return NULL; } @@ -401,7 +436,7 @@ _get_dump(PyObject *self) if (!cache) { return NULL; } - cache = PyObject_CallMethod(cache, "copy"); + cache = PyObject_CallMethod(cache, "copy", NULL); if (!cache) { return NULL; } @@ -409,7 +444,7 @@ _get_dump(PyObject *self) if (!negative_cache) { return NULL; } - negative_cache = PyObject_CallMethod(negative_cache, "copy"); + negative_cache = PyObject_CallMethod(negative_cache, "copy", NULL); if (!negative_cache) { return NULL; } @@ -430,9 +465,8 @@ PyDoc_STRVAR(_abc_init_doc, static PyObject * _abc_init(PyObject *self) { - abc *result = NULL; - PyObject *ns, *bases, *items, *abstracts, *base_abstracts; - PyObject *key, *value, *item, *iter, *registry, *cache; + PyObject *ns, *bases, *keys, *abstracts, *base_abstracts; + PyObject *key, *value, *item, *iter, *registry, *cache, *ref; Py_ssize_t pos = 0; int ret; /* Set up inheritance registry. */ @@ -473,12 +507,11 @@ _abc_init(PyObject *self) ns = _PyObject_GetAttrId(self, &PyId___dict__); keys = PyMapping_Keys(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ if (!keys) { - Py_DECREF(result); return NULL; } for (pos = 0; pos < PySequence_Size(keys); pos++) { key = PySequence_GetItem(keys, pos); - if () { + if (!key) { Py_DECREF(keys); goto error; } @@ -507,7 +540,7 @@ _abc_init(PyObject *self) Py_DECREF(keys); /* Stage 2: inherited abstract methods. */ - bases = PyTuple_GET_ITEM(args, 1); + bases = _PyObject_GetAttrId(self, &PyId___bases__); for (pos = 0; pos < PyTuple_Size(bases); pos++) { item = PyTuple_GET_ITEM(bases, pos); ret = _PyObject_LookupAttrId(item, &PyId___abstractmethods__, &base_abstracts); @@ -520,7 +553,7 @@ _abc_init(PyObject *self) goto error; } while ((key = PyIter_Next(iter))) { - ret = _PyObject_LookupAttr((PyObject *)result, key, &value); + ret = _PyObject_LookupAttr(self, key, &value); if (ret < 0) { Py_DECREF(key); Py_DECREF(iter); @@ -548,7 +581,7 @@ _abc_init(PyObject *self) goto error; } } - if (_PyObject_SetAttrId((PyObject *)result, &PyId___abstractmethods__, abstracts) < 0) { + if (_PyObject_SetAttrId(self, &PyId___abstractmethods__, abstracts) < 0) { goto error; } Py_RETURN_NONE; @@ -557,11 +590,11 @@ _abc_init(PyObject *self) return NULL; } -PyDoc_STRVAR(_abc_subclasscheck_doc, +PyDoc_STRVAR(_abc_register_doc, "Internal ABC helper for subclasss registration. Should be never used outside abc module"); static PyObject * -_abc_register(abc *self, PyObject *args) +_abc_register(PyObject *self, PyObject *args) { PyObject *subclass = NULL; int result; @@ -631,7 +664,7 @@ _abc_instancecheck(PyObject *self, PyObject *args) Py_RETURN_FALSE; } /* Fall back to the subclass check. */ - return PyObject_CallMethod((self, "__subclasscheck__", "O", subclass); + return PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); } result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); if (!result) { @@ -793,7 +826,7 @@ static struct PyMethodDef module_functions[] = { {"_get_dump", (PyCFunction)_get_dump, METH_O, _get_dump_doc}, {"_abc_register", (PyCFunction)_abc_register, METH_VARARGS, - _abc_resiter_doc}, + _abc_register_doc}, {"_abc_instancecheck", (PyCFunction)_abc_instancecheck, METH_VARARGS, _abc_instancecheck_doc}, {"_abc_subclasscheck", (PyCFunction)_abc_subclasscheck, METH_VARARGS, @@ -817,5 +850,10 @@ static struct PyModuleDef _abcmodule = { PyMODINIT_FUNC PyInit__abc(void) { + abc_invalidation_counter = PyLong_FromLong(0); + _the_registry = PyDict_New(); + _the_cache = PyDict_New(); + _the_negative_cache = PyDict_New(); + _the_cache_version = PyDict_New(); return PyModule_Create(&_abcmodule); } From 3eb0a6003462e0eee0b8355e2f52aed9a185ae34 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 26 Jan 2018 23:59:10 +0000 Subject: [PATCH 28/93] Fix two review comments --- Modules/_abc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/_abc.c b/Modules/_abc.c index 9850026ed0c8ad..68edea80fa57e1 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -738,6 +738,12 @@ _abc_subclasscheck(PyObject *self, PyObject *args) } return Py_False; } + if (ok != Py_NotImplemented) { + Py_DECREF(ok); + PyErr_SetString(PyExc_AssertionError, "__subclasshook__ must return either" + " False, True, or NotImplemented"); + return NULL; + } Py_DECREF(ok); /* 4. Check if it's a direct subclass. */ mro = ((PyTypeObject *)subclass)->tp_mro; @@ -777,6 +783,10 @@ _abc_subclasscheck(PyObject *self, PyObject *args) /* 6. Check if it's a subclass of a subclass (recursive). */ subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); + if(!PyList_Check(subclasses)) { + PyErr_SetString(PyExc_TypeError, "__subclasses__() must return a list"); + return NULL; + } for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (result > 0) { From ed36b76e91382cd03bad28b0a8fb56abc8f695f2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 00:48:34 +0000 Subject: [PATCH 29/93] More fixes, test_abc passes --- Modules/_abc.c | 82 +++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 68edea80fa57e1..6d25b5d6342fc6 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -15,7 +15,6 @@ PyDoc_STRVAR(_abc__doc__, "Module contains faster C implementation of abc.ABCMeta"); -_Py_IDENTIFIER(stdout); _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); @@ -135,11 +134,11 @@ _in_cache(PyObject *self, PyObject *cls) PyObject *key, *cache; key = PyWeakref_NewRef(self, NULL); if (!key) { - return 0; + return -1; } cache = PyObject_GetItem(_the_cache, key); if (!cache) { - return 0; + return -1; } return _in_weak_set(cache, cls); } @@ -150,11 +149,11 @@ _in_negative_cache(PyObject *self, PyObject *cls) PyObject *key, *cache; key = PyWeakref_NewRef(self, NULL); if (!key) { - return 0; + return -1; } cache = PyObject_GetItem(_the_negative_cache, key); if (!cache) { - return 0; + return -1; } return _in_weak_set(cache, cls); } @@ -337,13 +336,20 @@ PyDoc_STRVAR(_reset_registry_doc, Should be only used by refleak.py"); int -_reset_registry(PyObject *self) +_reset_registry(PyObject *args) { - PyObject *registry = _get_registry(self); + PyObject *self, *registry; + if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { + return 0; + } + registry = _get_registry(self); if (!registry) { return 0; } - return PySet_Clear(registry); + if (PySet_Clear(registry) < 0) { + return 0; + } + return 1; } @@ -358,7 +364,7 @@ _reset_negative_cache(PyObject *self) if (!cache) { return 0; } - if (!PySet_Clear(cache)) { + if (PySet_Clear(cache) < 0) { return 0; } return 1; @@ -370,9 +376,13 @@ PyDoc_STRVAR(_reset_caches_doc, Should be only used by refleak.py"); int -_reset_caches(PyObject *self) +_reset_caches(PyObject *args) { - PyObject *cache, *key = PyWeakref_NewRef(self, NULL); + PyObject *self, *cache, *key; + if (!PyArg_UnpackTuple(args, "_reset_caches", 1, 1, &self)) { + return 0; + } + key = PyWeakref_NewRef(self, NULL); if (!key) { return 0; } @@ -417,9 +427,13 @@ negative cache version. Don't call this function directly,\n\ instead use ABC._dump_registry() for a nice repr."); PyObject * -_get_dump(PyObject *self) +_get_dump(PyObject *m, PyObject *args) { - PyObject *key, *registry, *cache, *negative_cache, *cache_version, *res = PyTuple_New(4); + PyObject *self, *key, *registry, *cache, *negative_cache, *cache_version; + PyObject *res = PyTuple_New(4); + if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { + return NULL; + } registry = _get_registry(self); if (!registry) { return NULL; @@ -463,12 +477,15 @@ PyDoc_STRVAR(_abc_init_doc, "Internal ABC helper for class set-up. Should be never used outside abc module"); static PyObject * -_abc_init(PyObject *self) +_abc_init(PyObject *m, PyObject *args) { - PyObject *ns, *bases, *keys, *abstracts, *base_abstracts; + PyObject *ns, *bases, *keys, *abstracts, *base_abstracts, *self; PyObject *key, *value, *item, *iter, *registry, *cache, *ref; Py_ssize_t pos = 0; int ret; + if (!PyArg_UnpackTuple(args, "_abc_init", 1, 1, &self)) { + return NULL; + } /* Set up inheritance registry. */ ref = PyWeakref_NewRef(self, NULL); if (!ref) { @@ -541,6 +558,9 @@ _abc_init(PyObject *self) /* Stage 2: inherited abstract methods. */ bases = _PyObject_GetAttrId(self, &PyId___bases__); + if (!bases) { + return NULL; + } for (pos = 0; pos < PyTuple_Size(bases); pos++) { item = PyTuple_GET_ITEM(bases, pos); ret = _PyObject_LookupAttrId(item, &PyId___abstractmethods__, &base_abstracts); @@ -594,18 +614,18 @@ PyDoc_STRVAR(_abc_register_doc, "Internal ABC helper for subclasss registration. Should be never used outside abc module"); static PyObject * -_abc_register(PyObject *self, PyObject *args) +_abc_register(PyObject *m, PyObject *args) { - PyObject *subclass = NULL; + PyObject *self, *subclass = NULL; int result; - if (!PyArg_UnpackTuple(args, "register", 1, 1, &subclass)) { + if (!PyArg_UnpackTuple(args, "_abc_register", 2, 2, &self, &subclass)) { return NULL; } if (!PyType_Check(subclass)) { PyErr_SetString(PyExc_TypeError, "Can only register classes"); return NULL; } - result = PyObject_IsSubclass(subclass, (PyObject *)self); + result = PyObject_IsSubclass(subclass, self); if (result > 0) { Py_INCREF(subclass); return subclass; /* Already a subclass. */ @@ -615,7 +635,7 @@ _abc_register(PyObject *self, PyObject *args) } /* Subtle: test for cycles *after* testing for "already a subclass"; this means we allow X.register(X) and interpret it as a no-op. */ - result = PyObject_IsSubclass((PyObject *)self, subclass); + result = PyObject_IsSubclass(self, subclass); if (result > 0) { /* This would create a cycle, which is bad for the algorithm below. */ PyErr_SetString(PyExc_RuntimeError, "Refusing to create an inheritance cycle"); @@ -637,11 +657,11 @@ PyDoc_STRVAR(_abc_instancecheck_doc, "Internal ABC helper for instance checks. Should be never used outside abc module"); static PyObject * -_abc_instancecheck(PyObject *self, PyObject *args) +_abc_instancecheck(PyObject *m, PyObject *args) { - PyObject *result, *subclass, *subtype, *instance = NULL; + PyObject *self, *result, *subclass, *subtype, *instance = NULL; int incache; - if (!PyArg_UnpackTuple(args, "__instancecheck__", 1, 1, &instance)) { + if (!PyArg_UnpackTuple(args, "_abc_instancecheck", 2, 2, &self, &instance)) { return NULL; } subclass = _PyObject_GetAttrId(instance, &PyId___class__); @@ -682,13 +702,13 @@ PyDoc_STRVAR(_abc_subclasscheck_doc, "Internal ABC helper for subclasss checks. Should be never used outside abc module"); static PyObject * -_abc_subclasscheck(PyObject *self, PyObject *args) +_abc_subclasscheck(PyObject *m, PyObject *args) { - PyObject *subclasses, *subclass = NULL; + PyObject *self, *subclasses, *subclass = NULL; PyObject *ok, *mro, *key, *rkey; Py_ssize_t pos; int incache, result; - if (!PyArg_UnpackTuple(args, "__subclasscheck__", 1, 1, &subclass)) { + if (!PyArg_UnpackTuple(args, "_abc_subclasscheck", 2, 2, &self, &subclass)) { return NULL; } /* TODO: clear the registry from dead refs from time to time @@ -827,13 +847,13 @@ get_cache_token(void) static struct PyMethodDef module_functions[] = { {"get_cache_token", (PyCFunction)get_cache_token, METH_NOARGS, _cache_token_doc}, - {"_abc_init", (PyCFunction)_abc_init, METH_O, - _abc_init_doc}, - {"_reset_registry", (PyCFunction)_reset_registry, METH_O, + {"_abc_init", (PyCFunction)_abc_init, METH_VARARGS, _abc_init_doc}, - {"_reset_caches", (PyCFunction)_reset_caches, METH_O, + {"_reset_registry", (PyCFunction)_reset_registry, METH_VARARGS, + _reset_registry_doc}, + {"_reset_caches", (PyCFunction)_reset_caches, METH_VARARGS, _reset_caches_doc}, - {"_get_dump", (PyCFunction)_get_dump, METH_O, + {"_get_dump", (PyCFunction)_get_dump, METH_VARARGS, _get_dump_doc}, {"_abc_register", (PyCFunction)_abc_register, METH_VARARGS, _abc_register_doc}, From b2f75b96a15955e5df17aba0bbf2a5daa5c08a87 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 02:13:29 +0000 Subject: [PATCH 30/93] Fix some remaining problems --- Lib/test/libregrtest/refleak.py | 9 +-------- Lib/test/test_typing.py | 5 +++-- Modules/_abc.c | 34 +++++++++++++++++---------------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 43a7103a5923e5..4497aac0d7bfe2 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -5,6 +5,7 @@ import warnings from inspect import isabstract from test import support +from _abc import _reset_caches, _get_dump, _reset_registry def dash_R(the_module, test, indirect_test, huntrleaks): @@ -35,8 +36,6 @@ def dash_R(the_module, test, indirect_test, huntrleaks): for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = obj._abc_registry.copy() # bpo-31217: Integer pool to get a single integer object for the same # value. The pool is used to prevent false alarm when checking for memory @@ -134,12 +133,6 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) - for abc in abs_classes: - for obj in abc.__subclasses__() + [abc]: - for el in abcs.get(obj, set()): - obj._abc_registry.add(el) - obj._abc_cache.clear() - obj._abc_negative_cache.clear() clear_caches() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3f24faf376292f..c7973f9f19801e 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -22,6 +22,7 @@ from typing import IO, TextIO, BinaryIO from typing import Pattern, Match import abc +from _abc import _reset_registry, _reset_caches import typing import weakref @@ -761,8 +762,8 @@ class C(collections.abc.Mapping, Generic[T]): ... self.assertIsInstance(1, C) C[int] self.assertIsInstance(1, C) - C._abc_registry.clear() - C._abc_cache.clear() # To keep refleak hunting mode clean + _reset_registry(C) + _reset_caches(C) # To keep refleak hunting mode clean def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass diff --git a/Modules/_abc.c b/Modules/_abc.c index 6d25b5d6342fc6..0c74d5ad901502 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -335,24 +335,23 @@ PyDoc_STRVAR(_reset_registry_doc, \n\ Should be only used by refleak.py"); -int -_reset_registry(PyObject *args) +PyObject * +_reset_registry(PyObject *m, PyObject *args) { PyObject *self, *registry; - if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { - return 0; + if (!PyArg_UnpackTuple(args, "_reset_registry", 1, 1, &self)) { + return NULL; } registry = _get_registry(self); if (!registry) { - return 0; + return NULL; } if (PySet_Clear(registry) < 0) { - return 0; + return NULL; } - return 1; + Py_RETURN_NONE; } - int _reset_negative_cache(PyObject *self) { @@ -375,26 +374,29 @@ PyDoc_STRVAR(_reset_caches_doc, \n\ Should be only used by refleak.py"); -int -_reset_caches(PyObject *args) +PyObject * +_reset_caches(PyObject *m, PyObject *args) { PyObject *self, *cache, *key; if (!PyArg_UnpackTuple(args, "_reset_caches", 1, 1, &self)) { - return 0; + return NULL; } key = PyWeakref_NewRef(self, NULL); if (!key) { - return 0; + return NULL; } cache = PyObject_GetItem(_the_cache, key); if (!cache) { - return 0; + return NULL; } - if (!PySet_Clear(cache)) { - return 0; + if (PySet_Clear(cache) < 0) { + return NULL; } /* also the second cache */ - return _reset_negative_cache(self); + if (!_reset_negative_cache(self)) { + return NULL; + } + Py_RETURN_NONE; } PyObject * From cdb5cdf7b35511495e669d9b309a4b3baa9f5d2d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 02:15:09 +0000 Subject: [PATCH 31/93] Update TODO --- Modules/_abc.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 0c74d5ad901502..9370799fe7a124 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,9 +1,6 @@ /* ABCMeta implementation */ -/* TODO: Get it compiled */ -/* TODO: Get tests passing */ -/* TODO: DECREF all WeakRefs! */ -/* TODO: Fix refleaks! */ +/* TODO: Fix refleaks: DECREF all WeakRefs! */ /* TODO: Think (ask) about thread-safety. */ /* TODO: Add checks only to those calls that can fail and use _GET_SIZE etc. */ /* TODO: Think about inlining some calls (like __subclasses__) and/or using macros */ From a1a3a52eb3146419b8b102c8c5824d2d5a4a0b0a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 02:39:34 +0000 Subject: [PATCH 32/93] Add missing statics --- Modules/_abc.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 9370799fe7a124..406ed2f6ca853f 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -108,7 +108,7 @@ static PyTypeObject _guarded_set_type = { gset_new /* tp_new */ }; -int +static int _in_weak_set(PyObject *set, PyObject *obj) { PyObject *ref; @@ -125,7 +125,7 @@ _in_weak_set(PyObject *set, PyObject *obj) return res; } -int +static int _in_cache(PyObject *self, PyObject *cls) { PyObject *key, *cache; @@ -140,7 +140,7 @@ _in_cache(PyObject *self, PyObject *cls) return _in_weak_set(cache, cls); } -int +static int _in_negative_cache(PyObject *self, PyObject *cls) { PyObject *key, *cache; @@ -200,7 +200,7 @@ static PyMethodDef _destroy_guarded_def = { "_destroy_guarded", (PyCFunction) _destroy_guarded, METH_O }; -int +static int _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) { PyObject *ref, *wr; @@ -227,7 +227,7 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) return 1; } -PyObject * +static PyObject * _get_registry(PyObject *self) { PyObject *key; @@ -243,7 +243,7 @@ _get_registry(PyObject *self) return registry->data; } -int +static int _enter_iter(PyObject *self) { PyObject *key; @@ -260,7 +260,7 @@ _enter_iter(PyObject *self) return 1; } -int +static int _exit_iter(PyObject *self) { PyObject *key, *ref; @@ -285,7 +285,7 @@ _exit_iter(PyObject *self) return 1; } -int +static int _add_to_registry(PyObject *self, PyObject *cls) { PyObject *key, *registry; @@ -297,7 +297,7 @@ _add_to_registry(PyObject *self, PyObject *cls) return _add_to_weak_set(registry, cls, 1); } -int +static int _add_to_cache(PyObject *self, PyObject *cls) { PyObject *key, *cache; @@ -312,7 +312,7 @@ _add_to_cache(PyObject *self, PyObject *cls) return _add_to_weak_set(cache, cls, 0); } -int +static int _add_to_negative_cache(PyObject *self, PyObject *cls) { PyObject *key, *cache; @@ -332,7 +332,7 @@ PyDoc_STRVAR(_reset_registry_doc, \n\ Should be only used by refleak.py"); -PyObject * +static PyObject * _reset_registry(PyObject *m, PyObject *args) { PyObject *self, *registry; @@ -349,7 +349,7 @@ _reset_registry(PyObject *m, PyObject *args) Py_RETURN_NONE; } -int +static int _reset_negative_cache(PyObject *self) { PyObject *cache, *key = PyWeakref_NewRef(self, NULL); @@ -371,7 +371,7 @@ PyDoc_STRVAR(_reset_caches_doc, \n\ Should be only used by refleak.py"); -PyObject * +static PyObject * _reset_caches(PyObject *m, PyObject *args) { PyObject *self, *cache, *key; @@ -396,7 +396,7 @@ _reset_caches(PyObject *m, PyObject *args) Py_RETURN_NONE; } -PyObject * +static PyObject * _get_negative_cache_version(PyObject *self) { PyObject *key; @@ -407,7 +407,7 @@ _get_negative_cache_version(PyObject *self) return PyObject_GetItem(_the_cache_version, key); } -int +static int _set_negative_cache_version(PyObject *self, PyObject *version) { PyObject *key; @@ -425,7 +425,7 @@ Return shallow copies of registry, of both caches, and\n\ negative cache version. Don't call this function directly,\n\ instead use ABC._dump_registry() for a nice repr."); -PyObject * +static PyObject * _get_dump(PyObject *m, PyObject *args) { PyObject *self, *key, *registry, *cache, *negative_cache, *cache_version; From a66b08c066ba3c1e92c646b7356ee4477b49ce45 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 27 Jan 2018 14:47:16 +0900 Subject: [PATCH 33/93] Build on Windows --- PC/config.c | 2 ++ PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 3 files changed, 6 insertions(+) diff --git a/PC/config.c b/PC/config.c index 91f15b5867d5da..0b6b43e16cc6be 100644 --- a/PC/config.c +++ b/PC/config.c @@ -5,6 +5,7 @@ #include "Python.h" +extern PyObject* PyInit__abc(void); extern PyObject* PyInit_array(void); extern PyObject* PyInit_audioop(void); extern PyObject* PyInit_binascii(void); @@ -78,6 +79,7 @@ extern PyObject* PyInit_imp(void); struct _inittab _PyImport_Inittab[] = { + {"_abc", PyInit__abc}, {"array", PyInit_array}, {"_ast", PyInit__ast}, {"audioop", PyInit_audioop}, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f30945508698af..d19b5f5acf8923 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -228,6 +228,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 5f980388e5f321..b51fd54f8b4db6 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -473,6 +473,9 @@ + + Modules + Modules From 86af9ae424ce74ee12ac24ab7f50cfa45a4ae0d1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 28 Jan 2018 03:42:13 +0900 Subject: [PATCH 34/93] Refactor __abstractmethods__ calculation. --- Modules/_abc.c | 200 ++++++++++++++++++++++++++++--------------------- 1 file changed, 116 insertions(+), 84 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 406ed2f6ca853f..74b8d3490dfebd 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -472,123 +472,97 @@ _get_dump(PyObject *m, PyObject *args) return res; } -PyDoc_STRVAR(_abc_init_doc, -"Internal ABC helper for class set-up. Should be never used outside abc module"); - -static PyObject * -_abc_init(PyObject *m, PyObject *args) +// Compute set of abstract method names. +static int +compute_abstract_methods(PyObject *self) { - PyObject *ns, *bases, *keys, *abstracts, *base_abstracts, *self; - PyObject *key, *value, *item, *iter, *registry, *cache, *ref; - Py_ssize_t pos = 0; - int ret; - if (!PyArg_UnpackTuple(args, "_abc_init", 1, 1, &self)) { - return NULL; - } - /* Set up inheritance registry. */ - ref = PyWeakref_NewRef(self, NULL); - if (!ref) { - return NULL; - } - registry = gset_new(&_guarded_set_type, NULL, NULL); - if (!registry) { - return NULL; - } - if (PyDict_SetItem(_the_registry, ref, registry) < 0) { - return NULL; - } - cache = PySet_New(NULL); - if (!cache) { - return NULL; - } - if (PyDict_SetItem(_the_cache, ref, cache) < 0) { - return NULL; - } - cache = PySet_New(NULL); - if (!cache) { - return NULL; - } - if (PyDict_SetItem(_the_negative_cache, ref, cache) < 0) { - return NULL; - } - if (PyDict_SetItem(_the_cache_version, ref, abc_invalidation_counter) < 0) { - return NULL; - } - if (!(abstracts = PyFrozenSet_New(NULL))) { - return NULL; + int ret = -1; + PyObject *abstracts = PyFrozenSet_New(NULL); + if (!abstracts) { + return -1; } - /* Compute set of abstract method names in two stages: - Stage 1: direct abstract methods. + + PyObject *ns=NULL, *items=NULL, *bases=NULL; // Py_CLEAR()ed on error. + + /* Stage 1: direct abstract methods. (It is safe to assume everything is fine since type.__new__ succeeded.) */ ns = _PyObject_GetAttrId(self, &PyId___dict__); - keys = PyMapping_Keys(ns); /* TODO: Fast path for exact dicts with PyDict_Next */ - if (!keys) { - return NULL; + if (!ns) { + goto error; } - for (pos = 0; pos < PySequence_Size(keys); pos++) { - key = PySequence_GetItem(keys, pos); - if (!key) { - Py_DECREF(keys); + + /* TODO: Fast path for exact dicts with PyDict_Next */ + items = PyMapping_Items(ns); + if (!items) { + goto error; + } + + for (Py_ssize_t pos = 0; pos < PyList_GET_SIZE(items); pos++) { + PyObject *it = PySequence_Fast( + PyList_GET_ITEM(items, pos), + "items() returned non-sequence item"); + if (!it) { goto error; } - value = PyObject_GetItem(ns, key); - if (!value) { - Py_DECREF(keys); - Py_DECREF(key); + if (PySequence_Fast_GET_SIZE(it) != 2) { + PyErr_SetString(PyExc_TypeError, "items() returned not 2-tuple"); + Py_DECREF(it); goto error; } + + // borrowed + PyObject *key = PySequence_Fast_GET_ITEM(it, 0); + PyObject *value = PySequence_Fast_GET_ITEM(it, 1); int is_abstract = _PyObject_IsAbstract(value); - if (is_abstract < 0) { - Py_DECREF(keys); - Py_DECREF(key); - Py_DECREF(value); + if (is_abstract < 0 || + (is_abstract && PySet_Add(abstracts, key) < 0)) { + Py_DECREF(it); goto error; } - if (is_abstract && PySet_Add(abstracts, key) < 0) { - Py_DECREF(keys); - Py_DECREF(key); - Py_DECREF(value); - goto error; - } - Py_DECREF(key); - Py_DECREF(value); } - Py_DECREF(keys); /* Stage 2: inherited abstract methods. */ bases = _PyObject_GetAttrId(self, &PyId___bases__); if (!bases) { - return NULL; + goto error; + } + if (!PyTuple_Check(bases)) { + PyErr_SetString(PyExc_TypeError, "__bases__ is not tuple"); + goto error; } - for (pos = 0; pos < PyTuple_Size(bases); pos++) { - item = PyTuple_GET_ITEM(bases, pos); - ret = _PyObject_LookupAttrId(item, &PyId___abstractmethods__, &base_abstracts); - if (ret < 0) { + + for (Py_ssize_t pos = 0; pos < PyTuple_GET_SIZE(bases); pos++) { + PyObject *item = PyTuple_GET_ITEM(bases, pos); // borrowed + PyObject *base_abstracts, *iter; + + if (_PyObject_LookupAttrId(item, &PyId___abstractmethods__, + &base_abstracts) < 0) { goto error; - } else if (!ret) { + } + if (base_abstracts == NULL) { continue; } if (!(iter = PyObject_GetIter(base_abstracts))) { + Py_DECREF(base_abstracts); goto error; } + + PyObject *key, *value; while ((key = PyIter_Next(iter))) { - ret = _PyObject_LookupAttr(self, key, &value); - if (ret < 0) { + if (_PyObject_LookupAttr(self, key, &value) < 0) { Py_DECREF(key); Py_DECREF(iter); goto error; - } else if (!ret) { + } + if (value == NULL) { Py_DECREF(key); continue; } + int is_abstract = _PyObject_IsAbstract(value); Py_DECREF(value); - if (is_abstract < 0) { - Py_DECREF(key); - Py_DECREF(iter); - goto error; - } - if (is_abstract && PySet_Add(abstracts, key) < 0) { + if (is_abstract < 0 || + (is_abstract && PySet_Add(abstracts, key) < 0)) { Py_DECREF(key); Py_DECREF(iter); goto error; @@ -600,9 +574,67 @@ _abc_init(PyObject *m, PyObject *args) goto error; } } + if (_PyObject_SetAttrId(self, &PyId___abstractmethods__, abstracts) < 0) { goto error; } + + ret = 0; +error: + Py_DECREF(abstracts); + Py_CLEAR(ns); + Py_CLEAR(items); + Py_CLEAR(bases); + return ret; +} + +PyDoc_STRVAR(_abc_init_doc, +"Internal ABC helper for class set-up. Should be never used outside abc module"); + +static PyObject * +_abc_init(PyObject *m, PyObject *args) +{ + PyObject *bases, *keys, *abstracts, *base_abstracts, *self; + PyObject *key, *value, *item, *iter, *registry, *cache, *ref; + Py_ssize_t pos = 0; + int ret; + if (!PyArg_UnpackTuple(args, "_abc_init", 1, 1, &self)) { + return NULL; + } + + if (compute_abstract_methods(self) < 0) { + return NULL; + } + + /* Set up inheritance registry. */ + ref = PyWeakref_NewRef(self, NULL); + if (!ref) { + return NULL; + } + registry = gset_new(&_guarded_set_type, NULL, NULL); + if (!registry) { + return NULL; + } + if (PyDict_SetItem(_the_registry, ref, registry) < 0) { + return NULL; + } + cache = PySet_New(NULL); + if (!cache) { + return NULL; + } + if (PyDict_SetItem(_the_cache, ref, cache) < 0) { + return NULL; + } + cache = PySet_New(NULL); + if (!cache) { + return NULL; + } + if (PyDict_SetItem(_the_negative_cache, ref, cache) < 0) { + return NULL; + } + if (PyDict_SetItem(_the_cache_version, ref, abc_invalidation_counter) < 0) { + return NULL; + } Py_RETURN_NONE; error: Py_DECREF(abstracts); From b22232a999c59d21e7c8e7557d053f7a39f8cd16 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 20:09:59 +0000 Subject: [PATCH 35/93] Refactor via _abc_impl --- Modules/_abc.c | 302 +++++++++++++++++++++++++++---------------------- 1 file changed, 167 insertions(+), 135 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 74b8d3490dfebd..ab65cac67f4ae9 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -16,6 +16,7 @@ _Py_IDENTIFIER(__abstractmethods__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); +_Py_IDENTIFIER(_abc_impl); /* A global counter that is incremented each time a class is registered as a virtual subclass of anything. It forces the @@ -24,11 +25,6 @@ _Py_IDENTIFIER(__bases__); external code. */ static PyObject *abc_invalidation_counter; -static PyObject *_the_registry; -static PyObject *_the_cache; -static PyObject *_the_negative_cache; -static PyObject *_the_cache_version; - typedef struct { PyObject_HEAD unsigned long iterating; @@ -108,6 +104,94 @@ static PyTypeObject _guarded_set_type = { gset_new /* tp_new */ }; +typedef struct { + PyObject_HEAD + _guarded_set *_abc_registry; + PyObject *_abc_cache; + PyObject *_abc_negative_cache; + PyObject *_abc_negative_cache_version; +} _abc_data; + +static void +abc_data_dealloc(_abc_data *self) +{ + if (self->_abc_registry != NULL) { + gset_dealloc(self->_abc_registry); + } + if (self->_abc_cache != NULL) { + PySet_Clear(self->_abc_cache); + } + if (self->_abc_negative_cache != NULL) { + PySet_Clear(self->_abc_negative_cache); + } + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + abc_data *self; + PyObject *registry; + + self = (_abc_data *) type->tp_alloc(type, 0); + if (self != NULL) { + registry = gset_new(&_guarded_set_type, NULL, NULL) + if (!registry) { + return NULL; + } + self->_abc_registry = registry; + self->_abc_cache = PySet_New(NULL); + self->_abc_negative_cache = PySet_New(NULL); + if (!self->_abc_cache || !self->_abc_neative_cache) { + return NULL; + } + self->_abc_negative_cache_version = abc_invalidation_counter; + Py_INCREF(abc_invalidation_counter); + } + return (PyObject *) self; +} + +static PyTypeObject _abc_data_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "_abc_data", /*tp_name*/ + sizeof(_abc_data), /*tp_size*/ + 0, /*tp_itemsize*/ + (destructor)abc_data_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + abc_data_new /* tp_new */ +}; + static int _in_weak_set(PyObject *set, PyObject *obj) { @@ -128,31 +212,23 @@ _in_weak_set(PyObject *set, PyObject *obj) static int _in_cache(PyObject *self, PyObject *cls) { - PyObject *key, *cache; - key = PyWeakref_NewRef(self, NULL); - if (!key) { - return -1; - } - cache = PyObject_GetItem(_the_cache, key); - if (!cache) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return -1; } - return _in_weak_set(cache, cls); + return _in_weak_set(impl->_abc_cache, cls); } static int _in_negative_cache(PyObject *self, PyObject *cls) { - PyObject *key, *cache; - key = PyWeakref_NewRef(self, NULL); - if (!key) { - return -1; - } - cache = PyObject_GetItem(_the_negative_cache, key); - if (!cache) { + _abc_data *data; + data = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!data) { return -1; } - return _in_weak_set(cache, cls); + return _in_weak_set(data->_abc_negative_cache, cls); } static PyObject * @@ -179,6 +255,7 @@ _destroy_guarded(PyObject *setweakref, PyObject *objweakref) { PyObject *set; _guarded_set *gset; + int res; set = PyWeakref_GET_OBJECT(setweakref); if (set == Py_None) { Py_RETURN_NONE; @@ -186,7 +263,11 @@ _destroy_guarded(PyObject *setweakref, PyObject *objweakref) Py_INCREF(set); gset = (_guarded_set *)set; if (gset->iterating) { - PyList_Append(gset->pending, objweakref); + res = PyList_Append(gset->pending, objweakref); + if (!res) { + Py_DECREF(set); + return NULL; + } } else { if (PySet_Discard(gset->data, objweakref) < 0) { return NULL; @@ -230,47 +311,38 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) static PyObject * _get_registry(PyObject *self) { - PyObject *key; - _guarded_set *registry; - key = PyWeakref_NewRef(self, NULL); - if (!key) { - return 0; - } - registry = (_guarded_set *)PyObject_GetItem(_the_registry, key); - if (!registry) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return NULL; } - return registry->data; + return impl->_abc_registry->data; } static int _enter_iter(PyObject *self) { - PyObject *key; - _guarded_set *registry; - key = PyWeakref_NewRef(self, NULL); - if (!key) { - return 0; - } - registry = (_guarded_set *)PyObject_GetItem(_the_registry, key); - if (!registry) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return 0; } - registry->iterating++; + impl->_abc_registry->iterating++; return 1; } static int _exit_iter(PyObject *self) { - PyObject *key, *ref; + PyObject *ref; _guarded_set *registry; + _abc_data *impl; int pos; - key = PyWeakref_NewRef(self, NULL); - if (!key) { + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return 0; } - registry = (_guarded_set *) PyObject_GetItem(_the_registry, key); + registry = impl->_abc_registry if (!registry) { return 0; } @@ -280,7 +352,12 @@ _exit_iter(PyObject *self) } for (pos = 0; pos < PyList_GET_SIZE(registry->pending); pos++) { ref = PyObject_CallMethod(registry->pending, "pop", NULL); - PySet_Discard(registry->data, ref); + if (!ref) { + return 0; + } + if (PySet_Discard(registry->data, ref) < 0) { + return 0; + } } return 1; } @@ -288,43 +365,34 @@ _exit_iter(PyObject *self) static int _add_to_registry(PyObject *self, PyObject *cls) { - PyObject *key, *registry; - key = PyWeakref_NewRef(self, NULL); - if (!key) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return 0; } - registry = PyObject_GetItem(_the_registry, key); - return _add_to_weak_set(registry, cls, 1); + return _add_to_weak_set(impl->_abc_registry, cls, 1); } static int _add_to_cache(PyObject *self, PyObject *cls) { - PyObject *key, *cache; - key = PyWeakref_NewRef(self, NULL); - if (!key) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return 0; } - cache = PyObject_GetItem(_the_cache, key); - if (!cache) { - return 0; - } - return _add_to_weak_set(cache, cls, 0); + return _add_to_weak_set(impl->_abc_cache, cls, 0); } static int _add_to_negative_cache(PyObject *self, PyObject *cls) { - PyObject *key, *cache; - key = PyWeakref_NewRef(self, NULL); - if (!key) { - return 0; - } - cache = PyObject_GetItem(_the_negative_cache, key); - if (!cache) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return 0; } - return _add_to_weak_set(cache, cls, 0); + return _add_to_weak_set(impl->_abc_negative_cache, cls, 0); } PyDoc_STRVAR(_reset_registry_doc, @@ -352,15 +420,12 @@ _reset_registry(PyObject *m, PyObject *args) static int _reset_negative_cache(PyObject *self) { - PyObject *cache, *key = PyWeakref_NewRef(self, NULL); - if (!key) { - return 0; - } - cache = PyObject_GetItem(_the_negative_cache, key); - if (!cache) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return 0; } - if (PySet_Clear(cache) < 0) { + if (PySet_Clear(impl->_abc_negative_cache) < 0) { return 0; } return 1; @@ -374,19 +439,16 @@ Should be only used by refleak.py"); static PyObject * _reset_caches(PyObject *m, PyObject *args) { - PyObject *self, *cache, *key; + PyObject *self; + _abc_data *impl; if (!PyArg_UnpackTuple(args, "_reset_caches", 1, 1, &self)) { return NULL; } - key = PyWeakref_NewRef(self, NULL); - if (!key) { + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return NULL; } - cache = PyObject_GetItem(_the_cache, key); - if (!cache) { - return NULL; - } - if (PySet_Clear(cache) < 0) { + if (PySet_Clear(impl->_abc_cache) < 0) { return NULL; } /* also the second cache */ @@ -399,23 +461,26 @@ _reset_caches(PyObject *m, PyObject *args) static PyObject * _get_negative_cache_version(PyObject *self) { - PyObject *key; - key = PyWeakref_NewRef(self, NULL); - if (!key) { + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return NULL; } - return PyObject_GetItem(_the_cache_version, key); + return impl->_abc_negative_cache_version; } static int _set_negative_cache_version(PyObject *self, PyObject *version) { - PyObject *key; - key = PyWeakref_NewRef(self, NULL); - if (!key) { - return 0; + _abc_data *impl; + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { + return NULL; } - return PyObject_SetItem(_the_cache_version, key, version); + Py_DECREF(impl->_abc_negative_cache_version); + impl->_abc_negative_cache_version = abc_invalidation_counter; + Py_INCREF(impl->_abc_negative_cache_version); + Py_RETURN_NONE; } PyDoc_STRVAR(_get_dump_doc, @@ -428,8 +493,9 @@ instead use ABC._dump_registry() for a nice repr."); static PyObject * _get_dump(PyObject *m, PyObject *args) { - PyObject *self, *key, *registry, *cache, *negative_cache, *cache_version; + PyObject *self, *registry, *cache, *negative_cache, *cache_version; PyObject *res = PyTuple_New(4); + _abc_data *impl; if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { return NULL; } @@ -441,23 +507,15 @@ _get_dump(PyObject *m, PyObject *args) if (!registry) { return NULL; } - key = PyWeakref_NewRef(self, NULL); - if (!key) { + impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + if (!impl) { return NULL; } - cache = PyObject_GetItem(_the_cache, key); + cache = PyObject_CallMethod(impl->_abc_cache, "copy", NULL); if (!cache) { return NULL; } - cache = PyObject_CallMethod(cache, "copy", NULL); - if (!cache) { - return NULL; - } - negative_cache = PyObject_GetItem(_the_negative_cache, key); - if (!negative_cache) { - return NULL; - } - negative_cache = PyObject_CallMethod(negative_cache, "copy", NULL); + negative_cache = PyObject_CallMethod(impl->_abc_negative_cache, "copy", NULL); if (!negative_cache) { return NULL; } @@ -594,10 +652,7 @@ PyDoc_STRVAR(_abc_init_doc, static PyObject * _abc_init(PyObject *m, PyObject *args) { - PyObject *bases, *keys, *abstracts, *base_abstracts, *self; - PyObject *key, *value, *item, *iter, *registry, *cache, *ref; - Py_ssize_t pos = 0; - int ret; + PyObject *self, *data; if (!PyArg_UnpackTuple(args, "_abc_init", 1, 1, &self)) { return NULL; } @@ -607,38 +662,14 @@ _abc_init(PyObject *m, PyObject *args) } /* Set up inheritance registry. */ - ref = PyWeakref_NewRef(self, NULL); - if (!ref) { - return NULL; - } - registry = gset_new(&_guarded_set_type, NULL, NULL); - if (!registry) { - return NULL; - } - if (PyDict_SetItem(_the_registry, ref, registry) < 0) { - return NULL; - } - cache = PySet_New(NULL); - if (!cache) { + data = abc_data_new(&_abc_data_type, NULL, NULL); + if (!data) { return NULL; } - if (PyDict_SetItem(_the_cache, ref, cache) < 0) { - return NULL; - } - cache = PySet_New(NULL); - if (!cache) { - return NULL; - } - if (PyDict_SetItem(_the_negative_cache, ref, cache) < 0) { - return NULL; - } - if (PyDict_SetItem(_the_cache_version, ref, abc_invalidation_counter) < 0) { + if (!_PyObject_GetAttrId(self, PyId__abc_impl, data)) { return NULL; } Py_RETURN_NONE; -error: - Py_DECREF(abstracts); - return NULL; } PyDoc_STRVAR(_abc_register_doc, @@ -680,6 +711,7 @@ _abc_register(PyObject *m, PyObject *args) } Py_INCREF(subclass); /* Invalidate negative cache */ + Py_DECREF(abc_invalidation_counter); abc_invalidation_counter = PyNumber_Add(abc_invalidation_counter, PyLong_FromLong(1)); return subclass; } @@ -711,7 +743,7 @@ _abc_instancecheck(PyObject *m, PyObject *args) return NULL; } if ((PyObject_RichCompareBool(_get_negative_cache_version(self), - abc_invalidation_counter, Py_EQ) > 0) && incache) { + abc_invalidation_counter, Py_EQ) > 0) && incache) { Py_RETURN_FALSE; } /* Fall back to the subclass check. */ From e51c5caeb5d6d139e842786832dc967d4ad1006e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 20:37:41 +0000 Subject: [PATCH 36/93] Fix some refleaks --- Modules/_abc.c | 100 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 34 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index ab65cac67f4ae9..bde16b27053d17 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -130,19 +130,19 @@ abc_data_dealloc(_abc_data *self) static PyObject * abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - abc_data *self; + _abc_data *self; PyObject *registry; self = (_abc_data *) type->tp_alloc(type, 0); if (self != NULL) { - registry = gset_new(&_guarded_set_type, NULL, NULL) + registry = gset_new(&_guarded_set_type, NULL, NULL); if (!registry) { return NULL; } - self->_abc_registry = registry; + self->_abc_registry = (_guarded_set *)registry; self->_abc_cache = PySet_New(NULL); self->_abc_negative_cache = PySet_New(NULL); - if (!self->_abc_cache || !self->_abc_neative_cache) { + if (!self->_abc_cache || !self->_abc_negative_cache) { return NULL; } self->_abc_negative_cache_version = abc_invalidation_counter; @@ -213,22 +213,28 @@ static int _in_cache(PyObject *self, PyObject *cls) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + int res; + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return -1; } - return _in_weak_set(impl->_abc_cache, cls); + res = _in_weak_set(impl->_abc_cache, cls); + Py_DECREF(impl); + return res; } static int _in_negative_cache(PyObject *self, PyObject *cls) { - _abc_data *data; - data = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); - if (!data) { + _abc_data *impl; + int res; + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + if (!impl) { return -1; } - return _in_weak_set(data->_abc_negative_cache, cls); + res = _in_weak_set(impl->_abc_negative_cache, cls); + Py_DECREF(impl); + return res; } static PyObject * @@ -312,22 +318,26 @@ static PyObject * _get_registry(PyObject *self) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + PyObject *res; + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return NULL; } - return impl->_abc_registry->data; + res = impl->_abc_registry->data; + Py_DECREF(impl); + return res; } static int _enter_iter(PyObject *self) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return 0; } impl->_abc_registry->iterating++; + Py_DECREF(impl); return 1; } @@ -338,27 +348,32 @@ _exit_iter(PyObject *self) _guarded_set *registry; _abc_data *impl; int pos; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return 0; } - registry = impl->_abc_registry + registry = impl->_abc_registry; if (!registry) { + Py_DECREF(impl); return 0; } registry->iterating--; if (registry->iterating) { + Py_DECREF(impl); return 1; } for (pos = 0; pos < PyList_GET_SIZE(registry->pending); pos++) { ref = PyObject_CallMethod(registry->pending, "pop", NULL); if (!ref) { + Py_DECREF(impl); return 0; } if (PySet_Discard(registry->data, ref) < 0) { + Py_DECREF(impl); return 0; } } + Py_DECREF(impl); return 1; } @@ -366,33 +381,42 @@ static int _add_to_registry(PyObject *self, PyObject *cls) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + int res; + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return 0; } - return _add_to_weak_set(impl->_abc_registry, cls, 1); + res = _add_to_weak_set((PyObject *)(impl->_abc_registry), cls, 1); + Py_DECREF(impl); + return res; } static int _add_to_cache(PyObject *self, PyObject *cls) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + int res; + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return 0; } - return _add_to_weak_set(impl->_abc_cache, cls, 0); + res = _add_to_weak_set(impl->_abc_cache, cls, 0); + Py_DECREF(impl); + return res; } static int _add_to_negative_cache(PyObject *self, PyObject *cls) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + int res; + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return 0; } - return _add_to_weak_set(impl->_abc_negative_cache, cls, 0); + res = _add_to_weak_set(impl->_abc_negative_cache, cls, 0); + Py_DECREF(impl); + return res; } PyDoc_STRVAR(_reset_registry_doc, @@ -421,13 +445,15 @@ static int _reset_negative_cache(PyObject *self) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return 0; } if (PySet_Clear(impl->_abc_negative_cache) < 0) { + Py_DECREF(impl); return 0; } + Py_DECREF(impl); return 1; } @@ -444,15 +470,17 @@ _reset_caches(PyObject *m, PyObject *args) if (!PyArg_UnpackTuple(args, "_reset_caches", 1, 1, &self)) { return NULL; } - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return NULL; } if (PySet_Clear(impl->_abc_cache) < 0) { + Py_DECREF(impl); return NULL; } /* also the second cache */ if (!_reset_negative_cache(self)) { + Py_DECREF(impl); return NULL; } Py_RETURN_NONE; @@ -462,25 +490,29 @@ static PyObject * _get_negative_cache_version(PyObject *self) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + PyObject *res; + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return NULL; } - return impl->_abc_negative_cache_version; + res = impl->_abc_negative_cache_version; + Py_DECREF(impl); + return res; } static int _set_negative_cache_version(PyObject *self, PyObject *version) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { - return NULL; + return 0; } Py_DECREF(impl->_abc_negative_cache_version); impl->_abc_negative_cache_version = abc_invalidation_counter; Py_INCREF(impl->_abc_negative_cache_version); - Py_RETURN_NONE; + Py_DECREF(impl); + return 1; } PyDoc_STRVAR(_get_dump_doc, @@ -507,26 +539,30 @@ _get_dump(PyObject *m, PyObject *args) if (!registry) { return NULL; } - impl = (_abc_data *)_PyObject_GetAttrId(self, PyId__abc_impl); + impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return NULL; } cache = PyObject_CallMethod(impl->_abc_cache, "copy", NULL); if (!cache) { + Py_DECREF(impl); return NULL; } negative_cache = PyObject_CallMethod(impl->_abc_negative_cache, "copy", NULL); if (!negative_cache) { + Py_DECREF(impl); return NULL; } cache_version = _get_negative_cache_version(self); if (!cache_version) { + Py_DECREF(impl); return NULL; } PyTuple_SetItem(res, 0, registry); PyTuple_SetItem(res, 1, cache); PyTuple_SetItem(res, 2, negative_cache); PyTuple_SetItem(res, 3, cache_version); + Py_DECREF(impl); return res; } @@ -666,7 +702,7 @@ _abc_init(PyObject *m, PyObject *args) if (!data) { return NULL; } - if (!_PyObject_GetAttrId(self, PyId__abc_impl, data)) { + if (_PyObject_SetAttrId(self, &PyId__abc_impl, data) < 0) { return NULL; } Py_RETURN_NONE; @@ -944,9 +980,5 @@ PyMODINIT_FUNC PyInit__abc(void) { abc_invalidation_counter = PyLong_FromLong(0); - _the_registry = PyDict_New(); - _the_cache = PyDict_New(); - _the_negative_cache = PyDict_New(); - _the_cache_version = PyDict_New(); return PyModule_Create(&_abcmodule); } From bac7a437af3820583475be0c1bdc68673ae5f51f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 20:55:02 +0000 Subject: [PATCH 37/93] Add docs (required by some tests) and initialization --- Modules/_abc.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Modules/_abc.c b/Modules/_abc.c index bde16b27053d17..f1abb52fe84a3d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -63,6 +63,10 @@ gset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *) self; } +PyDoc_STRVAR(guarded_set_doc, +"Internal weak set guarded against deletion during iteration.\n\ +Used by ABC machinery."); + static PyTypeObject _guarded_set_type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "_guarded_set", /*tp_name*/ @@ -151,6 +155,9 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *) self; } +PyDoc_STRVAR(abc_data_doc, +"Internal state held by ABC machinery."); + static PyTypeObject _abc_data_type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "_abc_data", /*tp_name*/ @@ -979,6 +986,16 @@ static struct PyModuleDef _abcmodule = { PyMODINIT_FUNC PyInit__abc(void) { + if (PyType_Ready(&_abc_data_type) < 0) { + return NULL; + } + _abc_data_type.tp_doc = abc_data_doc; + + if (PyType_Ready(&_guarded_set_type) < 0) { + return NULL; + } + _guarded_set_type.tp_doc = guarded_set_doc; + abc_invalidation_counter = PyLong_FromLong(0); return PyModule_Create(&_abcmodule); } From 0d7513b6ccf88073aeff3bde0ccb92453f7207e9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 21:47:44 +0000 Subject: [PATCH 38/93] Outdated comment and detection of intrusions --- Modules/_abc.c | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index f1abb52fe84a3d..b05f455527b98a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -216,12 +216,28 @@ _in_weak_set(PyObject *set, PyObject *obj) return res; } +static _abc_data * +_get_impl(PyObject *self) +{ + PyObject *impl; + impl = _PyObject_GetAttrId(self, &PyId__abc_impl); + if (!impl) { + return NULL; + } + if (PY_TYPE(impl) != &_abc_data_type) { + PyErr_SetString(PyExc_TypeError, "_abc_impl is set to a wrong type"); + Py_DECREF(impl); + return NULL; + } + return (_abc_data *)impl; +} + static int _in_cache(PyObject *self, PyObject *cls) { _abc_data *impl; int res; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return -1; } @@ -235,7 +251,7 @@ _in_negative_cache(PyObject *self, PyObject *cls) { _abc_data *impl; int res; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return -1; } @@ -326,7 +342,7 @@ _get_registry(PyObject *self) { _abc_data *impl; PyObject *res; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return NULL; } @@ -339,7 +355,7 @@ static int _enter_iter(PyObject *self) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return 0; } @@ -355,7 +371,7 @@ _exit_iter(PyObject *self) _guarded_set *registry; _abc_data *impl; int pos; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return 0; } @@ -389,7 +405,7 @@ _add_to_registry(PyObject *self, PyObject *cls) { _abc_data *impl; int res; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return 0; } @@ -403,7 +419,7 @@ _add_to_cache(PyObject *self, PyObject *cls) { _abc_data *impl; int res; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return 0; } @@ -417,7 +433,7 @@ _add_to_negative_cache(PyObject *self, PyObject *cls) { _abc_data *impl; int res; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return 0; } @@ -452,7 +468,7 @@ static int _reset_negative_cache(PyObject *self) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return 0; } @@ -477,7 +493,7 @@ _reset_caches(PyObject *m, PyObject *args) if (!PyArg_UnpackTuple(args, "_reset_caches", 1, 1, &self)) { return NULL; } - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return NULL; } @@ -498,7 +514,7 @@ _get_negative_cache_version(PyObject *self) { _abc_data *impl; PyObject *res; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return NULL; } @@ -511,7 +527,7 @@ static int _set_negative_cache_version(PyObject *self, PyObject *version) { _abc_data *impl; - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return 0; } @@ -546,7 +562,7 @@ _get_dump(PyObject *m, PyObject *args) if (!registry) { return NULL; } - impl = (_abc_data *)_PyObject_GetAttrId(self, &PyId__abc_impl); + impl = _get_impl(self); if (!impl) { return NULL; } @@ -585,8 +601,7 @@ compute_abstract_methods(PyObject *self) PyObject *ns=NULL, *items=NULL, *bases=NULL; // Py_CLEAR()ed on error. - /* Stage 1: direct abstract methods. - (It is safe to assume everything is fine since type.__new__ succeeded.) */ + /* Stage 1: direct abstract methods. */ ns = _PyObject_GetAttrId(self, &PyId___dict__); if (!ns) { goto error; From c429f49ffaca5f5815a757aa65b62f0422a4a0ad Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 22:22:39 +0000 Subject: [PATCH 39/93] Minor fixes --- Lib/test/libregrtest/refleak.py | 5 +++++ Modules/_abc.c | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 4497aac0d7bfe2..d6faf585797218 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -36,6 +36,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue + for obj in abc.__subclasses__() + [abc]: + _reset_caches(obj) # bpo-31217: Integer pool to get a single integer object for the same # value. The pool is used to prevent false alarm when checking for memory @@ -133,6 +135,9 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) + for abc in abs_classes: + for obj in abc.__subclasses__() + [abc]: + _reset_caches(obj) clear_caches() diff --git a/Modules/_abc.c b/Modules/_abc.c index b05f455527b98a..e60a5d8c4a519c 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -224,7 +224,7 @@ _get_impl(PyObject *self) if (!impl) { return NULL; } - if (PY_TYPE(impl) != &_abc_data_type) { + if (Py_TYPE(impl) != &_abc_data_type) { PyErr_SetString(PyExc_TypeError, "_abc_impl is set to a wrong type"); Py_DECREF(impl); return NULL; @@ -581,6 +581,10 @@ _get_dump(PyObject *m, PyObject *args) Py_DECREF(impl); return NULL; } + Py_INCREF(registry); + Py_INCREF(cache); + Py_INCREF(negative_cache); + Py_INCREF(cache_version); PyTuple_SetItem(res, 0, registry); PyTuple_SetItem(res, 1, cache); PyTuple_SetItem(res, 2, negative_cache); From 357b56d42b98dbb7124dc92bb7b366c1b1d8beca Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Jan 2018 23:06:36 +0000 Subject: [PATCH 40/93] Few more refleaks --- Lib/test/libregrtest/refleak.py | 6 ++++- Modules/_abc.c | 39 ++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index d6faf585797218..691a3beaa7dec0 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -37,6 +37,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: + abcs[obj] = _get_dump(obj)[0] _reset_caches(obj) # bpo-31217: Integer pool to get a single integer object for the same @@ -137,7 +138,10 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: - _reset_caches(obj) + for ref in abcs.get(obj, set()): + if ref() is not None: + obj.register(ref()) + # _reset_caches(obj) clear_caches() diff --git a/Modules/_abc.c b/Modules/_abc.c index e60a5d8c4a519c..54950bad98b0ab 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -269,6 +269,7 @@ _destroy(PyObject *setweakref, PyObject *objweakref) Py_RETURN_NONE; Py_INCREF(set); if (PySet_Discard(set, objweakref) < 0) { + Py_DECREF(set); return NULL; } Py_DECREF(set); @@ -299,6 +300,7 @@ _destroy_guarded(PyObject *setweakref, PyObject *objweakref) } } else { if (PySet_Discard(gset->data, objweakref) < 0) { + Py_DECREF(set); return NULL; } } @@ -326,14 +328,19 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) } ref = PyWeakref_NewRef(obj, destroy_cb); if (!ref) { + Py_DECREF(wr); return 0; } if (guarded) { set = ((_guarded_set *) set)->data; } if (PySet_Add(set, ref) < 0) { + Py_DECREF(wr); + Py_DECREF(ref); return 0; } + Py_DECREF(wr); + Py_DECREF(ref); return 1; } @@ -392,9 +399,11 @@ _exit_iter(PyObject *self) return 0; } if (PySet_Discard(registry->data, ref) < 0) { + Py_DECREF(ref); Py_DECREF(impl); return 0; } + Py_DECREF(ref); } Py_DECREF(impl); return 1; @@ -506,6 +515,7 @@ _reset_caches(PyObject *m, PyObject *args) Py_DECREF(impl); return NULL; } + Py_DECREF(impl); Py_RETURN_NONE; } @@ -740,7 +750,7 @@ PyDoc_STRVAR(_abc_register_doc, static PyObject * _abc_register(PyObject *m, PyObject *args) { - PyObject *self, *subclass = NULL; + PyObject *self, *one, *subclass = NULL; int result; if (!PyArg_UnpackTuple(args, "_abc_register", 2, 2, &self, &subclass)) { return NULL; @@ -774,7 +784,9 @@ _abc_register(PyObject *m, PyObject *args) Py_INCREF(subclass); /* Invalidate negative cache */ Py_DECREF(abc_invalidation_counter); - abc_invalidation_counter = PyNumber_Add(abc_invalidation_counter, PyLong_FromLong(1)); + one = PyLong_FromLong(1); + abc_invalidation_counter = PyNumber_Add(abc_invalidation_counter, one); + Py_DECREF(one); return subclass; } @@ -793,32 +805,41 @@ _abc_instancecheck(PyObject *m, PyObject *args) /* Inline the cache checking. */ incache = _in_cache(self, subclass); if (incache < 0) { + Py_DECREF(subclass); return NULL; } if (incache > 0) { + Py_DECREF(subclass); Py_RETURN_TRUE; } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { incache = _in_negative_cache(self, subclass); if (incache < 0) { + Py_DECREF(subclass); return NULL; } if ((PyObject_RichCompareBool(_get_negative_cache_version(self), abc_invalidation_counter, Py_EQ) > 0) && incache) { + Py_DECREF(subclass); Py_RETURN_FALSE; } /* Fall back to the subclass check. */ - return PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); + result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); + Py_DECREF(subclass); + return result; } result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); if (!result) { + Py_DECREF(subclass); return NULL; } if (result == Py_True) { + Py_DECREF(subclass); return Py_True; } Py_DECREF(result); + Py_DECREF(subclass); return PyObject_CallMethod(self, "__subclasscheck__", "O", subtype); } @@ -904,7 +925,9 @@ _abc_subclasscheck(PyObject *m, PyObject *args) /* 5. Check if it's a subclass of a registered class (recursive). */ pos = 0; Py_hash_t hash; - _enter_iter(self); + if (!_enter_iter(self)) { + return NULL; + } while (_PySet_NextEntry(_get_registry(self), &pos, &key, &hash)) { rkey = PyWeakref_GetObject(key); if (rkey == Py_None) { @@ -920,11 +943,15 @@ _abc_subclasscheck(PyObject *m, PyObject *args) _exit_iter(self); return NULL; } - _exit_iter(self); + if (!_exit_iter(self)) { + return NULL; + } Py_RETURN_TRUE; } } - _exit_iter(self); + if (!_exit_iter(self)) { + return NULL; + } /* 6. Check if it's a subclass of a subclass (recursive). */ subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); From cd80fcb5aa6e126d9f1002a7b3447c427e8e8949 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 00:07:35 +0000 Subject: [PATCH 41/93] Restore unwanted changes --- Lib/abc.py | 3 ++- Lib/test/libregrtest/refleak.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 565ed707c076da..3cdf063da1ac39 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -143,7 +143,8 @@ class ABCMeta(type): # negative cache to be cleared before its next use. # Note: this counter is private. Use `abc.get_cache_token()` for # external code. - _abc_invalidation_counter = 0 + if not _C_speedup: + _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace, **kwargs): cls = super().__new__(mcls, name, bases, namespace, **kwargs) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 691a3beaa7dec0..4d286f00db8f59 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -38,7 +38,6 @@ def dash_R(the_module, test, indirect_test, huntrleaks): continue for obj in abc.__subclasses__() + [abc]: abcs[obj] = _get_dump(obj)[0] - _reset_caches(obj) # bpo-31217: Integer pool to get a single integer object for the same # value. The pool is used to prevent false alarm when checking for memory @@ -141,7 +140,6 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): for ref in abcs.get(obj, set()): if ref() is not None: obj.register(ref()) - # _reset_caches(obj) clear_caches() From 34e13c319b9aebc666027e1bcc2158818116179e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 01:37:20 +0000 Subject: [PATCH 42/93] Fix(?) some more refleaks --- Modules/_abc.c | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 54950bad98b0ab..5a1215bef9fe3d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -36,12 +36,8 @@ typedef struct { static void gset_dealloc(_guarded_set *self) { - if (self->data != NULL) { - PySet_Clear(self->data); - } - if (self->pending != NULL) { - PyList_Type.tp_dealloc(self->pending); - } + Py_DECREF(self->data); + Py_DECREF(self->pending); if (self->in_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) self); } @@ -119,15 +115,10 @@ typedef struct { static void abc_data_dealloc(_abc_data *self) { - if (self->_abc_registry != NULL) { - gset_dealloc(self->_abc_registry); - } - if (self->_abc_cache != NULL) { - PySet_Clear(self->_abc_cache); - } - if (self->_abc_negative_cache != NULL) { - PySet_Clear(self->_abc_negative_cache); - } + Py_DECREF(self->_abc_registry); + Py_DECREF(self->_abc_cache); + Py_DECREF(self->_abc_negative_cache); + Py_DECREF(self->_abc_negative_cache_version); Py_TYPE(self)->tp_free(self); } @@ -327,6 +318,7 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) destroy_cb = PyCFunction_NewEx(&_destroy_def, wr, NULL); } ref = PyWeakref_NewRef(obj, destroy_cb); + Py_DECREF(destroy_cb); if (!ref) { Py_DECREF(wr); return 0; @@ -741,6 +733,7 @@ _abc_init(PyObject *m, PyObject *args) if (_PyObject_SetAttrId(self, &PyId__abc_impl, data) < 0) { return NULL; } + Py_DECREF(data); Py_RETURN_NONE; } From c5633b6fb5a61d00bcb04829d9224d341b7fe7cc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 01:43:34 +0000 Subject: [PATCH 43/93] Reset caches between runs --- Lib/test/libregrtest/refleak.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 4d286f00db8f59..961636e75cf930 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -140,6 +140,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): for ref in abcs.get(obj, set()): if ref() is not None: obj.register(ref()) + _reset_caches(obj) clear_caches() From 3cbbc122f5283f88adcde356ce9947eea449b1e8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 28 Jan 2018 10:52:20 +0900 Subject: [PATCH 44/93] Fix abuse of borrowed reference --- Modules/_abc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_abc.c b/Modules/_abc.c index 5a1215bef9fe3d..916ee95141ddc5 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -635,12 +635,17 @@ compute_abstract_methods(PyObject *self) // borrowed PyObject *key = PySequence_Fast_GET_ITEM(it, 0); PyObject *value = PySequence_Fast_GET_ITEM(it, 1); + // items or it may be cleared while accessing __abstractmethod__ + // So we need to keep strong reference for key + Py_INCREF(key); int is_abstract = _PyObject_IsAbstract(value); if (is_abstract < 0 || (is_abstract && PySet_Add(abstracts, key) < 0)) { Py_DECREF(it); + Py_DECREF(key); goto error; } + Py_DECREF(key); } /* Stage 2: inherited abstract methods. */ From 23bcb0768f386aa3dfd22c19d54bbd3b09f77eee Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 02:56:29 +0000 Subject: [PATCH 45/93] Fix remaining refleaks --- Modules/_abc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 916ee95141ddc5..385ab0906912c3 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -646,6 +646,7 @@ compute_abstract_methods(PyObject *self) goto error; } Py_DECREF(key); + Py_DECREF(it); } /* Stage 2: inherited abstract methods. */ @@ -673,7 +674,7 @@ compute_abstract_methods(PyObject *self) Py_DECREF(base_abstracts); goto error; } - + Py_DECREF(base_abstracts); PyObject *key, *value; while ((key = PyIter_Next(iter))) { if (_PyObject_LookupAttr(self, key, &value) < 0) { From 0aab4799fc77e75cfa776a1aa17feb9b00781a1e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 03:18:47 +0000 Subject: [PATCH 46/93] Updare TODO, switch to common result agreement, few more checks --- Modules/_abc.c | 80 ++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 385ab0906912c3..de02656a090acc 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,6 +1,5 @@ /* ABCMeta implementation */ -/* TODO: Fix refleaks: DECREF all WeakRefs! */ /* TODO: Think (ask) about thread-safety. */ /* TODO: Add checks only to those calls that can fail and use _GET_SIZE etc. */ /* TODO: Think about inlining some calls (like __subclasses__) and/or using macros */ @@ -310,7 +309,7 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) PyObject *destroy_cb; wr = PyWeakref_NewRef(set, NULL); if (!wr) { - return 0; + return -1; } if (guarded) { destroy_cb = PyCFunction_NewEx(&_destroy_guarded_def, wr, NULL); @@ -321,7 +320,7 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) Py_DECREF(destroy_cb); if (!ref) { Py_DECREF(wr); - return 0; + return -1; } if (guarded) { set = ((_guarded_set *) set)->data; @@ -329,11 +328,11 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) if (PySet_Add(set, ref) < 0) { Py_DECREF(wr); Py_DECREF(ref); - return 0; + return -1; } Py_DECREF(wr); Py_DECREF(ref); - return 1; + return 0; } static PyObject * @@ -356,11 +355,11 @@ _enter_iter(PyObject *self) _abc_data *impl; impl = _get_impl(self); if (!impl) { - return 0; + return -1; } impl->_abc_registry->iterating++; Py_DECREF(impl); - return 1; + return 0; } static int @@ -372,33 +371,33 @@ _exit_iter(PyObject *self) int pos; impl = _get_impl(self); if (!impl) { - return 0; + return -1; } registry = impl->_abc_registry; if (!registry) { Py_DECREF(impl); - return 0; + return -1; } registry->iterating--; if (registry->iterating) { Py_DECREF(impl); - return 1; + return 0; } for (pos = 0; pos < PyList_GET_SIZE(registry->pending); pos++) { ref = PyObject_CallMethod(registry->pending, "pop", NULL); if (!ref) { Py_DECREF(impl); - return 0; + return -1; } if (PySet_Discard(registry->data, ref) < 0) { Py_DECREF(ref); Py_DECREF(impl); - return 0; + return -1; } Py_DECREF(ref); } Py_DECREF(impl); - return 1; + return 0; } static int @@ -408,7 +407,7 @@ _add_to_registry(PyObject *self, PyObject *cls) int res; impl = _get_impl(self); if (!impl) { - return 0; + return -1; } res = _add_to_weak_set((PyObject *)(impl->_abc_registry), cls, 1); Py_DECREF(impl); @@ -422,7 +421,7 @@ _add_to_cache(PyObject *self, PyObject *cls) int res; impl = _get_impl(self); if (!impl) { - return 0; + return -1; } res = _add_to_weak_set(impl->_abc_cache, cls, 0); Py_DECREF(impl); @@ -436,7 +435,7 @@ _add_to_negative_cache(PyObject *self, PyObject *cls) int res; impl = _get_impl(self); if (!impl) { - return 0; + return -1; } res = _add_to_weak_set(impl->_abc_negative_cache, cls, 0); Py_DECREF(impl); @@ -471,14 +470,14 @@ _reset_negative_cache(PyObject *self) _abc_data *impl; impl = _get_impl(self); if (!impl) { - return 0; + return -1; } if (PySet_Clear(impl->_abc_negative_cache) < 0) { Py_DECREF(impl); - return 0; + return -1; } Py_DECREF(impl); - return 1; + return 0; } PyDoc_STRVAR(_reset_caches_doc, @@ -503,7 +502,7 @@ _reset_caches(PyObject *m, PyObject *args) return NULL; } /* also the second cache */ - if (!_reset_negative_cache(self)) { + if (_reset_negative_cache(self) < 0) { Py_DECREF(impl); return NULL; } @@ -531,13 +530,13 @@ _set_negative_cache_version(PyObject *self, PyObject *version) _abc_data *impl; impl = _get_impl(self); if (!impl) { - return 0; + return -1; } Py_DECREF(impl->_abc_negative_cache_version); impl->_abc_negative_cache_version = abc_invalidation_counter; Py_INCREF(impl->_abc_negative_cache_version); Py_DECREF(impl); - return 1; + return 0; } PyDoc_STRVAR(_get_dump_doc, @@ -587,10 +586,13 @@ _get_dump(PyObject *m, PyObject *args) Py_INCREF(cache); Py_INCREF(negative_cache); Py_INCREF(cache_version); - PyTuple_SetItem(res, 0, registry); - PyTuple_SetItem(res, 1, cache); - PyTuple_SetItem(res, 2, negative_cache); - PyTuple_SetItem(res, 3, cache_version); + if ((PyTuple_SetItem(res, 0, registry) < 0) || + (PyTuple_SetItem(res, 1, cache) < 0) || + (PyTuple_SetItem(res, 2, negative_cache < 0) || + (PyTuple_SetItem(res, 3, cache_version) < 0)) { + Py_DECREF(impl); + return NuLL; + } Py_DECREF(impl); return res; } @@ -777,7 +779,7 @@ _abc_register(PyObject *m, PyObject *args) if (result < 0) { return NULL; } - if (!_add_to_registry(self, subclass)) { + if (_add_to_registry(self, subclass) < 0) { return NULL; } Py_INCREF(subclass); @@ -877,10 +879,12 @@ _abc_subclasscheck(PyObject *m, PyObject *args) if (PyObject_RichCompareBool(_get_negative_cache_version(self), abc_invalidation_counter, Py_LT) > 0) { /* Invalidate the negative cache. */ - if (!_reset_negative_cache(self)) { + if (_reset_negative_cache(self) < 0) { + return NULL; + } + if (_set_negative_cache_version(self, abc_invalidation_counter) < 0) { return NULL; } - _set_negative_cache_version(self, abc_invalidation_counter); } else if (incache) { Py_RETURN_FALSE; } @@ -890,14 +894,14 @@ _abc_subclasscheck(PyObject *m, PyObject *args) return NULL; } if (ok == Py_True) { - if (!_add_to_cache(self, subclass)) { + if (_add_to_cache(self, subclass) < 0) { Py_DECREF(ok); return NULL; } return Py_True; } if (ok == Py_False) { - if (!_add_to_negative_cache(self, subclass)) { + if (_add_to_negative_cache(self, subclass) < 0) { Py_DECREF(ok); return NULL; } @@ -914,7 +918,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) mro = ((PyTypeObject *)subclass)->tp_mro; for (pos = 0; pos < PyTuple_Size(mro); pos++) { if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { - if (!_add_to_cache(self, subclass)) { + if (_add_to_cache(self, subclass) < 0) { return NULL; } Py_RETURN_TRUE; @@ -924,7 +928,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) /* 5. Check if it's a subclass of a registered class (recursive). */ pos = 0; Py_hash_t hash; - if (!_enter_iter(self)) { + if (_enter_iter(self) < 0) { return NULL; } while (_PySet_NextEntry(_get_registry(self), &pos, &key, &hash)) { @@ -938,17 +942,17 @@ _abc_subclasscheck(PyObject *m, PyObject *args) return NULL; } if (result > 0) { - if (!_add_to_cache(self, subclass)) { + if (_add_to_cache(self, subclass) < 0) { _exit_iter(self); return NULL; } - if (!_exit_iter(self)) { + if (_exit_iter(self) < 0) { return NULL; } Py_RETURN_TRUE; } } - if (!_exit_iter(self)) { + if (_exit_iter(self) < 0) { return NULL; } @@ -961,7 +965,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (result > 0) { - if (!_add_to_cache(self, subclass)) { + if (_add_to_cache(self, subclass) < 0) { return NULL; } Py_DECREF(subclasses); @@ -974,7 +978,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } Py_DECREF(subclasses); /* No dice; update negative cache. */ - if (!_add_to_negative_cache(self, subclass)) { + if (_add_to_negative_cache(self, subclass) < 0) { return NULL; } Py_RETURN_FALSE; From 86e06604028d1f4b2f32bacdc049cf6122ba1bad Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 03:23:26 +0000 Subject: [PATCH 47/93] Fix typos --- Modules/_abc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index de02656a090acc..5b608fd9efdaa1 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -588,10 +588,10 @@ _get_dump(PyObject *m, PyObject *args) Py_INCREF(cache_version); if ((PyTuple_SetItem(res, 0, registry) < 0) || (PyTuple_SetItem(res, 1, cache) < 0) || - (PyTuple_SetItem(res, 2, negative_cache < 0) || + (PyTuple_SetItem(res, 2, negative_cache) < 0) || (PyTuple_SetItem(res, 3, cache_version) < 0)) { Py_DECREF(impl); - return NuLL; + return NULL; } Py_DECREF(impl); return res; From c55e482bb002baae955e68c95762776e8ef81078 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 03:30:08 +0000 Subject: [PATCH 48/93] Remove irrelevant TODOs, add few comments --- Modules/_abc.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 5b608fd9efdaa1..66380d2be5eaa1 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -27,8 +27,8 @@ static PyObject *abc_invalidation_counter; typedef struct { PyObject_HEAD unsigned long iterating; - PyObject *data; - PyObject *pending; + PyObject *data; /* The actual set of weak references. */ + PyObject *pending; /* Pending removals collected during iteration. */ PyObject *in_weakreflist; } _guarded_set; @@ -103,11 +103,14 @@ static PyTypeObject _guarded_set_type = { gset_new /* tp_new */ }; +/* This object stores internal state for ABCs. + Note that we can use normal sets for caches, + since they are never iterated over. */ typedef struct { PyObject_HEAD _guarded_set *_abc_registry; - PyObject *_abc_cache; - PyObject *_abc_negative_cache; + PyObject *_abc_cache; /* Normal set of weak references. */ + PyObject *_abc_negative_cache; /* Normal set of weak references. */ PyObject *_abc_negative_cache_version; } _abc_data; @@ -858,10 +861,6 @@ _abc_subclasscheck(PyObject *m, PyObject *args) if (!PyArg_UnpackTuple(args, "_abc_subclasscheck", 2, 2, &self, &subclass)) { return NULL; } - /* TODO: clear the registry from dead refs from time to time - on iteration here (have a counter for this) or maybe during a .register() call */ - /* TODO: Reset caches every n-th succes/failure correspondingly - so that they don't grow too large */ /* 1. Check cache. */ incache = _in_cache(self, subclass); From 5f9526ac6a3630fa6036026190cd6121a3710e73 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 28 Jan 2018 14:45:31 +0900 Subject: [PATCH 49/93] Use Py_ssize_t for iterating --- Modules/_abc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 66380d2be5eaa1..3dffe4ef64f1c5 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -26,7 +26,7 @@ static PyObject *abc_invalidation_counter; typedef struct { PyObject_HEAD - unsigned long iterating; + Py_ssize_t iterating; PyObject *data; /* The actual set of weak references. */ PyObject *pending; /* Pending removals collected during iteration. */ PyObject *in_weakreflist; From 8174b61c2823e490cc06272e942b46f94185ef80 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 28 Jan 2018 14:48:45 +0900 Subject: [PATCH 50/93] Use c99 designated initializer. --- Modules/_abc.c | 83 +++++++------------------------------------------- 1 file changed, 11 insertions(+), 72 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 3dffe4ef64f1c5..dad818a2b46301 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -63,45 +63,15 @@ PyDoc_STRVAR(guarded_set_doc, Used by ABC machinery."); static PyTypeObject _guarded_set_type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) + PyVarObject_HEAD_INIT(NULL, 0) "_guarded_set", /*tp_name*/ sizeof(_guarded_set), /*tp_size*/ - 0, /*tp_itemsize*/ - (destructor)gset_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(_guarded_set, in_weakreflist), /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - gset_new /* tp_new */ - }; + .tp_dealloc = (destructor)gset_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(_guarded_set, in_weakreflist), + .tp_alloc = PyType_GenericAlloc, + .tp_new = gset_new, +}; /* This object stores internal state for ABCs. Note that we can use normal sets for caches, @@ -155,41 +125,10 @@ static PyTypeObject _abc_data_type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "_abc_data", /*tp_name*/ sizeof(_abc_data), /*tp_size*/ - 0, /*tp_itemsize*/ - (destructor)abc_data_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - abc_data_new /* tp_new */ + .tp_dealloc = (destructor)abc_data_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_alloc = PyType_GenericAlloc, + .tp_new = abc_data_new, }; static int From bb8d623f4aa360cdd2d6a1d29d400a6d51d0a510 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 28 Jan 2018 14:57:21 +0900 Subject: [PATCH 51/93] Fix gset_new() and abc_data_new() --- Modules/_abc.c | 55 +++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index dad818a2b46301..65ad0c32e21fd3 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -46,13 +46,20 @@ gset_dealloc(_guarded_set *self) static PyObject * gset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - _guarded_set *self; - - self = (_guarded_set *) type->tp_alloc(type, 0); + PyObject *data = PySet_New(NULL); + if (data == NULL) { + return NULL; + } + PyObject *pending = PyList_New(0); + if (pending == NULL) { + Py_DECREF(data); + return NULL; + } + _guarded_set *self = (_guarded_set *) type->tp_alloc(type, 0); if (self != NULL) { self->iterating = 0; - self->data = PySet_New(NULL); - self->pending = PyList_New(0); + self->data = data; + self->pending = pending; self->in_weakreflist = NULL; } return (PyObject *) self; @@ -97,25 +104,31 @@ abc_data_dealloc(_abc_data *self) static PyObject * abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - _abc_data *self; - PyObject *registry; + PyObject *registry = NULL, *cache = NULL, *negative_cache = NULL; + registry = gset_new(&_guarded_set_type, NULL, NULL); + cache = PySet_New(NULL); + negative_cache = PySet_New(NULL); + if (!registry || !cache || !negative_cache) { + goto error; + } - self = (_abc_data *) type->tp_alloc(type, 0); - if (self != NULL) { - registry = gset_new(&_guarded_set_type, NULL, NULL); - if (!registry) { - return NULL; - } - self->_abc_registry = (_guarded_set *)registry; - self->_abc_cache = PySet_New(NULL); - self->_abc_negative_cache = PySet_New(NULL); - if (!self->_abc_cache || !self->_abc_negative_cache) { - return NULL; - } - self->_abc_negative_cache_version = abc_invalidation_counter; - Py_INCREF(abc_invalidation_counter); + _abc_data *self = (_abc_data *) type->tp_alloc(type, 0); + if (self == NULL) { + goto error; } + + self->_abc_registry = (_guarded_set *)registry; + self->_abc_cache = cache; + self->_abc_negative_cache = negative_cache; + self->_abc_negative_cache_version = abc_invalidation_counter; + Py_INCREF(abc_invalidation_counter); return (PyObject *) self; + +error: + Py_CLEAR(registry); + Py_CLEAR(cache); + Py_CLEAR(negative_cache); + return NULL; } PyDoc_STRVAR(abc_data_doc, From ef59e549d5490302eb1857075e6e2e0ce40a0084 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 28 Jan 2018 17:25:25 +0900 Subject: [PATCH 52/93] Massive refactoring... --- Modules/_abc.c | 388 ++++++++++++++++++++++--------------------------- 1 file changed, 174 insertions(+), 214 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 65ad0c32e21fd3..f08d6c549eb36a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -125,9 +125,9 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *) self; error: - Py_CLEAR(registry); - Py_CLEAR(cache); - Py_CLEAR(negative_cache); + Py_XDECREF(registry); + Py_XDECREF(cache); + Py_XDECREF(negative_cache); return NULL; } @@ -147,16 +147,15 @@ static PyTypeObject _abc_data_type = { static int _in_weak_set(PyObject *set, PyObject *obj) { - PyObject *ref; - int res; - ref = PyWeakref_NewRef(obj, NULL); + PyObject *ref = PyWeakref_NewRef(obj, NULL); if (!ref) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); return 0; } return -1; } - res = PySet_Contains(set, ref); + int res = PySet_Contains(set, ref); Py_DECREF(ref); return res; } @@ -164,8 +163,7 @@ _in_weak_set(PyObject *set, PyObject *obj) static _abc_data * _get_impl(PyObject *self) { - PyObject *impl; - impl = _PyObject_GetAttrId(self, &PyId__abc_impl); + PyObject *impl = _PyObject_GetAttrId(self, &PyId__abc_impl); if (!impl) { return NULL; } @@ -178,31 +176,15 @@ _get_impl(PyObject *self) } static int -_in_cache(PyObject *self, PyObject *cls) +_in_cache(_abc_data *impl, PyObject *cls) { - _abc_data *impl; - int res; - impl = _get_impl(self); - if (!impl) { - return -1; - } - res = _in_weak_set(impl->_abc_cache, cls); - Py_DECREF(impl); - return res; + return _in_weak_set(impl->_abc_cache, cls); } static int -_in_negative_cache(PyObject *self, PyObject *cls) +_in_negative_cache(_abc_data *impl, PyObject *cls) { - _abc_data *impl; - int res; - impl = _get_impl(self); - if (!impl) { - return -1; - } - res = _in_weak_set(impl->_abc_negative_cache, cls); - Py_DECREF(impl); - return res; + return _in_weak_set(impl->_abc_negative_cache, cls); } static PyObject * @@ -304,55 +286,39 @@ _get_registry(PyObject *self) return res; } -static int -_enter_iter(PyObject *self) +static void +_enter_iter(_guarded_set *gs) { - _abc_data *impl; - impl = _get_impl(self); - if (!impl) { - return -1; - } - impl->_abc_registry->iterating++; - Py_DECREF(impl); - return 0; + gs->iterating++; } static int -_exit_iter(PyObject *self) +_exit_iter(_guarded_set *gs) { - PyObject *ref; - _guarded_set *registry; - _abc_data *impl; - int pos; - impl = _get_impl(self); - if (!impl) { - return -1; - } - registry = impl->_abc_registry; - if (!registry) { - Py_DECREF(impl); - return -1; - } - registry->iterating--; - if (registry->iterating) { - Py_DECREF(impl); + gs->iterating--; + if (gs->iterating) { return 0; } - for (pos = 0; pos < PyList_GET_SIZE(registry->pending); pos++) { - ref = PyObject_CallMethod(registry->pending, "pop", NULL); - if (!ref) { - Py_DECREF(impl); - return -1; + + assert(PyList_CheckExact(gs->pending)); + Py_ssize_t pos = PyList_GET_SIZE(gs->pending); + while (pos > 0) { + pos--; + PyObject *ref = PyList_GET_ITEM(gs->pending, pos); + Py_INCREF(Py_None); + PyList_SET_ITEM(gs->pending, pos, Py_None); + + if (ref == Py_None) { + Py_DECREF(ref); + continue; } - if (PySet_Discard(registry->data, ref) < 0) { + if (PySet_Discard(gs->data, ref) < 0) { Py_DECREF(ref); - Py_DECREF(impl); return -1; } - Py_DECREF(ref); } - Py_DECREF(impl); - return 0; + return PyList_SetSlice(gs->pending, + 0, PyList_GET_SIZE(gs->pending), NULL); } static int @@ -370,17 +336,9 @@ _add_to_registry(PyObject *self, PyObject *cls) } static int -_add_to_cache(PyObject *self, PyObject *cls) +_add_to_cache(_abc_data *impl, PyObject *cls) { - _abc_data *impl; - int res; - impl = _get_impl(self); - if (!impl) { - return -1; - } - res = _add_to_weak_set(impl->_abc_cache, cls, 0); - Py_DECREF(impl); - return res; + return _add_to_weak_set(impl->_abc_cache, cls, 0); } static int @@ -420,19 +378,9 @@ _reset_registry(PyObject *m, PyObject *args) } static int -_reset_negative_cache(PyObject *self) +_reset_negative_cache(_abc_data *impl) { - _abc_data *impl; - impl = _get_impl(self); - if (!impl) { - return -1; - } - if (PySet_Clear(impl->_abc_negative_cache) < 0) { - Py_DECREF(impl); - return -1; - } - Py_DECREF(impl); - return 0; + return PySet_Clear(impl->_abc_negative_cache); } PyDoc_STRVAR(_reset_caches_doc, @@ -457,7 +405,7 @@ _reset_caches(PyObject *m, PyObject *args) return NULL; } /* also the second cache */ - if (_reset_negative_cache(self) < 0) { + if (_reset_negative_cache(impl) < 0) { Py_DECREF(impl); return NULL; } @@ -466,32 +414,16 @@ _reset_caches(PyObject *m, PyObject *args) } static PyObject * -_get_negative_cache_version(PyObject *self) +_get_negative_cache_version(_abc_data *impl) { - _abc_data *impl; - PyObject *res; - impl = _get_impl(self); - if (!impl) { - return NULL; - } - res = impl->_abc_negative_cache_version; - Py_DECREF(impl); - return res; + return impl->_abc_negative_cache_version; } -static int -_set_negative_cache_version(PyObject *self, PyObject *version) +static void +_set_negative_cache_version(_abc_data *impl) { - _abc_data *impl; - impl = _get_impl(self); - if (!impl) { - return -1; - } - Py_DECREF(impl->_abc_negative_cache_version); - impl->_abc_negative_cache_version = abc_invalidation_counter; - Py_INCREF(impl->_abc_negative_cache_version); - Py_DECREF(impl); - return 0; + Py_INCREF(abc_invalidation_counter); + Py_SETREF(impl->_abc_negative_cache_version, abc_invalidation_counter); } PyDoc_STRVAR(_get_dump_doc, @@ -505,7 +437,6 @@ static PyObject * _get_dump(PyObject *m, PyObject *args) { PyObject *self, *registry, *cache, *negative_cache, *cache_version; - PyObject *res = PyTuple_New(4); _abc_data *impl; if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { return NULL; @@ -532,22 +463,13 @@ _get_dump(PyObject *m, PyObject *args) Py_DECREF(impl); return NULL; } - cache_version = _get_negative_cache_version(self); + cache_version = _get_negative_cache_version(impl); if (!cache_version) { Py_DECREF(impl); return NULL; } - Py_INCREF(registry); - Py_INCREF(cache); - Py_INCREF(negative_cache); - Py_INCREF(cache_version); - if ((PyTuple_SetItem(res, 0, registry) < 0) || - (PyTuple_SetItem(res, 1, cache) < 0) || - (PyTuple_SetItem(res, 2, negative_cache) < 0) || - (PyTuple_SetItem(res, 3, cache_version) < 0)) { - Py_DECREF(impl); - return NULL; - } + PyObject *res = PyTuple_Pack(4, + registry, cache, negative_cache, cache_version); Py_DECREF(impl); return res; } @@ -667,9 +589,9 @@ compute_abstract_methods(PyObject *self) ret = 0; error: Py_DECREF(abstracts); - Py_CLEAR(ns); - Py_CLEAR(items); - Py_CLEAR(bases); + Py_XDECREF(ns); + Py_XDECREF(items); + Py_XDECREF(bases); return ret; } @@ -694,6 +616,7 @@ _abc_init(PyObject *m, PyObject *args) return NULL; } if (_PyObject_SetAttrId(self, &PyId__abc_impl, data) < 0) { + Py_DECREF(data); return NULL; } Py_DECREF(data); @@ -706,7 +629,7 @@ PyDoc_STRVAR(_abc_register_doc, static PyObject * _abc_register(PyObject *m, PyObject *args) { - PyObject *self, *one, *subclass = NULL; + PyObject *self, *subclass = NULL; int result; if (!PyArg_UnpackTuple(args, "_abc_register", 2, 2, &self, &subclass)) { return NULL; @@ -737,12 +660,21 @@ _abc_register(PyObject *m, PyObject *args) if (_add_to_registry(self, subclass) < 0) { return NULL; } - Py_INCREF(subclass); + /* Invalidate negative cache */ Py_DECREF(abc_invalidation_counter); - one = PyLong_FromLong(1); - abc_invalidation_counter = PyNumber_Add(abc_invalidation_counter, one); + PyObject *one = PyLong_FromLong(1); + if (one == NULL) { + return NULL; + } + PyObject *next_version = PyNumber_Add(abc_invalidation_counter, one); Py_DECREF(one); + if (next_version == NULL) { + return NULL; + } + Py_SETREF(abc_invalidation_counter, next_version); + + Py_INCREF(subclass); return subclass; } @@ -752,51 +684,60 @@ PyDoc_STRVAR(_abc_instancecheck_doc, static PyObject * _abc_instancecheck(PyObject *m, PyObject *args) { - PyObject *self, *result, *subclass, *subtype, *instance = NULL; - int incache; + PyObject *self, *result = NULL, *subclass = NULL, + *subtype, *instance = NULL; if (!PyArg_UnpackTuple(args, "_abc_instancecheck", 2, 2, &self, &instance)) { return NULL; } + + _abc_data *impl = _get_impl(self); + if (!impl) { + return NULL; + } + subclass = _PyObject_GetAttrId(instance, &PyId___class__); /* Inline the cache checking. */ - incache = _in_cache(self, subclass); + int incache = _in_cache(impl, subclass); if (incache < 0) { - Py_DECREF(subclass); - return NULL; + goto end; } if (incache > 0) { - Py_DECREF(subclass); - Py_RETURN_TRUE; + result = Py_True; + Py_INCREF(result); + goto end; } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { - incache = _in_negative_cache(self, subclass); - if (incache < 0) { - Py_DECREF(subclass); - return NULL; - } - if ((PyObject_RichCompareBool(_get_negative_cache_version(self), - abc_invalidation_counter, Py_EQ) > 0) && incache) { - Py_DECREF(subclass); - Py_RETURN_FALSE; + int r = PyObject_RichCompareBool( + _get_negative_cache_version(impl), + abc_invalidation_counter, Py_EQ); + assert(r >= 0); // Both should be PyLong + if (r > 0) { + incache = _in_negative_cache(impl, subclass); + if (incache < 0) { + goto end; + } + if (incache) { + result = Py_False; + Py_INCREF(result); + goto end; + } } /* Fall back to the subclass check. */ result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); - Py_DECREF(subclass); - return result; + goto end; } result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); - if (!result) { - Py_DECREF(subclass); - return NULL; - } - if (result == Py_True) { - Py_DECREF(subclass); - return Py_True; + if (!result || result == Py_True) { + goto end; } Py_DECREF(result); - Py_DECREF(subclass); - return PyObject_CallMethod(self, "__subclasscheck__", "O", subtype); + result = PyObject_CallMethod(self, "__subclasscheck__", "O", subtype); + +end: + Py_CLEAR(impl); + Py_CLEAR(subclass); + return result; } @@ -806,133 +747,152 @@ PyDoc_STRVAR(_abc_subclasscheck_doc, static PyObject * _abc_subclasscheck(PyObject *m, PyObject *args) { - PyObject *self, *subclasses, *subclass = NULL; + PyObject *self, *subclasses = NULL, *subclass = NULL, *result = NULL; PyObject *ok, *mro, *key, *rkey; Py_ssize_t pos; - int incache, result; + int incache; if (!PyArg_UnpackTuple(args, "_abc_subclasscheck", 2, 2, &self, &subclass)) { return NULL; } + _abc_data *impl = _get_impl(self); + if (!impl) { + return NULL; + } + /* 1. Check cache. */ - incache = _in_cache(self, subclass); + incache = _in_cache(impl, subclass); if (incache < 0) { - return NULL; + goto end; } if (incache > 0) { - Py_RETURN_TRUE; + result = Py_True; + goto end; } + /* 2. Check negative cache; may have to invalidate. */ - incache = _in_negative_cache(self, subclass); + incache = _in_negative_cache(impl, subclass); if (incache < 0) { - return NULL; + goto end; } - if (PyObject_RichCompareBool(_get_negative_cache_version(self), + if (PyObject_RichCompareBool(_get_negative_cache_version(impl), abc_invalidation_counter, Py_LT) > 0) { /* Invalidate the negative cache. */ - if (_reset_negative_cache(self) < 0) { - return NULL; - } - if (_set_negative_cache_version(self, abc_invalidation_counter) < 0) { - return NULL; + if (_reset_negative_cache(impl) < 0) { + goto end; } + _set_negative_cache_version(impl); } else if (incache) { - Py_RETURN_FALSE; + result = Py_False; + goto end; } + /* 3. Check the subclass hook. */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); if (!ok) { - return NULL; + goto end; } if (ok == Py_True) { - if (_add_to_cache(self, subclass) < 0) { - Py_DECREF(ok); - return NULL; + Py_DECREF(ok); + if (_add_to_cache(impl, subclass) < 0) { + goto end; } - return Py_True; + result = Py_True; + goto end; } if (ok == Py_False) { + Py_DECREF(ok); if (_add_to_negative_cache(self, subclass) < 0) { - Py_DECREF(ok); - return NULL; + goto end; } - return Py_False; + result = Py_False; + goto end; } if (ok != Py_NotImplemented) { Py_DECREF(ok); PyErr_SetString(PyExc_AssertionError, "__subclasshook__ must return either" " False, True, or NotImplemented"); - return NULL; + goto end; } Py_DECREF(ok); + /* 4. Check if it's a direct subclass. */ mro = ((PyTypeObject *)subclass)->tp_mro; + assert(PyTuple_Check(mro)); for (pos = 0; pos < PyTuple_Size(mro); pos++) { if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { - if (_add_to_cache(self, subclass) < 0) { - return NULL; + if (_add_to_cache(impl, subclass) < 0) { + goto end; } - Py_RETURN_TRUE; + result = Py_True; + goto end; } } /* 5. Check if it's a subclass of a registered class (recursive). */ pos = 0; Py_hash_t hash; - if (_enter_iter(self) < 0) { - return NULL; - } - while (_PySet_NextEntry(_get_registry(self), &pos, &key, &hash)) { + _enter_iter(impl->_abc_registry); + + while (_PySet_NextEntry(impl->_abc_registry->data, &pos, &key, &hash)) { rkey = PyWeakref_GetObject(key); if (rkey == Py_None) { continue; } - result = PyObject_IsSubclass(subclass, rkey); - if (result < 0) { - _exit_iter(self); - return NULL; + int r = PyObject_IsSubclass(subclass, rkey); + if (r < 0) { + _exit_iter(impl->_abc_registry); + goto end; } - if (result > 0) { - if (_add_to_cache(self, subclass) < 0) { - _exit_iter(self); - return NULL; + if (r > 0) { + if (_add_to_cache(impl, subclass) < 0) { + _exit_iter(impl->_abc_registry); + goto end; } - if (_exit_iter(self) < 0) { - return NULL; + if (_exit_iter(impl->_abc_registry) < 0) { + goto end; } - Py_RETURN_TRUE; + result = Py_True; + goto end; } } - if (_exit_iter(self) < 0) { - return NULL; + if (_exit_iter(impl->_abc_registry) < 0) { + goto end; } /* 6. Check if it's a subclass of a subclass (recursive). */ subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); if(!PyList_Check(subclasses)) { PyErr_SetString(PyExc_TypeError, "__subclasses__() must return a list"); - return NULL; + goto end; } for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { - result = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); - if (result > 0) { - if (_add_to_cache(self, subclass) < 0) { - return NULL; + int r = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); + if (r > 0) { + if (_add_to_cache(impl, subclass) < 0) { + goto end; } - Py_DECREF(subclasses); - Py_RETURN_TRUE; + result = Py_True; + goto end; } - if (result < 0) { - Py_DECREF(subclasses); - return NULL; + if (r < 0) { + goto end; } } - Py_DECREF(subclasses); + /* No dice; update negative cache. */ if (_add_to_negative_cache(self, subclass) < 0) { - return NULL; + goto end; + } + result = Py_False; + +end: + Py_XDECREF(impl); + Py_XDECREF(subclasses); + if (result != NULL) { + Py_INCREF(result); } - Py_RETURN_FALSE; + return result; } From 22699fe793eca9474dea7cf2ae4b3c38900d8044 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 28 Jan 2018 17:47:20 +0900 Subject: [PATCH 53/93] fixup --- Modules/_abc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index f08d6c549eb36a..b91f9f2b8ae39b 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -662,7 +662,6 @@ _abc_register(PyObject *m, PyObject *args) } /* Invalidate negative cache */ - Py_DECREF(abc_invalidation_counter); PyObject *one = PyLong_FromLong(1); if (one == NULL) { return NULL; From 95cbf34aad4f4cadd64c8f2d70f1a168b4596b49 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 14:45:06 +0000 Subject: [PATCH 54/93] Review comments --- Modules/_abc.c | 115 ++++++++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index b91f9f2b8ae39b..12d304cd76e1ff 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -56,12 +56,15 @@ gset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } _guarded_set *self = (_guarded_set *) type->tp_alloc(type, 0); - if (self != NULL) { - self->iterating = 0; - self->data = data; - self->pending = pending; - self->in_weakreflist = NULL; + if (self == NULL) { + Py_DECREF(data); + Py_DECREF(pending); + return NULL; } + self->iterating = 0; + self->data = data; + self->pending = pending; + self->in_weakreflist = NULL; return (PyObject *) self; } @@ -106,9 +109,18 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyObject *registry = NULL, *cache = NULL, *negative_cache = NULL; registry = gset_new(&_guarded_set_type, NULL, NULL); + if (registry == NULL) { + return NULL; + } cache = PySet_New(NULL); + if (cache == NULL) { + Py_DECREF(registry); + return NULL; + } negative_cache = PySet_New(NULL); - if (!registry || !cache || !negative_cache) { + if (negative_cache == NULL) { + Py_DECREF(registry); + Py_DECREF(cache); goto error; } @@ -125,9 +137,9 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *) self; error: - Py_XDECREF(registry); - Py_XDECREF(cache); - Py_XDECREF(negative_cache); + Py_DECREF(registry); + Py_DECREF(cache); + Py_DECREF(negative_cache); return NULL; } @@ -164,7 +176,7 @@ static _abc_data * _get_impl(PyObject *self) { PyObject *impl = _PyObject_GetAttrId(self, &PyId__abc_impl); - if (!impl) { + if (impl == NULL) { return NULL; } if (Py_TYPE(impl) != &_abc_data_type) { @@ -192,8 +204,9 @@ _destroy(PyObject *setweakref, PyObject *objweakref) { PyObject *set; set = PyWeakref_GET_OBJECT(setweakref); - if (set == Py_None) + if (set == Py_None) { Py_RETURN_NONE; + } Py_INCREF(set); if (PySet_Discard(set, objweakref) < 0) { Py_DECREF(set); @@ -212,7 +225,6 @@ _destroy_guarded(PyObject *setweakref, PyObject *objweakref) { PyObject *set; _guarded_set *gset; - int res; set = PyWeakref_GET_OBJECT(setweakref); if (set == Py_None) { Py_RETURN_NONE; @@ -220,8 +232,7 @@ _destroy_guarded(PyObject *setweakref, PyObject *objweakref) Py_INCREF(set); gset = (_guarded_set *)set; if (gset->iterating) { - res = PyList_Append(gset->pending, objweakref); - if (!res) { + if (PyList_Append(gset->pending, objweakref) < 0) { Py_DECREF(set); return NULL; } @@ -262,26 +273,20 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) if (guarded) { set = ((_guarded_set *) set)->data; } - if (PySet_Add(set, ref) < 0) { - Py_DECREF(wr); - Py_DECREF(ref); - return -1; - } + int ret = PySet_Add(set, ref); Py_DECREF(wr); Py_DECREF(ref); - return 0; + return ret; } static PyObject * _get_registry(PyObject *self) { - _abc_data *impl; - PyObject *res; - impl = _get_impl(self); - if (!impl) { + _abc_data *impl = _get_impl(self); + if (impl == NULL) { return NULL; } - res = impl->_abc_registry->data; + PyObject *res = impl->_abc_registry->data; Py_DECREF(impl); return res; } @@ -325,12 +330,11 @@ static int _add_to_registry(PyObject *self, PyObject *cls) { _abc_data *impl; - int res; impl = _get_impl(self); if (!impl) { return -1; } - res = _add_to_weak_set((PyObject *)(impl->_abc_registry), cls, 1); + int res = _add_to_weak_set((PyObject *)(impl->_abc_registry), cls, 1); Py_DECREF(impl); return res; } @@ -345,12 +349,11 @@ static int _add_to_negative_cache(PyObject *self, PyObject *cls) { _abc_data *impl; - int res; impl = _get_impl(self); if (!impl) { return -1; } - res = _add_to_weak_set(impl->_abc_negative_cache, cls, 0); + int res = _add_to_weak_set(impl->_abc_negative_cache, cls, 0); Py_DECREF(impl); return res; } @@ -368,7 +371,7 @@ _reset_registry(PyObject *m, PyObject *args) return NULL; } registry = _get_registry(self); - if (!registry) { + if (registry == NULL) { return NULL; } if (PySet_Clear(registry) < 0) { @@ -397,7 +400,7 @@ _reset_caches(PyObject *m, PyObject *args) return NULL; } impl = _get_impl(self); - if (!impl) { + if (impl == NULL) { return NULL; } if (PySet_Clear(impl->_abc_cache) < 0) { @@ -442,30 +445,37 @@ _get_dump(PyObject *m, PyObject *args) return NULL; } registry = _get_registry(self); - if (!registry) { + if (registry == NULL) { return NULL; } registry = PyObject_CallMethod(registry, "copy", NULL); - if (!registry) { + if (registry == NULL) { return NULL; } impl = _get_impl(self); - if (!impl) { + if (impl == NULL) { + Py_DECREF(registry); return NULL; } cache = PyObject_CallMethod(impl->_abc_cache, "copy", NULL); - if (!cache) { + if (cache == NULL) { + Py_DECREF(registry); Py_DECREF(impl); return NULL; } negative_cache = PyObject_CallMethod(impl->_abc_negative_cache, "copy", NULL); - if (!negative_cache) { + if (negative_cache == NULL) { + Py_DECREF(registry); Py_DECREF(impl); + Py_DECREF(cache); return NULL; } cache_version = _get_negative_cache_version(impl); - if (!cache_version) { + if (cache_version == NULL) { + Py_DECREF(registry); Py_DECREF(impl); + Py_DECREF(cache); + Py_DECREF(negative_cache); return NULL; } PyObject *res = PyTuple_Pack(4, @@ -484,7 +494,7 @@ compute_abstract_methods(PyObject *self) return -1; } - PyObject *ns=NULL, *items=NULL, *bases=NULL; // Py_CLEAR()ed on error. + PyObject *ns = NULL, *items = NULL, *bases = NULL; // Py_XDECREF()ed on error. /* Stage 1: direct abstract methods. */ ns = _PyObject_GetAttrId(self, &PyId___dict__); @@ -497,7 +507,7 @@ compute_abstract_methods(PyObject *self) if (!items) { goto error; } - + assert(PyList_Check(items)); for (Py_ssize_t pos = 0; pos < PyList_GET_SIZE(items); pos++) { PyObject *it = PySequence_Fast( PyList_GET_ITEM(items, pos), @@ -506,7 +516,7 @@ compute_abstract_methods(PyObject *self) goto error; } if (PySequence_Fast_GET_SIZE(it) != 2) { - PyErr_SetString(PyExc_TypeError, "items() returned not 2-tuple"); + PyErr_SetString(PyExc_TypeError, "items() returned not 2-tuple item"); Py_DECREF(it); goto error; } @@ -612,7 +622,7 @@ _abc_init(PyObject *m, PyObject *args) /* Set up inheritance registry. */ data = abc_data_new(&_abc_data_type, NULL, NULL); - if (!data) { + if (data == NULL) { return NULL; } if (_PyObject_SetAttrId(self, &PyId__abc_impl, data) < 0) { @@ -630,7 +640,6 @@ static PyObject * _abc_register(PyObject *m, PyObject *args) { PyObject *self, *subclass = NULL; - int result; if (!PyArg_UnpackTuple(args, "_abc_register", 2, 2, &self, &subclass)) { return NULL; } @@ -638,7 +647,7 @@ _abc_register(PyObject *m, PyObject *args) PyErr_SetString(PyExc_TypeError, "Can only register classes"); return NULL; } - result = PyObject_IsSubclass(subclass, self); + int result = PyObject_IsSubclass(subclass, self); if (result > 0) { Py_INCREF(subclass); return subclass; /* Already a subclass. */ @@ -690,7 +699,7 @@ _abc_instancecheck(PyObject *m, PyObject *args) } _abc_data *impl = _get_impl(self); - if (!impl) { + if (impl == NULL) { return NULL; } @@ -734,8 +743,8 @@ _abc_instancecheck(PyObject *m, PyObject *args) result = PyObject_CallMethod(self, "__subclasscheck__", "O", subtype); end: - Py_CLEAR(impl); - Py_CLEAR(subclass); + Py_XDECREF(impl); + Py_XDECREF(subclass); return result; } @@ -755,7 +764,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } _abc_data *impl = _get_impl(self); - if (!impl) { + if (impl == NULL) { return NULL; } @@ -774,8 +783,10 @@ _abc_subclasscheck(PyObject *m, PyObject *args) if (incache < 0) { goto end; } - if (PyObject_RichCompareBool(_get_negative_cache_version(impl), - abc_invalidation_counter, Py_LT) > 0) { + int r = PyObject_RichCompareBool(_get_negative_cache_version(impl), + abc_invalidation_counter, Py_LT); + assert(r >= 0); // Both should be PyLong + if (r > 0) { /* Invalidate the negative cache. */ if (_reset_negative_cache(impl) < 0) { goto end; @@ -819,7 +830,11 @@ _abc_subclasscheck(PyObject *m, PyObject *args) mro = ((PyTypeObject *)subclass)->tp_mro; assert(PyTuple_Check(mro)); for (pos = 0; pos < PyTuple_Size(mro); pos++) { - if ((PyObject *)self == PyTuple_GetItem(mro, pos)) { + PyObject *mro_item = PyTuple_GetItem(mro, pos); + if (mro_item == NULL) { + goto end; + } + if ((PyObject *)self == mro_item) { if (_add_to_cache(impl, subclass) < 0) { goto end; } From 4d596ccd805b22d9359e4a022317079596300f8d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 15:20:32 +0000 Subject: [PATCH 55/93] More refactoring --- Modules/_abc.c | 172 ++++++++++++++----------------------------------- 1 file changed, 49 insertions(+), 123 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 12d304cd76e1ff..51b581a14660b5 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -156,22 +156,6 @@ static PyTypeObject _abc_data_type = { .tp_new = abc_data_new, }; -static int -_in_weak_set(PyObject *set, PyObject *obj) -{ - PyObject *ref = PyWeakref_NewRef(obj, NULL); - if (!ref) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Clear(); - return 0; - } - return -1; - } - int res = PySet_Contains(set, ref); - Py_DECREF(ref); - return res; -} - static _abc_data * _get_impl(PyObject *self) { @@ -188,15 +172,19 @@ _get_impl(PyObject *self) } static int -_in_cache(_abc_data *impl, PyObject *cls) -{ - return _in_weak_set(impl->_abc_cache, cls); -} - -static int -_in_negative_cache(_abc_data *impl, PyObject *cls) +_in_weak_set(PyObject *set, PyObject *obj) { - return _in_weak_set(impl->_abc_negative_cache, cls); + PyObject *ref = PyWeakref_NewRef(obj, NULL); + if (!ref) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + int res = PySet_Contains(set, ref); + Py_DECREF(ref); + return res; } static PyObject * @@ -279,18 +267,6 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) return ret; } -static PyObject * -_get_registry(PyObject *self) -{ - _abc_data *impl = _get_impl(self); - if (impl == NULL) { - return NULL; - } - PyObject *res = impl->_abc_registry->data; - Py_DECREF(impl); - return res; -} - static void _enter_iter(_guarded_set *gs) { @@ -326,38 +302,6 @@ _exit_iter(_guarded_set *gs) 0, PyList_GET_SIZE(gs->pending), NULL); } -static int -_add_to_registry(PyObject *self, PyObject *cls) -{ - _abc_data *impl; - impl = _get_impl(self); - if (!impl) { - return -1; - } - int res = _add_to_weak_set((PyObject *)(impl->_abc_registry), cls, 1); - Py_DECREF(impl); - return res; -} - -static int -_add_to_cache(_abc_data *impl, PyObject *cls) -{ - return _add_to_weak_set(impl->_abc_cache, cls, 0); -} - -static int -_add_to_negative_cache(PyObject *self, PyObject *cls) -{ - _abc_data *impl; - impl = _get_impl(self); - if (!impl) { - return -1; - } - int res = _add_to_weak_set(impl->_abc_negative_cache, cls, 0); - Py_DECREF(impl); - return res; -} - PyDoc_STRVAR(_reset_registry_doc, "Internal ABC helper to reset registry of a given class.\n\ \n\ @@ -370,22 +314,19 @@ _reset_registry(PyObject *m, PyObject *args) if (!PyArg_UnpackTuple(args, "_reset_registry", 1, 1, &self)) { return NULL; } - registry = _get_registry(self); - if (registry == NULL) { + _abc_data *impl = _get_impl(self); + if (impl == NULL) { return NULL; } + *registry = impl->_abc_registry->data; if (PySet_Clear(registry) < 0) { + Py_DECREF(impl); return NULL; } + Py_DECREF(impl); Py_RETURN_NONE; } -static int -_reset_negative_cache(_abc_data *impl) -{ - return PySet_Clear(impl->_abc_negative_cache); -} - PyDoc_STRVAR(_reset_caches_doc, "Internal ABC helper to reset both caches of a given class.\n\ \n\ @@ -408,7 +349,7 @@ _reset_caches(PyObject *m, PyObject *args) return NULL; } /* also the second cache */ - if (_reset_negative_cache(impl) < 0) { + if (PySet_Clear(impl->_abc_negative_cache) < 0) { Py_DECREF(impl); return NULL; } @@ -416,19 +357,6 @@ _reset_caches(PyObject *m, PyObject *args) Py_RETURN_NONE; } -static PyObject * -_get_negative_cache_version(_abc_data *impl) -{ - return impl->_abc_negative_cache_version; -} - -static void -_set_negative_cache_version(_abc_data *impl) -{ - Py_INCREF(abc_invalidation_counter); - Py_SETREF(impl->_abc_negative_cache_version, abc_invalidation_counter); -} - PyDoc_STRVAR(_get_dump_doc, "Internal ABC helper for cache and registry debugging.\n\ \n\ @@ -444,42 +372,31 @@ _get_dump(PyObject *m, PyObject *args) if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { return NULL; } - registry = _get_registry(self); - if (registry == NULL) { + impl = _get_impl(self); + if (impl == NULL) { return NULL; } - registry = PyObject_CallMethod(registry, "copy", NULL); + registry = PyObject_CallMethod(impl->_abc_registry->data, "copy", NULL); if (registry == NULL) { - return NULL; - } - impl = _get_impl(self); - if (impl == NULL) { - Py_DECREF(registry); + Py_DECREF(impl); return NULL; } cache = PyObject_CallMethod(impl->_abc_cache, "copy", NULL); if (cache == NULL) { - Py_DECREF(registry); Py_DECREF(impl); + Py_DECREF(registry); return NULL; } negative_cache = PyObject_CallMethod(impl->_abc_negative_cache, "copy", NULL); if (negative_cache == NULL) { - Py_DECREF(registry); Py_DECREF(impl); - Py_DECREF(cache); - return NULL; - } - cache_version = _get_negative_cache_version(impl); - if (cache_version == NULL) { Py_DECREF(registry); - Py_DECREF(impl); Py_DECREF(cache); - Py_DECREF(negative_cache); return NULL; } + Py_INCREF(impl->_abc_negative_cache_version); /* PyTuple_Packdoesn't do this. */ PyObject *res = PyTuple_Pack(4, - registry, cache, negative_cache, cache_version); + registry, cache, negative_cache, impl->_abc_negative_cache_version); Py_DECREF(impl); return res; } @@ -666,9 +583,15 @@ _abc_register(PyObject *m, PyObject *args) if (result < 0) { return NULL; } - if (_add_to_registry(self, subclass) < 0) { + _abc_data *impl = _get_impl(self); + if (impl == NULL) { + return NULL; + } + if (_add_to_weak_set((PyObject *)(impl->_abc_registry), cls, 1) < 0) { + Py_DECREF(impl); return NULL; } + Py_DECREF(impl); /* Invalidate negative cache */ PyObject *one = PyLong_FromLong(1); @@ -705,7 +628,7 @@ _abc_instancecheck(PyObject *m, PyObject *args) subclass = _PyObject_GetAttrId(instance, &PyId___class__); /* Inline the cache checking. */ - int incache = _in_cache(impl, subclass); + int incache = _in_weak_set(impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -717,11 +640,11 @@ _abc_instancecheck(PyObject *m, PyObject *args) subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { int r = PyObject_RichCompareBool( - _get_negative_cache_version(impl), + impl->_abc_negative_cache_version, abc_invalidation_counter, Py_EQ); assert(r >= 0); // Both should be PyLong if (r > 0) { - incache = _in_negative_cache(impl, subclass); + incache = _in_weak_set(impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } @@ -769,7 +692,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* 1. Check cache. */ - incache = _in_cache(impl, subclass); + incache = _in_weak_set(impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -779,19 +702,22 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* 2. Check negative cache; may have to invalidate. */ - incache = _in_negative_cache(impl, subclass); + incache = _in_weak_set(impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } - int r = PyObject_RichCompareBool(_get_negative_cache_version(impl), + int r = PyObject_RichCompareBool(impl->_abc_negative_cache_version, abc_invalidation_counter, Py_LT); assert(r >= 0); // Both should be PyLong if (r > 0) { /* Invalidate the negative cache. */ - if (_reset_negative_cache(impl) < 0) { + if (PySet_Clear(impl->_abc_negative_cache) < 0) { goto end; } - _set_negative_cache_version(impl); + /* INCREF the new value of cache version, + then carefully DECREF the old one. */ + Py_INCREF(abc_invalidation_counter); + Py_SETREF(impl->_abc_negative_cache_version, abc_invalidation_counter); } else if (incache) { result = Py_False; goto end; @@ -804,7 +730,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } if (ok == Py_True) { Py_DECREF(ok); - if (_add_to_cache(impl, subclass) < 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { goto end; } result = Py_True; @@ -812,7 +738,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } if (ok == Py_False) { Py_DECREF(ok); - if (_add_to_negative_cache(self, subclass) < 0) { + if (_add_to_weak_set(impl->_abc_negative_cache, subclass, 0) < 0) { goto end; } result = Py_False; @@ -835,7 +761,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) goto end; } if ((PyObject *)self == mro_item) { - if (_add_to_cache(impl, subclass) < 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { goto end; } result = Py_True; @@ -859,7 +785,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) goto end; } if (r > 0) { - if (_add_to_cache(impl, subclass) < 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { _exit_iter(impl->_abc_registry); goto end; } @@ -883,7 +809,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { int r = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (r > 0) { - if (_add_to_cache(impl, subclass) < 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { goto end; } result = Py_True; @@ -895,7 +821,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* No dice; update negative cache. */ - if (_add_to_negative_cache(self, subclass) < 0) { + if (_add_to_weak_set(impl->_abc_negative_cache, subclass, 0) < 0) { goto end; } result = Py_False; From fa3cba3e6831d46c9686e8a382b304733cfe879c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 15:24:46 +0000 Subject: [PATCH 56/93] Typos and minor fixes --- Modules/_abc.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 51b581a14660b5..492f8ba4e36fe5 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -310,7 +310,7 @@ Should be only used by refleak.py"); static PyObject * _reset_registry(PyObject *m, PyObject *args) { - PyObject *self, *registry; + PyObject *self; if (!PyArg_UnpackTuple(args, "_reset_registry", 1, 1, &self)) { return NULL; } @@ -318,8 +318,7 @@ _reset_registry(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - *registry = impl->_abc_registry->data; - if (PySet_Clear(registry) < 0) { + if (PySet_Clear(impl->_abc_registry->data) < 0) { Py_DECREF(impl); return NULL; } @@ -367,7 +366,7 @@ instead use ABC._dump_registry() for a nice repr."); static PyObject * _get_dump(PyObject *m, PyObject *args) { - PyObject *self, *registry, *cache, *negative_cache, *cache_version; + PyObject *self, *registry, *cache, *negative_cache; _abc_data *impl; if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { return NULL; @@ -587,7 +586,7 @@ _abc_register(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - if (_add_to_weak_set((PyObject *)(impl->_abc_registry), cls, 1) < 0) { + if (_add_to_weak_set((PyObject *)(impl->_abc_registry), subclass, 1) < 0) { Py_DECREF(impl); return NULL; } From 6f18293403a52c8c6cf97f7add880acdd90b43d3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 15:47:52 +0000 Subject: [PATCH 57/93] Minor fixes and code style --- Modules/_abc.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 492f8ba4e36fe5..5fd3faad91f5fa 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -121,7 +121,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (negative_cache == NULL) { Py_DECREF(registry); Py_DECREF(cache); - goto error; + return NULL; } _abc_data *self = (_abc_data *) type->tp_alloc(type, 0); @@ -175,7 +175,7 @@ static int _in_weak_set(PyObject *set, PyObject *obj) { PyObject *ref = PyWeakref_NewRef(obj, NULL); - if (!ref) { + if (ref == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { PyErr_Clear(); return 0; @@ -244,7 +244,7 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) PyObject *ref, *wr; PyObject *destroy_cb; wr = PyWeakref_NewRef(set, NULL); - if (!wr) { + if (wr == NULL) { return -1; } if (guarded) { @@ -254,7 +254,7 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) } ref = PyWeakref_NewRef(obj, destroy_cb); Py_DECREF(destroy_cb); - if (!ref) { + if (ref == NULL) { Py_DECREF(wr); return -1; } @@ -297,9 +297,10 @@ _exit_iter(_guarded_set *gs) Py_DECREF(ref); return -1; } + Py_DECREF(ref); } return PyList_SetSlice(gs->pending, - 0, PyList_GET_SIZE(gs->pending), NULL); + 0, PyList_GET_SIZE(gs->pending), NULL); } PyDoc_STRVAR(_reset_registry_doc, @@ -647,7 +648,7 @@ _abc_instancecheck(PyObject *m, PyObject *args) if (incache < 0) { goto end; } - if (incache) { + if (incache > 0) { result = Py_False; Py_INCREF(result); goto end; @@ -658,7 +659,7 @@ _abc_instancecheck(PyObject *m, PyObject *args) goto end; } result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); - if (!result || result == Py_True) { + if (result == NULL || result == Py_True) { goto end; } Py_DECREF(result); @@ -717,14 +718,14 @@ _abc_subclasscheck(PyObject *m, PyObject *args) then carefully DECREF the old one. */ Py_INCREF(abc_invalidation_counter); Py_SETREF(impl->_abc_negative_cache_version, abc_invalidation_counter); - } else if (incache) { + } else if (incache > 0) { result = Py_False; goto end; } /* 3. Check the subclass hook. */ ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); - if (!ok) { + if (ok == NULL) { goto end; } if (ok == Py_True) { @@ -828,9 +829,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) end: Py_XDECREF(impl); Py_XDECREF(subclasses); - if (result != NULL) { - Py_INCREF(result); - } + Py_XINCREF(result); return result; } From 9100891a6a127402f409b37b90f8d4211ab22b0a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 15:50:26 +0000 Subject: [PATCH 58/93] Remove some unreachable code --- Modules/_abc.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 5fd3faad91f5fa..dff35805e6a957 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -288,11 +288,6 @@ _exit_iter(_guarded_set *gs) PyObject *ref = PyList_GET_ITEM(gs->pending, pos); Py_INCREF(Py_None); PyList_SET_ITEM(gs->pending, pos, Py_None); - - if (ref == Py_None) { - Py_DECREF(ref); - continue; - } if (PySet_Discard(gs->data, ref) < 0) { Py_DECREF(ref); return -1; From 36c564339d46712e7b55632ffb1e39891752d1f7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 17:55:29 +0000 Subject: [PATCH 59/93] Review comments --- Lib/abc.py | 12 +++++------- Modules/_abc.c | 27 +++++++++++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 3cdf063da1ac39..61dab50071a1e6 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -173,9 +173,6 @@ def register(cls, subclass): Returns the subclass, to allow usage as a class decorator. """ - if _C_speedup: - _abc_register(cls, subclass) - return subclass if not isinstance(subclass, type): raise TypeError("Can only register classes") if issubclass(subclass, cls): @@ -211,8 +208,6 @@ def _dump_registry(cls, file=None): def __instancecheck__(cls, instance): """Override for isinstance(instance, cls).""" - if _C_speedup: - return _abc_instancecheck(cls, instance) # Inline the cache checking subclass = instance.__class__ if subclass in cls._abc_cache: @@ -229,8 +224,6 @@ def __instancecheck__(cls, instance): def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" - if _C_speedup: - return _abc_subclasscheck(cls, subclass) # Check cache if subclass in cls._abc_cache: return True @@ -268,6 +261,11 @@ def __subclasscheck__(cls, subclass): cls._abc_negative_cache.add(subclass) return False +if _C_speedup: + ABCMeta.register = _abc_register + ABCMeta.__instancecheck__ = _abc_instancecheck + ABCMeta.__subclasscheck__ = _abc_subclasscheck + class ABC(metaclass=ABCMeta): """Helper class that provides a standard way to create an ABC using diff --git a/Modules/_abc.c b/Modules/_abc.c index dff35805e6a957..6c36952328316e 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,8 +1,7 @@ /* ABCMeta implementation */ -/* TODO: Think (ask) about thread-safety. */ -/* TODO: Add checks only to those calls that can fail and use _GET_SIZE etc. */ -/* TODO: Think about inlining some calls (like __subclasses__) and/or using macros */ +/* TODO: Test this hard in multithreaded context. */ +/* TODO: Think about inlining some calls and/or using macros */ /* TODO: Use separate branches with "fast paths" */ #include "Python.h" @@ -16,6 +15,8 @@ _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__bases__); _Py_IDENTIFIER(_abc_impl); +_Py_IDENTIFIER(__subclasscheck__); +_Py_IDENTIFIER(__subclasshook__); /* A global counter that is incremented each time a class is registered as a virtual subclass of anything. It forces the @@ -392,6 +393,12 @@ _get_dump(PyObject *m, PyObject *args) Py_INCREF(impl->_abc_negative_cache_version); /* PyTuple_Packdoesn't do this. */ PyObject *res = PyTuple_Pack(4, registry, cache, negative_cache, impl->_abc_negative_cache_version); + if (res == NULL) { + Py_DECREF(registry); + Py_DECREF(cache); + Py_DECREF(negative_cache); + Py_DECREF(impl->_abc_negative_cache_version); + } Py_DECREF(impl); return res; } @@ -402,7 +409,7 @@ compute_abstract_methods(PyObject *self) { int ret = -1; PyObject *abstracts = PyFrozenSet_New(NULL); - if (!abstracts) { + if (abstracts == NULL) { return -1; } @@ -650,15 +657,18 @@ _abc_instancecheck(PyObject *m, PyObject *args) } } /* Fall back to the subclass check. */ - result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); + result = _PyObject_CallMethodIdObjArgs(self, &PyId___subclasscheck__, + subclass, NULL); goto end; } - result = PyObject_CallMethod(self, "__subclasscheck__", "O", subclass); + result = _PyObject_CallMethodIdObjArgs(self, &PyId___subclasscheck__, + subclass, NULL); if (result == NULL || result == Py_True) { goto end; } Py_DECREF(result); - result = PyObject_CallMethod(self, "__subclasscheck__", "O", subtype); + result = _PyObject_CallMethodIdObjArgs(self, &PyId___subclasscheck__, + subtype, NULL); end: Py_XDECREF(impl); @@ -719,7 +729,8 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* 3. Check the subclass hook. */ - ok = PyObject_CallMethod((PyObject *)self, "__subclasshook__", "O", subclass); + ok = _PyObject_CallMethodIdObjArgs((PyObject *)self, &PyId___subclasshook__, + subclass, NULL); if (ok == NULL) { goto end; } From dd2abdaa077a6109f0352cb6bf3aab8d1c8fc176 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 18:44:30 +0000 Subject: [PATCH 60/93] Split the class into two separate implementations --- Lib/abc.py | 61 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 61dab50071a1e6..7aa41825b1182a 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -124,7 +124,7 @@ def setx(self, value): ... __isabstractmethod__ = True -class ABCMeta(type): +class _Py_ABCMeta(type): """Metaclass for defining Abstract Base Classes (ABCs). Use this metaclass to create an ABC. An ABC can be subclassed @@ -143,14 +143,10 @@ class ABCMeta(type): # negative cache to be cleared before its next use. # Note: this counter is private. Use `abc.get_cache_token()` for # external code. - if not _C_speedup: - _abc_invalidation_counter = 0 + _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace, **kwargs): cls = super().__new__(mcls, name, bases, namespace, **kwargs) - if _C_speedup: - _abc_init(cls) - return cls # Compute set of abstract method names abstracts = {name for name, value in namespace.items() @@ -190,15 +186,6 @@ def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) print(f"Inv. counter: {get_cache_token()}", file=file) - if _C_speedup: - (_abc_registry, _abc_cache, _abc_negative_cache, - _abc_negative_cache_version) = _get_dump(cls) - print(f"_abc_registry: {_abc_registry!r}", file=file) - print(f"_abc_cache: {_abc_cache!r}", file=file) - print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) - print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", - file=file) - return for name in cls.__dict__: if name.startswith("_abc_"): value = getattr(cls, name) @@ -261,10 +248,48 @@ def __subclasscheck__(cls, subclass): cls._abc_negative_cache.add(subclass) return False + +class _C_ABCMeta(type): + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + _abc_init(cls) + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + return _abc_register(cls, subclass) + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + return _abc_instancecheck(cls, instance) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + return _abc_subclasscheck(cls, subclass) + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + (_abc_registry, _abc_cache, _abc_negative_cache, + _abc_negative_cache_version) = _get_dump(cls) + print(f"_abc_registry: {_abc_registry!r}", file=file) + print(f"_abc_cache: {_abc_cache!r}", file=file) + print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) + print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", + file=file) + + if _C_speedup: - ABCMeta.register = _abc_register - ABCMeta.__instancecheck__ = _abc_instancecheck - ABCMeta.__subclasscheck__ = _abc_subclasscheck + ABCMeta = _C_ABCMeta +else: + ABCMeta = _Py_ABCMeta + +ABCMeta.__doc__ = _Py_ABCMeta.__doc__ +ABCMeta.__name__ = ABCMeta.__qualname__ = 'ABCMeta' class ABC(metaclass=ABCMeta): From 99d950c8babdad57568ab00083c505c82a09d035 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 18:55:40 +0000 Subject: [PATCH 61/93] Add version independent (Py vs C) cache clearing --- Lib/abc.py | 17 +++++++++++++++++ Lib/test/libregrtest/refleak.py | 4 ++-- Lib/test/test_typing.py | 5 ++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 7aa41825b1182a..4ad848f6b8a4b4 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -193,6 +193,15 @@ def _dump_registry(cls, file=None): value = set(value) print(f"{name}: {value!r}", file=file) + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + cls._abc_registry.clear() + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + cls._abc_cache.clear() + cls._abc_negative_cache.clear() + def __instancecheck__(cls, instance): """Override for isinstance(instance, cls).""" # Inline the cache checking @@ -282,6 +291,14 @@ def _dump_registry(cls, file=None): print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", file=file) + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + _reset_registry(cls) + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + _reset_caches(cls) + if _C_speedup: ABCMeta = _C_ABCMeta diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 961636e75cf930..765e06e20a11e0 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -5,7 +5,7 @@ import warnings from inspect import isabstract from test import support -from _abc import _reset_caches, _get_dump, _reset_registry +from _abc import _get_dump def dash_R(the_module, test, indirect_test, huntrleaks): @@ -140,7 +140,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): for ref in abcs.get(obj, set()): if ref() is not None: obj.register(ref()) - _reset_caches(obj) + obj._abc_caches_clear() clear_caches() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c7973f9f19801e..f56caa13a29594 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -22,7 +22,6 @@ from typing import IO, TextIO, BinaryIO from typing import Pattern, Match import abc -from _abc import _reset_registry, _reset_caches import typing import weakref @@ -762,8 +761,8 @@ class C(collections.abc.Mapping, Generic[T]): ... self.assertIsInstance(1, C) C[int] self.assertIsInstance(1, C) - _reset_registry(C) - _reset_caches(C) # To keep refleak hunting mode clean + C._abc_registry_clear() + C._abc_caches_clear() # To keep refleak hunting mode clean def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass From 3762d49a5326348e725e07fa5b7639a7f7687a25 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 19:02:41 +0000 Subject: [PATCH 62/93] Test the six-like tricky type.__new__(metaclass) with ABCMeta --- Lib/test/test_abc.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 61c2876708c256..1f680deb2116bd 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -405,6 +405,18 @@ class C(A, B): def test_ABC_has___slots__(self): self.assertTrue(hasattr(abc.ABC, '__slots__')) + def test_tricky_new_works(self): + def with_metaclass(meta, *bases): + class metaclass(type): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + class A: ... + class B: ... + class C(with_metaclass(abc.ABCMeta)): + pass + self.assertEqual(C.__class__, abc.ABCMeta) + class TestABCWithInitSubclass(unittest.TestCase): def test_works_with_init_subclass(self): From 404d1ce876f80336d5d3e174e0b98c87de38e70e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 20:35:44 +0000 Subject: [PATCH 63/93] Test both versions --- Lib/abc.py | 25 +- Lib/test/test_abc.py | 801 ++++++++++++++++++++++--------------------- 2 files changed, 416 insertions(+), 410 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 4ad848f6b8a4b4..00f69c20f63bc8 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -11,18 +11,19 @@ def get_cache_token(): current version of the ABC cache for virtual subclasses. The token changes with every call to ``register()`` on any ABC. """ - return ABCMeta._abc_invalidation_counter + return _Py_ABCMeta._abc_invalidation_counter -_C_speedup = False +from _weakrefset import WeakSet # Used by legacy _Py_ABCMeta + +_C_speedup = False try: from _abc import (get_cache_token, _abc_init, _abc_register, _abc_instancecheck, _abc_subclasscheck, _get_dump, _reset_registry, _reset_caches) except ImportError: - # We postpone this import to speed-up Python start-up time - from _weakrefset import WeakSet + pass else: _C_speedup = True @@ -161,7 +162,7 @@ def __new__(mcls, name, bases, namespace, **kwargs): cls._abc_registry = WeakSet() cls._abc_cache = WeakSet() cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + cls._abc_negative_cache_version = _Py_ABCMeta._abc_invalidation_counter return cls def register(cls, subclass): @@ -179,7 +180,7 @@ def register(cls, subclass): # This would create a cycle, which is bad for the algorithm below raise RuntimeError("Refusing to create an inheritance cycle") cls._abc_registry.add(subclass) - ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + _Py_ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache return subclass def _dump_registry(cls, file=None): @@ -211,7 +212,7 @@ def __instancecheck__(cls, instance): subtype = type(instance) if subtype is subclass: if (cls._abc_negative_cache_version == - ABCMeta._abc_invalidation_counter and + _Py_ABCMeta._abc_invalidation_counter and subclass in cls._abc_negative_cache): return False # Fall back to the subclass check. @@ -224,10 +225,10 @@ def __subclasscheck__(cls, subclass): if subclass in cls._abc_cache: return True # Check negative cache; may have to invalidate - if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + if cls._abc_negative_cache_version < _Py_ABCMeta._abc_invalidation_counter: # Invalidate the negative cache cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + cls._abc_negative_cache_version = _Py_ABCMeta._abc_invalidation_counter elif subclass in cls._abc_negative_cache: return False # Check the subclass hook @@ -305,8 +306,10 @@ def _abc_caches_clear(cls): else: ABCMeta = _Py_ABCMeta -ABCMeta.__doc__ = _Py_ABCMeta.__doc__ -ABCMeta.__name__ = ABCMeta.__qualname__ = 'ABCMeta' +_C_ABCMeta.__doc__ = _Py_ABCMeta.__doc__ + +_C_ABCMeta.__name__ = _C_ABCMeta.__qualname__ = 'ABCMeta' +_Py_ABCMeta.__name__ = _Py_ABCMeta.__qualname__ = 'ABCMeta' class ABC(metaclass=ABCMeta): diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 1f680deb2116bd..42fd211ab5fe55 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -8,427 +8,430 @@ import abc from inspect import isabstract +def test_factory(abc_ABCMeta): + class TestLegacyAPI(unittest.TestCase): -class TestLegacyAPI(unittest.TestCase): - - def test_abstractproperty_basics(self): - @abc.abstractproperty - def foo(self): pass - self.assertTrue(foo.__isabstractmethod__) - def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) - - class C(metaclass=abc.ABCMeta): + def test_abstractproperty_basics(self): @abc.abstractproperty - def foo(self): return 3 - self.assertRaises(TypeError, C) - class D(C): - @property - def foo(self): return super().foo - self.assertEqual(D().foo, 3) - self.assertFalse(getattr(D.foo, "__isabstractmethod__", False)) - - def test_abstractclassmethod_basics(self): - @abc.abstractclassmethod - def foo(cls): pass - self.assertTrue(foo.__isabstractmethod__) - @classmethod - def bar(cls): pass - self.assertFalse(getattr(bar, "__isabstractmethod__", False)) - - class C(metaclass=abc.ABCMeta): + def foo(self): pass + self.assertTrue(foo.__isabstractmethod__) + def bar(self): pass + self.assertFalse(hasattr(bar, "__isabstractmethod__")) + + class C(metaclass=abc_ABCMeta): + @abc.abstractproperty + def foo(self): return 3 + self.assertRaises(TypeError, C) + class D(C): + @property + def foo(self): return super().foo + self.assertEqual(D().foo, 3) + self.assertFalse(getattr(D.foo, "__isabstractmethod__", False)) + + def test_abstractclassmethod_basics(self): @abc.abstractclassmethod - def foo(cls): return cls.__name__ - self.assertRaises(TypeError, C) - class D(C): + def foo(cls): pass + self.assertTrue(foo.__isabstractmethod__) @classmethod - def foo(cls): return super().foo() - self.assertEqual(D.foo(), 'D') - self.assertEqual(D().foo(), 'D') - - def test_abstractstaticmethod_basics(self): - @abc.abstractstaticmethod - def foo(): pass - self.assertTrue(foo.__isabstractmethod__) - @staticmethod - def bar(): pass - self.assertFalse(getattr(bar, "__isabstractmethod__", False)) - - class C(metaclass=abc.ABCMeta): + def bar(cls): pass + self.assertFalse(getattr(bar, "__isabstractmethod__", False)) + + class C(metaclass=abc_ABCMeta): + @abc.abstractclassmethod + def foo(cls): return cls.__name__ + self.assertRaises(TypeError, C) + class D(C): + @classmethod + def foo(cls): return super().foo() + self.assertEqual(D.foo(), 'D') + self.assertEqual(D().foo(), 'D') + + def test_abstractstaticmethod_basics(self): @abc.abstractstaticmethod - def foo(): return 3 - self.assertRaises(TypeError, C) - class D(C): + def foo(): pass + self.assertTrue(foo.__isabstractmethod__) @staticmethod - def foo(): return 4 - self.assertEqual(D.foo(), 4) - self.assertEqual(D().foo(), 4) + def bar(): pass + self.assertFalse(getattr(bar, "__isabstractmethod__", False)) + class C(metaclass=abc_ABCMeta): + @abc.abstractstaticmethod + def foo(): return 3 + self.assertRaises(TypeError, C) + class D(C): + @staticmethod + def foo(): return 4 + self.assertEqual(D.foo(), 4) + self.assertEqual(D().foo(), 4) + + + class TestABC(unittest.TestCase): + + def test_ABC_helper(self): + # create an ABC using the helper class and perform basic checks + class C(abc.ABC): + @classmethod + @abc.abstractmethod + def foo(cls): return cls.__name__ + self.assertEqual(type(C), abc.ABCMeta) + self.assertRaises(TypeError, C) + class D(C): + @classmethod + def foo(cls): return super().foo() + self.assertEqual(D.foo(), 'D') -class TestABC(unittest.TestCase): - - def test_ABC_helper(self): - # create an ABC using the helper class and perform basic checks - class C(abc.ABC): - @classmethod + def test_abstractmethod_basics(self): @abc.abstractmethod - def foo(cls): return cls.__name__ - self.assertEqual(type(C), abc.ABCMeta) - self.assertRaises(TypeError, C) - class D(C): - @classmethod - def foo(cls): return super().foo() - self.assertEqual(D.foo(), 'D') - - def test_abstractmethod_basics(self): - @abc.abstractmethod - def foo(self): pass - self.assertTrue(foo.__isabstractmethod__) - def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) - - def test_abstractproperty_basics(self): - @property - @abc.abstractmethod - def foo(self): pass - self.assertTrue(foo.__isabstractmethod__) - def bar(self): pass - self.assertFalse(getattr(bar, "__isabstractmethod__", False)) - - class C(metaclass=abc.ABCMeta): + def foo(self): pass + self.assertTrue(foo.__isabstractmethod__) + def bar(self): pass + self.assertFalse(hasattr(bar, "__isabstractmethod__")) + + def test_abstractproperty_basics(self): @property @abc.abstractmethod - def foo(self): return 3 - self.assertRaises(TypeError, C) - class D(C): - @C.foo.getter - def foo(self): return super().foo - self.assertEqual(D().foo, 3) - - def test_abstractclassmethod_basics(self): - @classmethod - @abc.abstractmethod - def foo(cls): pass - self.assertTrue(foo.__isabstractmethod__) - @classmethod - def bar(cls): pass - self.assertFalse(getattr(bar, "__isabstractmethod__", False)) - - class C(metaclass=abc.ABCMeta): + def foo(self): pass + self.assertTrue(foo.__isabstractmethod__) + def bar(self): pass + self.assertFalse(getattr(bar, "__isabstractmethod__", False)) + + class C(metaclass=abc_ABCMeta): + @property + @abc.abstractmethod + def foo(self): return 3 + self.assertRaises(TypeError, C) + class D(C): + @C.foo.getter + def foo(self): return super().foo + self.assertEqual(D().foo, 3) + + def test_abstractclassmethod_basics(self): @classmethod @abc.abstractmethod - def foo(cls): return cls.__name__ - self.assertRaises(TypeError, C) - class D(C): + def foo(cls): pass + self.assertTrue(foo.__isabstractmethod__) @classmethod - def foo(cls): return super().foo() - self.assertEqual(D.foo(), 'D') - self.assertEqual(D().foo(), 'D') - - def test_abstractstaticmethod_basics(self): - @staticmethod - @abc.abstractmethod - def foo(): pass - self.assertTrue(foo.__isabstractmethod__) - @staticmethod - def bar(): pass - self.assertFalse(getattr(bar, "__isabstractmethod__", False)) - - class C(metaclass=abc.ABCMeta): + def bar(cls): pass + self.assertFalse(getattr(bar, "__isabstractmethod__", False)) + + class C(metaclass=abc_ABCMeta): + @classmethod + @abc.abstractmethod + def foo(cls): return cls.__name__ + self.assertRaises(TypeError, C) + class D(C): + @classmethod + def foo(cls): return super().foo() + self.assertEqual(D.foo(), 'D') + self.assertEqual(D().foo(), 'D') + + def test_abstractstaticmethod_basics(self): @staticmethod @abc.abstractmethod - def foo(): return 3 - self.assertRaises(TypeError, C) - class D(C): + def foo(): pass + self.assertTrue(foo.__isabstractmethod__) @staticmethod - def foo(): return 4 - self.assertEqual(D.foo(), 4) - self.assertEqual(D().foo(), 4) - - def test_abstractmethod_integration(self): - for abstractthing in [abc.abstractmethod, abc.abstractproperty, - abc.abstractclassmethod, - abc.abstractstaticmethod]: - class C(metaclass=abc.ABCMeta): - @abstractthing - def foo(self): pass # abstract - def bar(self): pass # concrete - self.assertEqual(C.__abstractmethods__, {"foo"}) - self.assertRaises(TypeError, C) # because foo is abstract - self.assertTrue(isabstract(C)) + def bar(): pass + self.assertFalse(getattr(bar, "__isabstractmethod__", False)) + + class C(metaclass=abc_ABCMeta): + @staticmethod + @abc.abstractmethod + def foo(): return 3 + self.assertRaises(TypeError, C) + class D(C): + @staticmethod + def foo(): return 4 + self.assertEqual(D.foo(), 4) + self.assertEqual(D().foo(), 4) + + def test_abstractmethod_integration(self): + for abstractthing in [abc.abstractmethod, abc.abstractproperty, + abc.abstractclassmethod, + abc.abstractstaticmethod]: + class C(metaclass=abc_ABCMeta): + @abstractthing + def foo(self): pass # abstract + def bar(self): pass # concrete + self.assertEqual(C.__abstractmethods__, {"foo"}) + self.assertRaises(TypeError, C) # because foo is abstract + self.assertTrue(isabstract(C)) + class D(C): + def bar(self): pass # concrete override of concrete + self.assertEqual(D.__abstractmethods__, {"foo"}) + self.assertRaises(TypeError, D) # because foo is still abstract + self.assertTrue(isabstract(D)) + class E(D): + def foo(self): pass + self.assertEqual(E.__abstractmethods__, set()) + E() # now foo is concrete, too + self.assertFalse(isabstract(E)) + class F(E): + @abstractthing + def bar(self): pass # abstract override of concrete + self.assertEqual(F.__abstractmethods__, {"bar"}) + self.assertRaises(TypeError, F) # because bar is abstract now + self.assertTrue(isabstract(F)) + + def test_descriptors_with_abstractmethod(self): + class C(metaclass=abc_ABCMeta): + @property + @abc.abstractmethod + def foo(self): return 3 + @foo.setter + @abc.abstractmethod + def foo(self, val): pass + self.assertRaises(TypeError, C) class D(C): - def bar(self): pass # concrete override of concrete - self.assertEqual(D.__abstractmethods__, {"foo"}) - self.assertRaises(TypeError, D) # because foo is still abstract - self.assertTrue(isabstract(D)) + @C.foo.getter + def foo(self): return super().foo + self.assertRaises(TypeError, D) class E(D): - def foo(self): pass - self.assertEqual(E.__abstractmethods__, set()) - E() # now foo is concrete, too - self.assertFalse(isabstract(E)) - class F(E): - @abstractthing - def bar(self): pass # abstract override of concrete - self.assertEqual(F.__abstractmethods__, {"bar"}) - self.assertRaises(TypeError, F) # because bar is abstract now - self.assertTrue(isabstract(F)) - - def test_descriptors_with_abstractmethod(self): - class C(metaclass=abc.ABCMeta): - @property - @abc.abstractmethod - def foo(self): return 3 - @foo.setter - @abc.abstractmethod - def foo(self, val): pass - self.assertRaises(TypeError, C) - class D(C): - @C.foo.getter - def foo(self): return super().foo - self.assertRaises(TypeError, D) - class E(D): - @D.foo.setter - def foo(self, val): pass - self.assertEqual(E().foo, 3) - # check that the property's __isabstractmethod__ descriptor does the - # right thing when presented with a value that fails truth testing: - class NotBool(object): - def __bool__(self): - raise ValueError() - __len__ = __bool__ - with self.assertRaises(ValueError): - class F(C): - def bar(self): + @D.foo.setter + def foo(self, val): pass + self.assertEqual(E().foo, 3) + # check that the property's __isabstractmethod__ descriptor does the + # right thing when presented with a value that fails truth testing: + class NotBool(object): + def __bool__(self): + raise ValueError() + __len__ = __bool__ + with self.assertRaises(ValueError): + class F(C): + def bar(self): + pass + bar.__isabstractmethod__ = NotBool() + foo = property(bar) + + + def test_customdescriptors_with_abstractmethod(self): + class Descriptor: + def __init__(self, fget, fset=None): + self._fget = fget + self._fset = fset + def getter(self, callable): + return Descriptor(callable, self._fget) + def setter(self, callable): + return Descriptor(self._fget, callable) + @property + def __isabstractmethod__(self): + return (getattr(self._fget, '__isabstractmethod__', False) + or getattr(self._fset, '__isabstractmethod__', False)) + class C(metaclass=abc_ABCMeta): + @Descriptor + @abc.abstractmethod + def foo(self): return 3 + @foo.setter + @abc.abstractmethod + def foo(self, val): pass + self.assertRaises(TypeError, C) + class D(C): + @C.foo.getter + def foo(self): return super().foo + self.assertRaises(TypeError, D) + class E(D): + @D.foo.setter + def foo(self, val): pass + self.assertFalse(E.foo.__isabstractmethod__) + + def test_metaclass_abc(self): + # Metaclasses can be ABCs, too. + class A(metaclass=abc_ABCMeta): + @abc.abstractmethod + def x(self): pass - bar.__isabstractmethod__ = NotBool() - foo = property(bar) - - - def test_customdescriptors_with_abstractmethod(self): - class Descriptor: - def __init__(self, fget, fset=None): - self._fget = fget - self._fset = fset - def getter(self, callable): - return Descriptor(callable, self._fget) - def setter(self, callable): - return Descriptor(self._fget, callable) - @property - def __isabstractmethod__(self): - return (getattr(self._fget, '__isabstractmethod__', False) - or getattr(self._fset, '__isabstractmethod__', False)) - class C(metaclass=abc.ABCMeta): - @Descriptor - @abc.abstractmethod - def foo(self): return 3 - @foo.setter - @abc.abstractmethod - def foo(self, val): pass - self.assertRaises(TypeError, C) - class D(C): - @C.foo.getter - def foo(self): return super().foo - self.assertRaises(TypeError, D) - class E(D): - @D.foo.setter - def foo(self, val): pass - self.assertFalse(E.foo.__isabstractmethod__) - - def test_metaclass_abc(self): - # Metaclasses can be ABCs, too. - class A(metaclass=abc.ABCMeta): - @abc.abstractmethod - def x(self): + self.assertEqual(A.__abstractmethods__, {"x"}) + class meta(type, A): + def x(self): + return 1 + class C(metaclass=meta): + pass + + def test_registration_basics(self): + class A(metaclass=abc_ABCMeta): + pass + class B(object): + pass + b = B() + self.assertFalse(issubclass(B, A)) + self.assertFalse(issubclass(B, (A,))) + self.assertNotIsInstance(b, A) + self.assertNotIsInstance(b, (A,)) + B1 = A.register(B) + self.assertTrue(issubclass(B, A)) + self.assertTrue(issubclass(B, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) + self.assertIs(B1, B) + class C(B): + pass + c = C() + self.assertTrue(issubclass(C, A)) + self.assertTrue(issubclass(C, (A,))) + self.assertIsInstance(c, A) + self.assertIsInstance(c, (A,)) + + def test_register_as_class_deco(self): + class A(metaclass=abc_ABCMeta): + pass + @A.register + class B(object): + pass + b = B() + self.assertTrue(issubclass(B, A)) + self.assertTrue(issubclass(B, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) + @A.register + class C(B): + pass + c = C() + self.assertTrue(issubclass(C, A)) + self.assertTrue(issubclass(C, (A,))) + self.assertIsInstance(c, A) + self.assertIsInstance(c, (A,)) + self.assertIs(C, A.register(C)) + + def test_isinstance_invalidation(self): + class A(metaclass=abc.ABCMeta): + pass + class B: + pass + b = B() + self.assertFalse(isinstance(b, A)) + self.assertFalse(isinstance(b, (A,))) + token_old = abc.get_cache_token() + A.register(B) + token_new = abc.get_cache_token() + self.assertNotEqual(token_old, token_new) + self.assertTrue(isinstance(b, A)) + self.assertTrue(isinstance(b, (A,))) + + def test_registration_builtins(self): + class A(metaclass=abc_ABCMeta): + pass + A.register(int) + self.assertIsInstance(42, A) + self.assertIsInstance(42, (A,)) + self.assertTrue(issubclass(int, A)) + self.assertTrue(issubclass(int, (A,))) + class B(A): + pass + B.register(str) + class C(str): pass + self.assertIsInstance("", A) + self.assertIsInstance("", (A,)) + self.assertTrue(issubclass(str, A)) + self.assertTrue(issubclass(str, (A,))) + self.assertTrue(issubclass(C, A)) + self.assertTrue(issubclass(C, (A,))) + + def test_registration_edge_cases(self): + class A(metaclass=abc_ABCMeta): + pass + A.register(A) # should pass silently + class A1(A): + pass + self.assertRaises(RuntimeError, A1.register, A) # cycles not allowed + class B(object): + pass + A1.register(B) # ok + A1.register(B) # should pass silently + class C(A): + pass + A.register(C) # should pass silently + self.assertRaises(RuntimeError, C.register, A) # cycles not allowed + C.register(B) # ok + + def test_register_non_class(self): + class A(metaclass=abc_ABCMeta): + pass + self.assertRaisesRegex(TypeError, "Can only register classes", + A.register, 4) + + def test_registration_transitiveness(self): + class A(metaclass=abc_ABCMeta): + pass + self.assertTrue(issubclass(A, A)) + self.assertTrue(issubclass(A, (A,))) + class B(metaclass=abc_ABCMeta): + pass + self.assertFalse(issubclass(A, B)) + self.assertFalse(issubclass(A, (B,))) + self.assertFalse(issubclass(B, A)) + self.assertFalse(issubclass(B, (A,))) + class C(metaclass=abc_ABCMeta): + pass + A.register(B) + class B1(B): + pass + self.assertTrue(issubclass(B1, A)) + self.assertTrue(issubclass(B1, (A,))) + class C1(C): + pass + B1.register(C1) + self.assertFalse(issubclass(C, B)) + self.assertFalse(issubclass(C, (B,))) + self.assertFalse(issubclass(C, B1)) + self.assertFalse(issubclass(C, (B1,))) + self.assertTrue(issubclass(C1, A)) + self.assertTrue(issubclass(C1, (A,))) + self.assertTrue(issubclass(C1, B)) + self.assertTrue(issubclass(C1, (B,))) + self.assertTrue(issubclass(C1, B1)) + self.assertTrue(issubclass(C1, (B1,))) + C1.register(int) + class MyInt(int): + pass + self.assertTrue(issubclass(MyInt, A)) + self.assertTrue(issubclass(MyInt, (A,))) + self.assertIsInstance(42, A) + self.assertIsInstance(42, (A,)) + + def test_all_new_methods_are_called(self): + class A(metaclass=abc_ABCMeta): + pass + class B(object): + counter = 0 + def __new__(cls): + B.counter += 1 + return super().__new__(cls) + class C(A, B): + pass + self.assertEqual(B.counter, 0) + C() + self.assertEqual(B.counter, 1) + + def test_ABC_has___slots__(self): + self.assertTrue(hasattr(abc.ABC, '__slots__')) + + def test_tricky_new_works(self): + def with_metaclass(meta, *bases): + class metaclass(type): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + class A: ... + class B: ... + class C(with_metaclass(abc_ABCMeta, A, B)): + pass + self.assertEqual(C.__class__, abc_ABCMeta) + + + class TestABCWithInitSubclass(unittest.TestCase): + def test_works_with_init_subclass(self): + saved_kwargs = {} + class ReceivesClassKwargs: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__() + saved_kwargs.update(kwargs) + class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3): pass - self.assertEqual(A.__abstractmethods__, {"x"}) - class meta(type, A): - def x(self): - return 1 - class C(metaclass=meta): - pass - - def test_registration_basics(self): - class A(metaclass=abc.ABCMeta): - pass - class B(object): - pass - b = B() - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) - self.assertNotIsInstance(b, A) - self.assertNotIsInstance(b, (A,)) - B1 = A.register(B) - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) - self.assertIsInstance(b, A) - self.assertIsInstance(b, (A,)) - self.assertIs(B1, B) - class C(B): - pass - c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) - self.assertIsInstance(c, A) - self.assertIsInstance(c, (A,)) - - def test_register_as_class_deco(self): - class A(metaclass=abc.ABCMeta): - pass - @A.register - class B(object): - pass - b = B() - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) - self.assertIsInstance(b, A) - self.assertIsInstance(b, (A,)) - @A.register - class C(B): - pass - c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) - self.assertIsInstance(c, A) - self.assertIsInstance(c, (A,)) - self.assertIs(C, A.register(C)) - - def test_isinstance_invalidation(self): - class A(metaclass=abc.ABCMeta): - pass - class B: - pass - b = B() - self.assertFalse(isinstance(b, A)) - self.assertFalse(isinstance(b, (A,))) - token_old = abc.get_cache_token() - A.register(B) - token_new = abc.get_cache_token() - self.assertNotEqual(token_old, token_new) - self.assertTrue(isinstance(b, A)) - self.assertTrue(isinstance(b, (A,))) - - def test_registration_builtins(self): - class A(metaclass=abc.ABCMeta): - pass - A.register(int) - self.assertIsInstance(42, A) - self.assertIsInstance(42, (A,)) - self.assertTrue(issubclass(int, A)) - self.assertTrue(issubclass(int, (A,))) - class B(A): - pass - B.register(str) - class C(str): pass - self.assertIsInstance("", A) - self.assertIsInstance("", (A,)) - self.assertTrue(issubclass(str, A)) - self.assertTrue(issubclass(str, (A,))) - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) - - def test_registration_edge_cases(self): - class A(metaclass=abc.ABCMeta): - pass - A.register(A) # should pass silently - class A1(A): - pass - self.assertRaises(RuntimeError, A1.register, A) # cycles not allowed - class B(object): - pass - A1.register(B) # ok - A1.register(B) # should pass silently - class C(A): - pass - A.register(C) # should pass silently - self.assertRaises(RuntimeError, C.register, A) # cycles not allowed - C.register(B) # ok - - def test_register_non_class(self): - class A(metaclass=abc.ABCMeta): - pass - self.assertRaisesRegex(TypeError, "Can only register classes", - A.register, 4) - - def test_registration_transitiveness(self): - class A(metaclass=abc.ABCMeta): - pass - self.assertTrue(issubclass(A, A)) - self.assertTrue(issubclass(A, (A,))) - class B(metaclass=abc.ABCMeta): - pass - self.assertFalse(issubclass(A, B)) - self.assertFalse(issubclass(A, (B,))) - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) - class C(metaclass=abc.ABCMeta): - pass - A.register(B) - class B1(B): - pass - self.assertTrue(issubclass(B1, A)) - self.assertTrue(issubclass(B1, (A,))) - class C1(C): - pass - B1.register(C1) - self.assertFalse(issubclass(C, B)) - self.assertFalse(issubclass(C, (B,))) - self.assertFalse(issubclass(C, B1)) - self.assertFalse(issubclass(C, (B1,))) - self.assertTrue(issubclass(C1, A)) - self.assertTrue(issubclass(C1, (A,))) - self.assertTrue(issubclass(C1, B)) - self.assertTrue(issubclass(C1, (B,))) - self.assertTrue(issubclass(C1, B1)) - self.assertTrue(issubclass(C1, (B1,))) - C1.register(int) - class MyInt(int): - pass - self.assertTrue(issubclass(MyInt, A)) - self.assertTrue(issubclass(MyInt, (A,))) - self.assertIsInstance(42, A) - self.assertIsInstance(42, (A,)) - - def test_all_new_methods_are_called(self): - class A(metaclass=abc.ABCMeta): - pass - class B(object): - counter = 0 - def __new__(cls): - B.counter += 1 - return super().__new__(cls) - class C(A, B): - pass - self.assertEqual(B.counter, 0) - C() - self.assertEqual(B.counter, 1) - - def test_ABC_has___slots__(self): - self.assertTrue(hasattr(abc.ABC, '__slots__')) - - def test_tricky_new_works(self): - def with_metaclass(meta, *bases): - class metaclass(type): - def __new__(cls, name, this_bases, d): - return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) - class A: ... - class B: ... - class C(with_metaclass(abc.ABCMeta)): - pass - self.assertEqual(C.__class__, abc.ABCMeta) - - -class TestABCWithInitSubclass(unittest.TestCase): - def test_works_with_init_subclass(self): - saved_kwargs = {} - class ReceivesClassKwargs: - def __init_subclass__(cls, **kwargs): - super().__init_subclass__() - saved_kwargs.update(kwargs) - class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3): - pass - self.assertEqual(saved_kwargs, dict(x=1, y=2, z=3)) + self.assertEqual(saved_kwargs, dict(x=1, y=2, z=3)) + return TestLegacyAPI, TestABC, TestABCWithInitSubclass +TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(abc._Py_ABCMeta) +TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(abc._C_ABCMeta) if __name__ == "__main__": unittest.main() From 0dc5faea8c9a71c37b43629069525b599a8bf018 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 20:41:35 +0000 Subject: [PATCH 64/93] Add comment about testing --- Lib/test/test_abc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 42fd211ab5fe55..12ea47836830a3 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -1,6 +1,9 @@ # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. +# Note: each test is run with _Py_ABCMeta and _C_ABCMeta, except for two tests, where we +# need to "sync" with actual abc.ABCMeta, these tests check get_cache_token and ABC helper. + """Unit tests for abc.py.""" import unittest From 287b26abf2c8e7969ead388707f5cc5fd3cad655 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Jan 2018 21:00:32 +0000 Subject: [PATCH 65/93] Always DECREF after PyTuplr_Pack --- Modules/_abc.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 6c36952328316e..f42df7ed54926a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -393,12 +393,10 @@ _get_dump(PyObject *m, PyObject *args) Py_INCREF(impl->_abc_negative_cache_version); /* PyTuple_Packdoesn't do this. */ PyObject *res = PyTuple_Pack(4, registry, cache, negative_cache, impl->_abc_negative_cache_version); - if (res == NULL) { - Py_DECREF(registry); - Py_DECREF(cache); - Py_DECREF(negative_cache); - Py_DECREF(impl->_abc_negative_cache_version); - } + Py_DECREF(registry); + Py_DECREF(cache); + Py_DECREF(negative_cache); + Py_DECREF(impl->_abc_negative_cache_version); Py_DECREF(impl); return res; } From ef34364092365982b11657ea0e363473582e24fc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 29 Jan 2018 16:28:42 +0900 Subject: [PATCH 66/93] Use PySet_New() instead of calling copy() method. --- Modules/_abc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index f42df7ed54926a..357353b70baaa6 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -372,18 +372,18 @@ _get_dump(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - registry = PyObject_CallMethod(impl->_abc_registry->data, "copy", NULL); + registry = PySet_New(impl->_abc_registry->data); if (registry == NULL) { Py_DECREF(impl); return NULL; } - cache = PyObject_CallMethod(impl->_abc_cache, "copy", NULL); + cache = PySet_New(impl->_abc_cache); if (cache == NULL) { Py_DECREF(impl); Py_DECREF(registry); return NULL; } - negative_cache = PyObject_CallMethod(impl->_abc_negative_cache, "copy", NULL); + negative_cache = PySet_New(impl->_abc_negative_cache); if (negative_cache == NULL) { Py_DECREF(impl); Py_DECREF(registry); From d4d78a1a48487923ab0e2bc964437d2d55b2e5a2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 29 Jan 2018 15:02:57 +0900 Subject: [PATCH 67/93] Copy set before iterating, and remove guarded set --- Modules/_abc.c | 250 +++++++++++++++---------------------------------- 1 file changed, 78 insertions(+), 172 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 357353b70baaa6..03486bd35135f9 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -25,71 +25,12 @@ _Py_IDENTIFIER(__subclasshook__); external code. */ static PyObject *abc_invalidation_counter; -typedef struct { - PyObject_HEAD - Py_ssize_t iterating; - PyObject *data; /* The actual set of weak references. */ - PyObject *pending; /* Pending removals collected during iteration. */ - PyObject *in_weakreflist; -} _guarded_set; - -static void -gset_dealloc(_guarded_set *self) -{ - Py_DECREF(self->data); - Py_DECREF(self->pending); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); - } - Py_TYPE(self)->tp_free(self); -} - -static PyObject * -gset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyObject *data = PySet_New(NULL); - if (data == NULL) { - return NULL; - } - PyObject *pending = PyList_New(0); - if (pending == NULL) { - Py_DECREF(data); - return NULL; - } - _guarded_set *self = (_guarded_set *) type->tp_alloc(type, 0); - if (self == NULL) { - Py_DECREF(data); - Py_DECREF(pending); - return NULL; - } - self->iterating = 0; - self->data = data; - self->pending = pending; - self->in_weakreflist = NULL; - return (PyObject *) self; -} - -PyDoc_STRVAR(guarded_set_doc, -"Internal weak set guarded against deletion during iteration.\n\ -Used by ABC machinery."); - -static PyTypeObject _guarded_set_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_guarded_set", /*tp_name*/ - sizeof(_guarded_set), /*tp_size*/ - .tp_dealloc = (destructor)gset_dealloc, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_weaklistoffset = offsetof(_guarded_set, in_weakreflist), - .tp_alloc = PyType_GenericAlloc, - .tp_new = gset_new, -}; - /* This object stores internal state for ABCs. Note that we can use normal sets for caches, since they are never iterated over. */ typedef struct { PyObject_HEAD - _guarded_set *_abc_registry; + PyObject *_abc_registry; PyObject *_abc_cache; /* Normal set of weak references. */ PyObject *_abc_negative_cache; /* Normal set of weak references. */ PyObject *_abc_negative_cache_version; @@ -109,7 +50,7 @@ static PyObject * abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyObject *registry = NULL, *cache = NULL, *negative_cache = NULL; - registry = gset_new(&_guarded_set_type, NULL, NULL); + registry = PySet_New(NULL); if (registry == NULL) { return NULL; } @@ -130,7 +71,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - self->_abc_registry = (_guarded_set *)registry; + self->_abc_registry = registry; self->_abc_cache = cache; self->_abc_negative_cache = negative_cache; self->_abc_negative_cache_version = abc_invalidation_counter; @@ -209,38 +150,8 @@ static PyMethodDef _destroy_def = { "_destroy", (PyCFunction) _destroy, METH_O }; -static PyObject * -_destroy_guarded(PyObject *setweakref, PyObject *objweakref) -{ - PyObject *set; - _guarded_set *gset; - set = PyWeakref_GET_OBJECT(setweakref); - if (set == Py_None) { - Py_RETURN_NONE; - } - Py_INCREF(set); - gset = (_guarded_set *)set; - if (gset->iterating) { - if (PyList_Append(gset->pending, objweakref) < 0) { - Py_DECREF(set); - return NULL; - } - } else { - if (PySet_Discard(gset->data, objweakref) < 0) { - Py_DECREF(set); - return NULL; - } - } - Py_DECREF(set); - Py_RETURN_NONE; -} - -static PyMethodDef _destroy_guarded_def = { - "_destroy_guarded", (PyCFunction) _destroy_guarded, METH_O -}; - static int -_add_to_weak_set(PyObject *set, PyObject *obj, int guarded) +_add_to_weak_set(PyObject *set, PyObject *obj) { PyObject *ref, *wr; PyObject *destroy_cb; @@ -248,56 +159,19 @@ _add_to_weak_set(PyObject *set, PyObject *obj, int guarded) if (wr == NULL) { return -1; } - if (guarded) { - destroy_cb = PyCFunction_NewEx(&_destroy_guarded_def, wr, NULL); - } else { - destroy_cb = PyCFunction_NewEx(&_destroy_def, wr, NULL); - } + destroy_cb = PyCFunction_NewEx(&_destroy_def, wr, NULL); ref = PyWeakref_NewRef(obj, destroy_cb); Py_DECREF(destroy_cb); if (ref == NULL) { Py_DECREF(wr); return -1; } - if (guarded) { - set = ((_guarded_set *) set)->data; - } int ret = PySet_Add(set, ref); Py_DECREF(wr); Py_DECREF(ref); return ret; } -static void -_enter_iter(_guarded_set *gs) -{ - gs->iterating++; -} - -static int -_exit_iter(_guarded_set *gs) -{ - gs->iterating--; - if (gs->iterating) { - return 0; - } - - assert(PyList_CheckExact(gs->pending)); - Py_ssize_t pos = PyList_GET_SIZE(gs->pending); - while (pos > 0) { - pos--; - PyObject *ref = PyList_GET_ITEM(gs->pending, pos); - Py_INCREF(Py_None); - PyList_SET_ITEM(gs->pending, pos, Py_None); - if (PySet_Discard(gs->data, ref) < 0) { - Py_DECREF(ref); - return -1; - } - Py_DECREF(ref); - } - return PyList_SetSlice(gs->pending, - 0, PyList_GET_SIZE(gs->pending), NULL); -} PyDoc_STRVAR(_reset_registry_doc, "Internal ABC helper to reset registry of a given class.\n\ @@ -315,7 +189,7 @@ _reset_registry(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - if (PySet_Clear(impl->_abc_registry->data) < 0) { + if (PySet_Clear(impl->_abc_registry) < 0) { Py_DECREF(impl); return NULL; } @@ -372,7 +246,7 @@ _get_dump(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - registry = PySet_New(impl->_abc_registry->data); + registry = PySet_New(impl->_abc_registry); if (registry == NULL) { Py_DECREF(impl); return NULL; @@ -587,7 +461,7 @@ _abc_register(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - if (_add_to_weak_set((PyObject *)(impl->_abc_registry), subclass, 1) < 0) { + if (_add_to_weak_set(impl->_abc_registry, subclass) < 0) { Py_DECREF(impl); return NULL; } @@ -675,6 +549,12 @@ _abc_instancecheck(PyObject *m, PyObject *args) } +// Return -1 when exception occured. +// Return 1 when result is set. +// Return 0 otherwise. +static int subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, + PyObject **result); + PyDoc_STRVAR(_abc_subclasscheck_doc, "Internal ABC helper for subclasss checks. Should be never used outside abc module"); @@ -682,7 +562,7 @@ static PyObject * _abc_subclasscheck(PyObject *m, PyObject *args) { PyObject *self, *subclasses = NULL, *subclass = NULL, *result = NULL; - PyObject *ok, *mro, *key, *rkey; + PyObject *ok, *mro; Py_ssize_t pos; int incache; if (!PyArg_UnpackTuple(args, "_abc_subclasscheck", 2, 2, &self, &subclass)) { @@ -734,7 +614,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } if (ok == Py_True) { Py_DECREF(ok); - if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -742,7 +622,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } if (ok == Py_False) { Py_DECREF(ok); - if (_add_to_weak_set(impl->_abc_negative_cache, subclass, 0) < 0) { + if (_add_to_weak_set(impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -765,7 +645,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) goto end; } if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -774,33 +654,8 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* 5. Check if it's a subclass of a registered class (recursive). */ - pos = 0; - Py_hash_t hash; - _enter_iter(impl->_abc_registry); - - while (_PySet_NextEntry(impl->_abc_registry->data, &pos, &key, &hash)) { - rkey = PyWeakref_GetObject(key); - if (rkey == Py_None) { - continue; - } - int r = PyObject_IsSubclass(subclass, rkey); - if (r < 0) { - _exit_iter(impl->_abc_registry); - goto end; - } - if (r > 0) { - if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { - _exit_iter(impl->_abc_registry); - goto end; - } - if (_exit_iter(impl->_abc_registry) < 0) { - goto end; - } - result = Py_True; - goto end; - } - } - if (_exit_iter(impl->_abc_registry) < 0) { + if (subclasscheck_check_registry(impl, subclass, &result)) { + // Exception occured or result is set. goto end; } @@ -813,7 +668,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { int r = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (r > 0) { - if (_add_to_weak_set(impl->_abc_cache, subclass, 0) < 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -825,7 +680,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* No dice; update negative cache. */ - if (_add_to_weak_set(impl->_abc_negative_cache, subclass, 0) < 0) { + if (_add_to_weak_set(impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -838,6 +693,62 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } +static int +subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, + PyObject **result) +{ + Py_ssize_t registry_size = PySet_Size(impl->_abc_registry); + if (registry_size == 0) { + return 0; + } + + int ret = 0; + + // Weakref callback may remove entry from set. + // Se we take snapshot of registry first. + PyObject **copy = PyMem_Malloc(sizeof(PyObject*) * registry_size); + PyObject *key; + Py_ssize_t pos = 0; + Py_hash_t hash; + Py_ssize_t i = 0; + + while (_PySet_NextEntry(impl->_abc_registry, &pos, &key, &hash)) { + Py_INCREF(key); + copy[i++] = key; + } + assert(i == registry_size); + + for (i = 0; i < registry_size; i++) { + PyObject *rkey = PyWeakref_GetObject(copy[i]); + if (rkey == Py_None) { + continue; + } + Py_INCREF(rkey); + int r = PyObject_IsSubclass(subclass, rkey); + Py_DECREF(rkey); + if (r < 0) { + ret = -1; + break; + } + if (r > 0) { + if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { + ret = -1; + break; + } + *result = Py_True; + ret = 1; + break; + } + } + + for (i = 0; i < registry_size; i++) { + Py_DECREF(copy[i]); + } + PyMem_Free(copy); + return ret; +} + + PyDoc_STRVAR(_cache_token_doc, "Returns the current ABC cache token.\n\ \n\ @@ -893,11 +804,6 @@ PyInit__abc(void) } _abc_data_type.tp_doc = abc_data_doc; - if (PyType_Ready(&_guarded_set_type) < 0) { - return NULL; - } - _guarded_set_type.tp_doc = guarded_set_doc; - abc_invalidation_counter = PyLong_FromLong(0); return PyModule_Create(&_abcmodule); } From f58822edbb75b3b170070806552284beb01a96ab Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 30 Jan 2018 10:54:18 +0900 Subject: [PATCH 68/93] Add NULL check after PyWeakref_GetObject --- Modules/_abc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_abc.c b/Modules/_abc.c index 03486bd35135f9..f7b7368e366806 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -720,6 +720,11 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, for (i = 0; i < registry_size; i++) { PyObject *rkey = PyWeakref_GetObject(copy[i]); + if (rkey == NULL) { + // Someone inject non-weakref type in the registry. + ret = -1; + break; + } if (rkey == Py_None) { continue; } From 3b74bdc9b160fe8ea272b91b8593c70f4fd5fe1f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 30 Jan 2018 10:54:41 +0900 Subject: [PATCH 69/93] Add fast path for looking register --- Modules/_abc.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index f7b7368e366806..a85babdadda69c 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -702,7 +702,16 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return 0; } - int ret = 0; + // Fast path: check subclass is in weakref directly. + int ret = _in_weak_set(impl->_abc_registry, subclass); + if (ret < 0) { + *result = NULL; + return -1; + } + if (ret > 0) { + *result = Py_True; + return 1; + } // Weakref callback may remove entry from set. // Se we take snapshot of registry first. From db1c852a70d59fb44c35878ae35a05a29853d058 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 30 Jan 2018 12:53:56 +0900 Subject: [PATCH 70/93] Check negative cache version before cache lookup --- Modules/_abc.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 357353b70baaa6..d5380e5cbe409a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -705,10 +705,6 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* 2. Check negative cache; may have to invalidate. */ - incache = _in_weak_set(impl->_abc_negative_cache, subclass); - if (incache < 0) { - goto end; - } int r = PyObject_RichCompareBool(impl->_abc_negative_cache_version, abc_invalidation_counter, Py_LT); assert(r >= 0); // Both should be PyLong @@ -721,9 +717,16 @@ _abc_subclasscheck(PyObject *m, PyObject *args) then carefully DECREF the old one. */ Py_INCREF(abc_invalidation_counter); Py_SETREF(impl->_abc_negative_cache_version, abc_invalidation_counter); - } else if (incache > 0) { - result = Py_False; - goto end; + } + else { + incache = _in_weak_set(impl->_abc_negative_cache, subclass); + if (incache < 0) { + goto end; + } + if (incache > 0) { + result = Py_False; + goto end; + } } /* 3. Check the subclass hook. */ From 538472606a03762d7c850a9a1b0188ffd08b4392 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 12 Feb 2018 17:29:55 +0900 Subject: [PATCH 71/93] Fix nits pointed by pppery. --- Modules/_abc.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 4dd4a083643929..ccf60c8734da25 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -116,6 +116,9 @@ _get_impl(PyObject *self) static int _in_weak_set(PyObject *set, PyObject *obj) { + if (PySet_Size(set) == 0) { + return 0; + } PyObject *ref = PyWeakref_NewRef(obj, NULL); if (ref == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { @@ -302,12 +305,13 @@ compute_abstract_methods(PyObject *self) for (Py_ssize_t pos = 0; pos < PyList_GET_SIZE(items); pos++) { PyObject *it = PySequence_Fast( PyList_GET_ITEM(items, pos), - "items() returned non-sequence item"); + "items() returned non-iterable"); if (!it) { goto error; } if (PySequence_Fast_GET_SIZE(it) != 2) { - PyErr_SetString(PyExc_TypeError, "items() returned not 2-tuple item"); + PyErr_SetString(PyExc_TypeError, + "items() returned item which size is not 2"); Py_DECREF(it); goto error; } @@ -535,7 +539,7 @@ _abc_instancecheck(PyObject *m, PyObject *args) } result = _PyObject_CallMethodIdObjArgs(self, &PyId___subclasscheck__, subclass, NULL); - if (result == NULL || result == Py_True) { + if (result == NULL || PyObject_IsTrue(result)) { goto end; } Py_DECREF(result); @@ -700,11 +704,6 @@ static int subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, PyObject **result) { - Py_ssize_t registry_size = PySet_Size(impl->_abc_registry); - if (registry_size == 0) { - return 0; - } - // Fast path: check subclass is in weakref directly. int ret = _in_weak_set(impl->_abc_registry, subclass); if (ret < 0) { @@ -716,6 +715,10 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return 1; } + Py_ssize_t registry_size = PySet_Size(impl->_abc_registry); + if (registry_size == 0) { + return 0; + } // Weakref callback may remove entry from set. // Se we take snapshot of registry first. PyObject **copy = PyMem_Malloc(sizeof(PyObject*) * registry_size); From a48eecc2dee7a6c143e792b839f3cf97b4c2836d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 14 Feb 2018 19:23:10 +0900 Subject: [PATCH 72/93] Create set lazily --- Modules/_abc.c | 68 ++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index ccf60c8734da25..8d1e0ee401aad7 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -39,9 +39,9 @@ typedef struct { static void abc_data_dealloc(_abc_data *self) { - Py_DECREF(self->_abc_registry); - Py_DECREF(self->_abc_cache); - Py_DECREF(self->_abc_negative_cache); + Py_XDECREF(self->_abc_registry); + Py_XDECREF(self->_abc_cache); + Py_XDECREF(self->_abc_negative_cache); Py_DECREF(self->_abc_negative_cache_version); Py_TYPE(self)->tp_free(self); } @@ -49,40 +49,14 @@ abc_data_dealloc(_abc_data *self) static PyObject * abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *registry = NULL, *cache = NULL, *negative_cache = NULL; - registry = PySet_New(NULL); - if (registry == NULL) { - return NULL; - } - cache = PySet_New(NULL); - if (cache == NULL) { - Py_DECREF(registry); - return NULL; - } - negative_cache = PySet_New(NULL); - if (negative_cache == NULL) { - Py_DECREF(registry); - Py_DECREF(cache); - return NULL; - } - _abc_data *self = (_abc_data *) type->tp_alloc(type, 0); if (self == NULL) { - goto error; + return NULL; } - self->_abc_registry = registry; - self->_abc_cache = cache; - self->_abc_negative_cache = negative_cache; self->_abc_negative_cache_version = abc_invalidation_counter; Py_INCREF(abc_invalidation_counter); return (PyObject *) self; - -error: - Py_DECREF(registry); - Py_DECREF(cache); - Py_DECREF(negative_cache); - return NULL; } PyDoc_STRVAR(abc_data_doc, @@ -116,7 +90,7 @@ _get_impl(PyObject *self) static int _in_weak_set(PyObject *set, PyObject *obj) { - if (PySet_Size(set) == 0) { + if (set == NULL || PySet_Size(set) == 0) { return 0; } PyObject *ref = PyWeakref_NewRef(obj, NULL); @@ -154,8 +128,16 @@ static PyMethodDef _destroy_def = { }; static int -_add_to_weak_set(PyObject *set, PyObject *obj) +_add_to_weak_set(PyObject **pset, PyObject *obj) { + if (*pset == NULL) { + *pset = PySet_New(NULL); + if (*pset == NULL) { + return -1; + } + } + + PyObject *set = *pset; PyObject *ref, *wr; PyObject *destroy_cb; wr = PyWeakref_NewRef(set, NULL); @@ -192,7 +174,7 @@ _reset_registry(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - if (PySet_Clear(impl->_abc_registry) < 0) { + if (impl->_abc_registry != NULL && PySet_Clear(impl->_abc_registry) < 0) { Py_DECREF(impl); return NULL; } @@ -465,7 +447,7 @@ _abc_register(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - if (_add_to_weak_set(impl->_abc_registry, subclass) < 0) { + if (_add_to_weak_set(&impl->_abc_registry, subclass) < 0) { Py_DECREF(impl); return NULL; } @@ -594,7 +576,8 @@ _abc_subclasscheck(PyObject *m, PyObject *args) assert(r >= 0); // Both should be PyLong if (r > 0) { /* Invalidate the negative cache. */ - if (PySet_Clear(impl->_abc_negative_cache) < 0) { + if (impl->_abc_negative_cache != NULL && + PySet_Clear(impl->_abc_negative_cache) < 0) { goto end; } /* INCREF the new value of cache version, @@ -621,7 +604,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } if (ok == Py_True) { Py_DECREF(ok); - if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -629,7 +612,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } if (ok == Py_False) { Py_DECREF(ok); - if (_add_to_weak_set(impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -652,7 +635,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) goto end; } if ((PyObject *)self == mro_item) { - if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -675,7 +658,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { int r = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); if (r > 0) { - if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -687,7 +670,7 @@ _abc_subclasscheck(PyObject *m, PyObject *args) } /* No dice; update negative cache. */ - if (_add_to_weak_set(impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -715,6 +698,9 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return 1; } + if (impl->_abc_registry == NULL) { + return 0; + } Py_ssize_t registry_size = PySet_Size(impl->_abc_registry); if (registry_size == 0) { return 0; @@ -751,7 +737,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, break; } if (r > 0) { - if (_add_to_weak_set(impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { ret = -1; break; } From 16a8db18359a02429a7681b5a63ae31a52d43638 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Feb 2018 01:52:04 +0900 Subject: [PATCH 73/93] Explicitly set NULL --- Modules/_abc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_abc.c b/Modules/_abc.c index 8d1e0ee401aad7..ec0a6f427091d4 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -54,6 +54,9 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + self->_abc_registry = NULL; + self->_abc_cache = NULL; + self->_abc_negative_cache = NULL; self->_abc_negative_cache_version = abc_invalidation_counter; Py_INCREF(abc_invalidation_counter); return (PyObject *) self; From 86a9b8d68f06b11e74ff4e4d34fe2c469341afc9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Feb 2018 07:58:13 +0900 Subject: [PATCH 74/93] Check PyObject_IsTrue() error --- Modules/_abc.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index ec0a6f427091d4..b6192baef70e8d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -524,12 +524,23 @@ _abc_instancecheck(PyObject *m, PyObject *args) } result = _PyObject_CallMethodIdObjArgs(self, &PyId___subclasscheck__, subclass, NULL); - if (result == NULL || PyObject_IsTrue(result)) { + if (result == NULL) { goto end; } - Py_DECREF(result); - result = _PyObject_CallMethodIdObjArgs(self, &PyId___subclasscheck__, - subtype, NULL); + + switch (PyObject_IsTrue(result)) { + case -1: + Py_DECREF(result); + result = NULL; + break; + case 0: + Py_DECREF(result); + result = _PyObject_CallMethodIdObjArgs(self, &PyId___subclasscheck__, + subtype, NULL); + break; + case 1: // Nothing to do. + break; + } end: Py_XDECREF(impl); From 36c201323546954553554e120729432c6be8c727 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Feb 2018 08:11:45 +0900 Subject: [PATCH 75/93] Strip TODO comments. --- Modules/_abc.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index b6192baef70e8d..6233de28340ea2 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,9 +1,5 @@ /* ABCMeta implementation */ -/* TODO: Test this hard in multithreaded context. */ -/* TODO: Think about inlining some calls and/or using macros */ -/* TODO: Use separate branches with "fast paths" */ - #include "Python.h" #include "structmember.h" @@ -281,7 +277,8 @@ compute_abstract_methods(PyObject *self) goto error; } - /* TODO: Fast path for exact dicts with PyDict_Next */ + // We can't use PyDict_Next(ns) even when ns is dict because + // _PyObject_IsAbstract() can mutate ns. items = PyMapping_Items(ns); if (!items) { goto error; From 09c537099659099def36dac29be8809c16da688f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Feb 2018 08:18:55 +0900 Subject: [PATCH 76/93] Add NEWS entry --- .../next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst diff --git a/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst b/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst new file mode 100644 index 00000000000000..16a77c4524741f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst @@ -0,0 +1,2 @@ +``_abc`` module is added. It is speedup module implemented in C for +``abc``. It makes Python startup 10% faster. From 207d8e92d1e2844ec4bcbe7c3a10be7cc67c4383 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Feb 2018 15:59:28 +0900 Subject: [PATCH 77/93] Fix _reset_caches --- Modules/_abc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 6233de28340ea2..5ccf956d66502b 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -198,12 +198,13 @@ _reset_caches(PyObject *m, PyObject *args) if (impl == NULL) { return NULL; } - if (PySet_Clear(impl->_abc_cache) < 0) { + if (impl->_abc_cache != NULL && PySet_Clear(impl->_abc_cache) < 0) { Py_DECREF(impl); return NULL; } /* also the second cache */ - if (PySet_Clear(impl->_abc_negative_cache) < 0) { + if (impl->_abc_negative_cache != NULL && + PySet_Clear(impl->_abc_negative_cache) < 0) { Py_DECREF(impl); return NULL; } From a15377b6590b4be91df9c41ecadb603d8ca43d3f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Feb 2018 16:01:38 +0900 Subject: [PATCH 78/93] Add default: Py_UNREACHABLE() --- Modules/_abc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_abc.c b/Modules/_abc.c index 5ccf956d66502b..facf0286b90865 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -538,6 +538,8 @@ _abc_instancecheck(PyObject *m, PyObject *args) break; case 1: // Nothing to do. break; + default: + Py_UNREACHABLE(); } end: From d31da13cc43b234f1e6c74b89394f9f0747d488f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Feb 2018 20:13:01 +0900 Subject: [PATCH 79/93] Rephrase NEWS entry --- .../next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst b/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst index 16a77c4524741f..1cf29b3495d835 100644 --- a/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst +++ b/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst @@ -1,2 +1,3 @@ ``_abc`` module is added. It is speedup module implemented in C for -``abc``. It makes Python startup 10% faster. +``abc``. Creating ABC subclass and calling ``isinstance`` or ``issubclass`` +with ABC subclass are up to 1.5x faster. From 48de70ea9212f2fa23175a6c4c99e9cb189b6579 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 16 Feb 2018 01:26:08 +0000 Subject: [PATCH 80/93] Factor out Python version to a separate file --- Lib/abc.py | 265 ++++++++++--------------------------------- Lib/test/test_abc.py | 23 ++-- 2 files changed, 72 insertions(+), 216 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 00f69c20f63bc8..22539f9b067655 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -4,30 +4,6 @@ """Abstract Base Classes (ABCs) according to PEP 3119.""" -def get_cache_token(): - """Returns the current ABC cache token. - - The token is an opaque object (supporting equality testing) identifying the - current version of the ABC cache for virtual subclasses. The token changes - with every call to ``register()`` on any ABC. - """ - return _Py_ABCMeta._abc_invalidation_counter - - -from _weakrefset import WeakSet # Used by legacy _Py_ABCMeta - - -_C_speedup = False -try: - from _abc import (get_cache_token, _abc_init, _abc_register, - _abc_instancecheck, _abc_subclasscheck, _get_dump, - _reset_registry, _reset_caches) -except ImportError: - pass -else: - _C_speedup = True - - def abstractmethod(funcobj): """A decorator indicating abstract methods. @@ -125,191 +101,66 @@ def setx(self, value): ... __isabstractmethod__ = True -class _Py_ABCMeta(type): - """Metaclass for defining Abstract Base Classes (ABCs). - - Use this metaclass to create an ABC. An ABC can be subclassed - directly, and then acts as a mix-in class. You can also register - unrelated concrete classes (even built-in classes) and unrelated - ABCs as 'virtual subclasses' -- these and their descendants will - be considered subclasses of the registering ABC by the built-in - issubclass() function, but the registering ABC won't show up in - their MRO (Method Resolution Order) nor will method - implementations defined by the registering ABC be callable (not - even via super()). - """ - - # A global counter that is incremented each time a class is - # registered as a virtual subclass of anything. It forces the - # negative cache to be cleared before its next use. - # Note: this counter is private. Use `abc.get_cache_token()` for - # external code. - _abc_invalidation_counter = 0 - - def __new__(mcls, name, bases, namespace, **kwargs): - cls = super().__new__(mcls, name, bases, namespace, **kwargs) - # Compute set of abstract method names - abstracts = {name - for name, value in namespace.items() - if getattr(value, "__isabstractmethod__", False)} - for base in bases: - for name in getattr(base, "__abstractmethods__", set()): - value = getattr(cls, name, None) - if getattr(value, "__isabstractmethod__", False): - abstracts.add(name) - cls.__abstractmethods__ = frozenset(abstracts) - # Set up inheritance registry - cls._abc_registry = WeakSet() - cls._abc_cache = WeakSet() - cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = _Py_ABCMeta._abc_invalidation_counter - return cls - - def register(cls, subclass): - """Register a virtual subclass of an ABC. - - Returns the subclass, to allow usage as a class decorator. - """ - if not isinstance(subclass, type): - raise TypeError("Can only register classes") - if issubclass(subclass, cls): - return subclass # Already a subclass - # Subtle: test for cycles *after* testing for "already a subclass"; - # this means we allow X.register(X) and interpret it as a no-op. - if issubclass(cls, subclass): - # This would create a cycle, which is bad for the algorithm below - raise RuntimeError("Refusing to create an inheritance cycle") - cls._abc_registry.add(subclass) - _Py_ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache - return subclass - - def _dump_registry(cls, file=None): - """Debug helper to print the ABC registry.""" - print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) - print(f"Inv. counter: {get_cache_token()}", file=file) - for name in cls.__dict__: - if name.startswith("_abc_"): - value = getattr(cls, name) - if isinstance(value, WeakSet): - value = set(value) - print(f"{name}: {value!r}", file=file) - - def _abc_registry_clear(cls): - """Clear the registry (for debugging or testing).""" - cls._abc_registry.clear() - - def _abc_caches_clear(cls): - """Clear the caches (for debugging or testing).""" - cls._abc_cache.clear() - cls._abc_negative_cache.clear() - - def __instancecheck__(cls, instance): - """Override for isinstance(instance, cls).""" - # Inline the cache checking - subclass = instance.__class__ - if subclass in cls._abc_cache: - return True - subtype = type(instance) - if subtype is subclass: - if (cls._abc_negative_cache_version == - _Py_ABCMeta._abc_invalidation_counter and - subclass in cls._abc_negative_cache): - return False - # Fall back to the subclass check. - return cls.__subclasscheck__(subclass) - return any(cls.__subclasscheck__(c) for c in {subclass, subtype}) - - def __subclasscheck__(cls, subclass): - """Override for issubclass(subclass, cls).""" - # Check cache - if subclass in cls._abc_cache: - return True - # Check negative cache; may have to invalidate - if cls._abc_negative_cache_version < _Py_ABCMeta._abc_invalidation_counter: - # Invalidate the negative cache - cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = _Py_ABCMeta._abc_invalidation_counter - elif subclass in cls._abc_negative_cache: - return False - # Check the subclass hook - ok = cls.__subclasshook__(subclass) - if ok is not NotImplemented: - assert isinstance(ok, bool) - if ok: - cls._abc_cache.add(subclass) - else: - cls._abc_negative_cache.add(subclass) - return ok - # Check if it's a direct subclass - if cls in getattr(subclass, '__mro__', ()): - cls._abc_cache.add(subclass) - return True - # Check if it's a subclass of a registered class (recursive) - for rcls in cls._abc_registry: - if issubclass(subclass, rcls): - cls._abc_cache.add(subclass) - return True - # Check if it's a subclass of a subclass (recursive) - for scls in cls.__subclasses__(): - if issubclass(subclass, scls): - cls._abc_cache.add(subclass) - return True - # No dice; update negative cache - cls._abc_negative_cache.add(subclass) - return False - - -class _C_ABCMeta(type): - def __new__(mcls, name, bases, namespace, **kwargs): - cls = super().__new__(mcls, name, bases, namespace, **kwargs) - _abc_init(cls) - return cls - - def register(cls, subclass): - """Register a virtual subclass of an ABC. - - Returns the subclass, to allow usage as a class decorator. - """ - return _abc_register(cls, subclass) - - def __instancecheck__(cls, instance): - """Override for isinstance(instance, cls).""" - return _abc_instancecheck(cls, instance) - - def __subclasscheck__(cls, subclass): - """Override for issubclass(subclass, cls).""" - return _abc_subclasscheck(cls, subclass) - - def _dump_registry(cls, file=None): - """Debug helper to print the ABC registry.""" - print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) - print(f"Inv. counter: {get_cache_token()}", file=file) - (_abc_registry, _abc_cache, _abc_negative_cache, - _abc_negative_cache_version) = _get_dump(cls) - print(f"_abc_registry: {_abc_registry!r}", file=file) - print(f"_abc_cache: {_abc_cache!r}", file=file) - print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) - print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", - file=file) - - def _abc_registry_clear(cls): - """Clear the registry (for debugging or testing).""" - _reset_registry(cls) - - def _abc_caches_clear(cls): - """Clear the caches (for debugging or testing).""" - _reset_caches(cls) - - -if _C_speedup: - ABCMeta = _C_ABCMeta +try: + from _abc import (get_cache_token, _abc_init, _abc_register, + _abc_instancecheck, _abc_subclasscheck, _get_dump, + _reset_registry, _reset_caches) +except ImportError: + from _py_abc import ABCMeta, get_cache_token else: - ABCMeta = _Py_ABCMeta - -_C_ABCMeta.__doc__ = _Py_ABCMeta.__doc__ + class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + _abc_init(cls) + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + return _abc_register(cls, subclass) + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + return _abc_instancecheck(cls, instance) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + return _abc_subclasscheck(cls, subclass) + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + (_abc_registry, _abc_cache, _abc_negative_cache, + _abc_negative_cache_version) = _get_dump(cls) + print(f"_abc_registry: {_abc_registry!r}", file=file) + print(f"_abc_cache: {_abc_cache!r}", file=file) + print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) + print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", + file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + _reset_registry(cls) + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + _reset_caches(cls) -_C_ABCMeta.__name__ = _C_ABCMeta.__qualname__ = 'ABCMeta' -_Py_ABCMeta.__name__ = _Py_ABCMeta.__qualname__ = 'ABCMeta' class ABC(metaclass=ABCMeta): diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 12ea47836830a3..af26c1d6b8c5ea 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -1,17 +1,18 @@ # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. -# Note: each test is run with _Py_ABCMeta and _C_ABCMeta, except for two tests, where we -# need to "sync" with actual abc.ABCMeta, these tests check get_cache_token and ABC helper. +# Note: each test is run with Python and C versions of ABCMeta. Except for +# test_ABC_helper(), which assures that abc.ABC is an instance of abc.ABCMeta. """Unit tests for abc.py.""" import unittest import abc +import _py_abc from inspect import isabstract -def test_factory(abc_ABCMeta): +def test_factory(abc_ABCMeta, abc_get_cache_token): class TestLegacyAPI(unittest.TestCase): def test_abstractproperty_basics(self): @@ -296,16 +297,16 @@ class C(B): self.assertIs(C, A.register(C)) def test_isinstance_invalidation(self): - class A(metaclass=abc.ABCMeta): + class A(metaclass=abc_ABCMeta): pass class B: pass b = B() self.assertFalse(isinstance(b, A)) self.assertFalse(isinstance(b, (A,))) - token_old = abc.get_cache_token() + token_old = abc_get_cache_token() A.register(B) - token_new = abc.get_cache_token() + token_new = abc_get_cache_token() self.assertNotEqual(token_old, token_new) self.assertTrue(isinstance(b, A)) self.assertTrue(isinstance(b, (A,))) @@ -423,18 +424,22 @@ class C(with_metaclass(abc_ABCMeta, A, B)): class TestABCWithInitSubclass(unittest.TestCase): def test_works_with_init_subclass(self): + class abc_ABC(metaclass=abc_ABCMeta): + __slots__ = () saved_kwargs = {} class ReceivesClassKwargs: def __init_subclass__(cls, **kwargs): super().__init_subclass__() saved_kwargs.update(kwargs) - class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3): + class Receiver(ReceivesClassKwargs, abc_ABC, x=1, y=2, z=3): pass self.assertEqual(saved_kwargs, dict(x=1, y=2, z=3)) return TestLegacyAPI, TestABC, TestABCWithInitSubclass -TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(abc._Py_ABCMeta) -TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(abc._C_ABCMeta) +TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(abc.ABCMeta, + abc.get_cache_token) +TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(_py_abc.ABCMeta, + _py_abc.get_cache_token) if __name__ == "__main__": unittest.main() From 3bd0666eaef9d8c63ece1ee2966ae02284bd9f6e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 16 Feb 2018 01:26:31 +0000 Subject: [PATCH 81/93] Factor out Python version to a separate file --- Lib/_py_abc.py | 145 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 Lib/_py_abc.py diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py new file mode 100644 index 00000000000000..c14e471e8021c2 --- /dev/null +++ b/Lib/_py_abc.py @@ -0,0 +1,145 @@ +from _weakrefset import WeakSet + + +def get_cache_token(): + """Returns the current ABC cache token. + + The token is an opaque object (supporting equality testing) identifying the + current version of the ABC cache for virtual subclasses. The token changes + with every call to ``register()`` on any ABC. + """ + return ABCMeta._abc_invalidation_counter + + +class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + + # A global counter that is incremented each time a class is + # registered as a virtual subclass of anything. It forces the + # negative cache to be cleared before its next use. + # Note: this counter is private. Use `abc.get_cache_token()` for + # external code. + _abc_invalidation_counter = 0 + + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + # Compute set of abstract method names + abstracts = {name + for name, value in namespace.items() + if getattr(value, "__isabstractmethod__", False)} + for base in bases: + for name in getattr(base, "__abstractmethods__", set()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = frozenset(abstracts) + # Set up inheritance registry + cls._abc_registry = WeakSet() + cls._abc_cache = WeakSet() + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + if not isinstance(subclass, type): + raise TypeError("Can only register classes") + if issubclass(subclass, cls): + return subclass # Already a subclass + # Subtle: test for cycles *after* testing for "already a subclass"; + # this means we allow X.register(X) and interpret it as a no-op. + if issubclass(cls, subclass): + # This would create a cycle, which is bad for the algorithm below + raise RuntimeError("Refusing to create an inheritance cycle") + cls._abc_registry.add(subclass) + ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + return subclass + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + for name in cls.__dict__: + if name.startswith("_abc_"): + value = getattr(cls, name) + if isinstance(value, WeakSet): + value = set(value) + print(f"{name}: {value!r}", file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + cls._abc_registry.clear() + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + cls._abc_cache.clear() + cls._abc_negative_cache.clear() + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + # Inline the cache checking + subclass = instance.__class__ + if subclass in cls._abc_cache: + return True + subtype = type(instance) + if subtype is subclass: + if (cls._abc_negative_cache_version == + ABCMeta._abc_invalidation_counter and + subclass in cls._abc_negative_cache): + return False + # Fall back to the subclass check. + return cls.__subclasscheck__(subclass) + return any(cls.__subclasscheck__(c) for c in {subclass, subtype}) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + # Check cache + if subclass in cls._abc_cache: + return True + # Check negative cache; may have to invalidate + if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + # Invalidate the negative cache + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + elif subclass in cls._abc_negative_cache: + return False + # Check the subclass hook + ok = cls.__subclasshook__(subclass) + if ok is not NotImplemented: + assert isinstance(ok, bool) + if ok: + cls._abc_cache.add(subclass) + else: + cls._abc_negative_cache.add(subclass) + return ok + # Check if it's a direct subclass + if cls in getattr(subclass, '__mro__', ()): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a registered class (recursive) + for rcls in cls._abc_registry: + if issubclass(subclass, rcls): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a subclass (recursive) + for scls in cls.__subclasses__(): + if issubclass(subclass, scls): + cls._abc_cache.add(subclass) + return True + # No dice; update negative cache + cls._abc_negative_cache.add(subclass) + return False From 702347ace7e45114019fa284aff5e9880bf01b52 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 16 Feb 2018 01:27:29 +0000 Subject: [PATCH 82/93] Remove extra whitespace --- Lib/abc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/abc.py b/Lib/abc.py index 22539f9b067655..a7705281256056 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -162,7 +162,6 @@ def _abc_caches_clear(cls): _reset_caches(cls) - class ABC(metaclass=ABCMeta): """Helper class that provides a standard way to create an ABC using inheritance. From e0c978b1b266e382637913293344442387576623 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 16 Feb 2018 01:59:03 +0000 Subject: [PATCH 83/93] Add more details to NEWS --- .../2018-02-15-08-18-52.bpo-31333.4fF-gM.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst b/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst index 1cf29b3495d835..63fc72a5c11be6 100644 --- a/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst +++ b/Misc/NEWS.d/next/Library/2018-02-15-08-18-52.bpo-31333.4fF-gM.rst @@ -1,3 +1,10 @@ -``_abc`` module is added. It is speedup module implemented in C for -``abc``. Creating ABC subclass and calling ``isinstance`` or ``issubclass`` -with ABC subclass are up to 1.5x faster. +``_abc`` module is added. It is a speedup module with C implementations for +various functions and methods in ``abc``. Creating an ABC subclass and calling +``isinstance`` or ``issubclass`` with an ABC subclass are up to 1.5x faster. +In addition, this makes Python start-up up to 10% faster. + +Note that the new implementation hides internal registry and caches, previously +accessible via private attributes ``_abc_registry``, ``_abc_cache``, and +``_abc_negative_cache``. There are three debugging helper methods that can be +used instead ``_dump_registry``, ``_abc_registry_clear``, and +``_abc_caches_clear``. From b370dfe1c25a3a9cd357a2166aebebc886890d97 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 16 Feb 2018 02:04:15 +0000 Subject: [PATCH 84/93] Fix an import in refleak test --- Lib/test/libregrtest/refleak.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 765e06e20a11e0..6724488fcfb088 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -5,7 +5,13 @@ import warnings from inspect import isabstract from test import support -from _abc import _get_dump +try: + from _abc import _get_dump +except ImportError: + def _get_dump(cls): + # For legacy Python version + return (cls._abc_registry, cls._abc_cache, + cls._abc_negative_cache, cls._abc_negative_cache_version) def dash_R(the_module, test, indirect_test, huntrleaks): From a1ae0a7cc0fddb2b7d62caacba770b174bd0c729 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 16 Feb 2018 02:18:52 +0000 Subject: [PATCH 85/93] Restart tests From 4746211769405fe6e23c00a81613adb7917cf350 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 16 Feb 2018 12:55:46 +0000 Subject: [PATCH 86/93] Make order of subclass checks in Python version stable and consistent with the C version; fix space in a comment --- Lib/_py_abc.py | 2 +- Modules/_abc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index c14e471e8021c2..6f42ef32fa6961 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -103,7 +103,7 @@ def __instancecheck__(cls, instance): return False # Fall back to the subclass check. return cls.__subclasscheck__(subclass) - return any(cls.__subclasscheck__(c) for c in {subclass, subtype}) + return any(cls.__subclasscheck__(c) for c in (subclass, subtype)) def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" diff --git a/Modules/_abc.c b/Modules/_abc.c index facf0286b90865..9aac9cd9efefe3 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -249,7 +249,7 @@ _get_dump(PyObject *m, PyObject *args) Py_DECREF(cache); return NULL; } - Py_INCREF(impl->_abc_negative_cache_version); /* PyTuple_Packdoesn't do this. */ + Py_INCREF(impl->_abc_negative_cache_version); /* PyTuple_Pack doesn't do this. */ PyObject *res = PyTuple_Pack(4, registry, cache, negative_cache, impl->_abc_negative_cache_version); Py_DECREF(registry); From 001b416685b5a0c24d56eeb41a11255bd29ee875 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 17 Feb 2018 15:37:01 +0000 Subject: [PATCH 87/93] Convert _abc to Argument Clinic --- Modules/_abc.c | 202 ++++++++++++++++++++++------------------ Modules/clinic/_abc.c.h | 162 ++++++++++++++++++++++++++++++++ 2 files changed, 275 insertions(+), 89 deletions(-) create mode 100644 Modules/clinic/_abc.c.h diff --git a/Modules/_abc.c b/Modules/_abc.c index 9aac9cd9efefe3..deb18cc218e099 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -2,6 +2,12 @@ #include "Python.h" #include "structmember.h" +#include "clinic/_abc.c.h" + +/*[clinic input] +module _abc +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=964f5328e1aefcda]*/ PyDoc_STRVAR(_abc__doc__, "Module contains faster C implementation of abc.ABCMeta"); @@ -156,19 +162,21 @@ _add_to_weak_set(PyObject **pset, PyObject *obj) return ret; } +/*[clinic input] +_abc._reset_registry + + self: object + / + +Internal ABC helper to reset registry of a given class. -PyDoc_STRVAR(_reset_registry_doc, -"Internal ABC helper to reset registry of a given class.\n\ -\n\ -Should be only used by refleak.py"); +Should be only used by refleak.py +[clinic start generated code]*/ static PyObject * -_reset_registry(PyObject *m, PyObject *args) +_abc__reset_registry(PyObject *module, PyObject *self) +/*[clinic end generated code: output=92d591a43566cc10 input=12a0b7eb339ac35c]*/ { - PyObject *self; - if (!PyArg_UnpackTuple(args, "_reset_registry", 1, 1, &self)) { - return NULL; - } _abc_data *impl = _get_impl(self); if (impl == NULL) { return NULL; @@ -181,20 +189,22 @@ _reset_registry(PyObject *m, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(_reset_caches_doc, -"Internal ABC helper to reset both caches of a given class.\n\ -\n\ -Should be only used by refleak.py"); +/*[clinic input] +_abc._reset_caches + + self: object + / + +Internal ABC helper to reset both caches of a given class. + +Should be only used by refleak.py +[clinic start generated code]*/ static PyObject * -_reset_caches(PyObject *m, PyObject *args) +_abc__reset_caches(PyObject *module, PyObject *self) +/*[clinic end generated code: output=f296f0d5c513f80c input=c0ac616fd8acfb6f]*/ { - PyObject *self; - _abc_data *impl; - if (!PyArg_UnpackTuple(args, "_reset_caches", 1, 1, &self)) { - return NULL; - } - impl = _get_impl(self); + _abc_data *impl = _get_impl(self); if (impl == NULL) { return NULL; } @@ -212,22 +222,25 @@ _reset_caches(PyObject *m, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(_get_dump_doc, -"Internal ABC helper for cache and registry debugging.\n\ -\n\ -Return shallow copies of registry, of both caches, and\n\ -negative cache version. Don't call this function directly,\n\ -instead use ABC._dump_registry() for a nice repr."); +/*[clinic input] +_abc._get_dump + + self: object + / + +Internal ABC helper for cache and registry debugging. + +Return shallow copies of registry, of both caches, and +negative cache version. Don't call this function directly, +instead use ABC._dump_registry() for a nice repr. +[clinic start generated code]*/ static PyObject * -_get_dump(PyObject *m, PyObject *args) +_abc__get_dump(PyObject *module, PyObject *self) +/*[clinic end generated code: output=9d9569a8e2c1c443 input=2c5deb1bfe9e3c79]*/ { - PyObject *self, *registry, *cache, *negative_cache; - _abc_data *impl; - if (!PyArg_UnpackTuple(args, "_get_dump", 1, 1, &self)) { - return NULL; - } - impl = _get_impl(self); + PyObject *registry, *cache, *negative_cache; + _abc_data *impl = _get_impl(self); if (impl == NULL) { return NULL; } @@ -383,17 +396,20 @@ compute_abstract_methods(PyObject *self) return ret; } -PyDoc_STRVAR(_abc_init_doc, -"Internal ABC helper for class set-up. Should be never used outside abc module"); +/*[clinic input] +_abc._abc_init + + self: object + / + +Internal ABC helper for class set-up. Should be never used outside abc module +[clinic start generated code]*/ static PyObject * -_abc_init(PyObject *m, PyObject *args) +_abc__abc_init(PyObject *module, PyObject *self) +/*[clinic end generated code: output=594757375714cda1 input=6061a045416da18b]*/ { - PyObject *self, *data; - if (!PyArg_UnpackTuple(args, "_abc_init", 1, 1, &self)) { - return NULL; - } - + PyObject *data; if (compute_abstract_methods(self) < 0) { return NULL; } @@ -411,16 +427,20 @@ _abc_init(PyObject *m, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(_abc_register_doc, -"Internal ABC helper for subclasss registration. Should be never used outside abc module"); +/*[clinic input] +_abc._abc_register + + self: object + subclass: object + / + +Internal ABC helper for subclasss registration. Should be never used outside abc module +[clinic start generated code]*/ static PyObject * -_abc_register(PyObject *m, PyObject *args) +_abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) +/*[clinic end generated code: output=7851e7668c963524 input=4b94c86e77e9e901]*/ { - PyObject *self, *subclass = NULL; - if (!PyArg_UnpackTuple(args, "_abc_register", 2, 2, &self, &subclass)) { - return NULL; - } if (!PyType_Check(subclass)) { PyErr_SetString(PyExc_TypeError, "Can only register classes"); return NULL; @@ -470,18 +490,23 @@ _abc_register(PyObject *m, PyObject *args) return subclass; } -PyDoc_STRVAR(_abc_instancecheck_doc, -"Internal ABC helper for instance checks. Should be never used outside abc module"); + +/*[clinic input] +_abc._abc_instancecheck + + self: object + instance: object + / + +Internal ABC helper for instance checks. Should be never used outside abc module +[clinic start generated code]*/ static PyObject * -_abc_instancecheck(PyObject *m, PyObject *args) +_abc__abc_instancecheck_impl(PyObject *module, PyObject *self, + PyObject *instance) +/*[clinic end generated code: output=b8b5148f63b6b56f input=5b15699272795a93]*/ { - PyObject *self, *result = NULL, *subclass = NULL, - *subtype, *instance = NULL; - if (!PyArg_UnpackTuple(args, "_abc_instancecheck", 2, 2, &self, &instance)) { - return NULL; - } - + PyObject *subtype, *result = NULL, *subclass = NULL; _abc_data *impl = _get_impl(self); if (impl == NULL) { return NULL; @@ -555,20 +580,24 @@ _abc_instancecheck(PyObject *m, PyObject *args) static int subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, PyObject **result); -PyDoc_STRVAR(_abc_subclasscheck_doc, -"Internal ABC helper for subclasss checks. Should be never used outside abc module"); +/*[clinic input] +_abc._abc_subclasscheck + + self: object + subclass: object + / + +Internal ABC helper for subclasss checks. Should be never used outside abc module +[clinic start generated code]*/ static PyObject * -_abc_subclasscheck(PyObject *m, PyObject *args) +_abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, + PyObject *subclass) +/*[clinic end generated code: output=b56c9e4a530e3894 input=4c87faea511976a8]*/ { - PyObject *self, *subclasses = NULL, *subclass = NULL, *result = NULL; - PyObject *ok, *mro; + PyObject *ok, *mro, *subclasses = NULL, *result = NULL; Py_ssize_t pos; int incache; - if (!PyArg_UnpackTuple(args, "_abc_subclasscheck", 2, 2, &self, &subclass)) { - return NULL; - } - _abc_data *impl = _get_impl(self); if (impl == NULL) { return NULL; @@ -768,38 +797,33 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return ret; } +/*[clinic input] +_abc.get_cache_token + +Returns the current ABC cache token. -PyDoc_STRVAR(_cache_token_doc, -"Returns the current ABC cache token.\n\ -\n\ -The token is an opaque object (supporting equality testing) identifying the\n\ -current version of the ABC cache for virtual subclasses. The token changes\n\ -with every call to ``register()`` on any ABC."); +The token is an opaque object (supporting equality testing) identifying the +current version of the ABC cache for virtual subclasses. The token changes +with every call to ``register()`` on any ABC. +[clinic start generated code]*/ static PyObject * -get_cache_token(void) +_abc_get_cache_token_impl(PyObject *module) +/*[clinic end generated code: output=c7d87841e033dacc input=2a19dea381467239]*/ { Py_INCREF(abc_invalidation_counter); return abc_invalidation_counter; } static struct PyMethodDef module_functions[] = { - {"get_cache_token", (PyCFunction)get_cache_token, METH_NOARGS, - _cache_token_doc}, - {"_abc_init", (PyCFunction)_abc_init, METH_VARARGS, - _abc_init_doc}, - {"_reset_registry", (PyCFunction)_reset_registry, METH_VARARGS, - _reset_registry_doc}, - {"_reset_caches", (PyCFunction)_reset_caches, METH_VARARGS, - _reset_caches_doc}, - {"_get_dump", (PyCFunction)_get_dump, METH_VARARGS, - _get_dump_doc}, - {"_abc_register", (PyCFunction)_abc_register, METH_VARARGS, - _abc_register_doc}, - {"_abc_instancecheck", (PyCFunction)_abc_instancecheck, METH_VARARGS, - _abc_instancecheck_doc}, - {"_abc_subclasscheck", (PyCFunction)_abc_subclasscheck, METH_VARARGS, - _abc_subclasscheck_doc}, + _ABC_GET_CACHE_TOKEN_METHODDEF + _ABC__ABC_INIT_METHODDEF + _ABC__RESET_REGISTRY_METHODDEF + _ABC__RESET_CACHES_METHODDEF + _ABC__GET_DUMP_METHODDEF + _ABC__ABC_REGISTER_METHODDEF + _ABC__ABC_INSTANCECHECK_METHODDEF + _ABC__ABC_SUBCLASSCHECK_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h new file mode 100644 index 00000000000000..4428dc2ab143d2 --- /dev/null +++ b/Modules/clinic/_abc.c.h @@ -0,0 +1,162 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_abc__reset_registry__doc__, +"_reset_registry($module, self, /)\n" +"--\n" +"\n" +"Internal ABC helper to reset registry of a given class.\n" +"\n" +"Should be only used by refleak.py"); + +#define _ABC__RESET_REGISTRY_METHODDEF \ + {"_reset_registry", (PyCFunction)_abc__reset_registry, METH_O, _abc__reset_registry__doc__}, + +PyDoc_STRVAR(_abc__reset_caches__doc__, +"_reset_caches($module, self, /)\n" +"--\n" +"\n" +"Internal ABC helper to reset both caches of a given class.\n" +"\n" +"Should be only used by refleak.py"); + +#define _ABC__RESET_CACHES_METHODDEF \ + {"_reset_caches", (PyCFunction)_abc__reset_caches, METH_O, _abc__reset_caches__doc__}, + +PyDoc_STRVAR(_abc__get_dump__doc__, +"_get_dump($module, self, /)\n" +"--\n" +"\n" +"Internal ABC helper for cache and registry debugging.\n" +"\n" +"Return shallow copies of registry, of both caches, and\n" +"negative cache version. Don\'t call this function directly,\n" +"instead use ABC._dump_registry() for a nice repr."); + +#define _ABC__GET_DUMP_METHODDEF \ + {"_get_dump", (PyCFunction)_abc__get_dump, METH_O, _abc__get_dump__doc__}, + +PyDoc_STRVAR(_abc__abc_init__doc__, +"_abc_init($module, self, /)\n" +"--\n" +"\n" +"Internal ABC helper for class set-up. Should be never used outside abc module"); + +#define _ABC__ABC_INIT_METHODDEF \ + {"_abc_init", (PyCFunction)_abc__abc_init, METH_O, _abc__abc_init__doc__}, + +PyDoc_STRVAR(_abc__abc_register__doc__, +"_abc_register($module, self, subclass, /)\n" +"--\n" +"\n" +"Internal ABC helper for subclasss registration. Should be never used outside abc module"); + +#define _ABC__ABC_REGISTER_METHODDEF \ + {"_abc_register", (PyCFunction)_abc__abc_register, METH_FASTCALL, _abc__abc_register__doc__}, + +static PyObject * +_abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass); + +static PyObject * +_abc__abc_register(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *self; + PyObject *subclass; + + if (!_PyArg_UnpackStack(args, nargs, "_abc_register", + 2, 2, + &self, &subclass)) { + goto exit; + } + return_value = _abc__abc_register_impl(module, self, subclass); + +exit: + return return_value; +} + +PyDoc_STRVAR(_abc__abc_instancecheck__doc__, +"_abc_instancecheck($module, self, instance, /)\n" +"--\n" +"\n" +"Internal ABC helper for instance checks. Should be never used outside abc module"); + +#define _ABC__ABC_INSTANCECHECK_METHODDEF \ + {"_abc_instancecheck", (PyCFunction)_abc__abc_instancecheck, METH_FASTCALL, _abc__abc_instancecheck__doc__}, + +static PyObject * +_abc__abc_instancecheck_impl(PyObject *module, PyObject *self, + PyObject *instance); + +static PyObject * +_abc__abc_instancecheck(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *self; + PyObject *instance; + + if (!_PyArg_UnpackStack(args, nargs, "_abc_instancecheck", + 2, 2, + &self, &instance)) { + goto exit; + } + return_value = _abc__abc_instancecheck_impl(module, self, instance); + +exit: + return return_value; +} + +PyDoc_STRVAR(_abc__abc_subclasscheck__doc__, +"_abc_subclasscheck($module, self, subclass, /)\n" +"--\n" +"\n" +"Internal ABC helper for subclasss checks. Should be never used outside abc module"); + +#define _ABC__ABC_SUBCLASSCHECK_METHODDEF \ + {"_abc_subclasscheck", (PyCFunction)_abc__abc_subclasscheck, METH_FASTCALL, _abc__abc_subclasscheck__doc__}, + +static PyObject * +_abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, + PyObject *subclass); + +static PyObject * +_abc__abc_subclasscheck(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *self; + PyObject *subclass; + + if (!_PyArg_UnpackStack(args, nargs, "_abc_subclasscheck", + 2, 2, + &self, &subclass)) { + goto exit; + } + return_value = _abc__abc_subclasscheck_impl(module, self, subclass); + +exit: + return return_value; +} + +PyDoc_STRVAR(_abc_get_cache_token__doc__, +"get_cache_token($module, /)\n" +"--\n" +"\n" +"Returns the current ABC cache token.\n" +"\n" +"The token is an opaque object (supporting equality testing) identifying the\n" +"current version of the ABC cache for virtual subclasses. The token changes\n" +"with every call to ``register()`` on any ABC."); + +#define _ABC_GET_CACHE_TOKEN_METHODDEF \ + {"get_cache_token", (PyCFunction)_abc_get_cache_token, METH_NOARGS, _abc_get_cache_token__doc__}, + +static PyObject * +_abc_get_cache_token_impl(PyObject *module); + +static PyObject * +_abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _abc_get_cache_token_impl(module); +} +/*[clinic end generated code: output=3e785943d168e371 input=a9049054013a1b77]*/ From 289c41413fdb1ba4f1265e362a203a5f623ac816 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 17 Feb 2018 16:02:12 +0000 Subject: [PATCH 88/93] Regenerate clinic --- Modules/_abc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index deb18cc218e099..7fcff07ead9f4c 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -809,7 +809,7 @@ with every call to ``register()`` on any ABC. static PyObject * _abc_get_cache_token_impl(PyObject *module) -/*[clinic end generated code: output=c7d87841e033dacc input=2a19dea381467239]*/ +/*[clinic end generated code: output=c7d87841e033dacc input=1d49ab7218687f59]*/ { Py_INCREF(abc_invalidation_counter); return abc_invalidation_counter; From ac0c63911e6824a6c5b9231deb633cb9c5d38cc0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 17 Feb 2018 21:19:52 +0000 Subject: [PATCH 89/93] Switch from Python invalidation counter to C long long --- Modules/_abc.c | 53 +++++++++++++++++--------------------------------- 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 7fcff07ead9f4c..963ba0eb766234 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -25,7 +25,7 @@ _Py_IDENTIFIER(__subclasshook__); negative cache to be cleared before its next use. Note: this counter is private. Use `abc.get_cache_token()` for external code. */ -static PyObject *abc_invalidation_counter; +static unsigned long long abc_invalidation_counter = 0; /* This object stores internal state for ABCs. Note that we can use normal sets for caches, @@ -35,7 +35,7 @@ typedef struct { PyObject *_abc_registry; PyObject *_abc_cache; /* Normal set of weak references. */ PyObject *_abc_negative_cache; /* Normal set of weak references. */ - PyObject *_abc_negative_cache_version; + unsigned long long _abc_negative_cache_version; } _abc_data; static void @@ -44,7 +44,6 @@ abc_data_dealloc(_abc_data *self) Py_XDECREF(self->_abc_registry); Py_XDECREF(self->_abc_cache); Py_XDECREF(self->_abc_negative_cache); - Py_DECREF(self->_abc_negative_cache_version); Py_TYPE(self)->tp_free(self); } @@ -60,7 +59,6 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->_abc_cache = NULL; self->_abc_negative_cache = NULL; self->_abc_negative_cache_version = abc_invalidation_counter; - Py_INCREF(abc_invalidation_counter); return (PyObject *) self; } @@ -239,7 +237,7 @@ static PyObject * _abc__get_dump(PyObject *module, PyObject *self) /*[clinic end generated code: output=9d9569a8e2c1c443 input=2c5deb1bfe9e3c79]*/ { - PyObject *registry, *cache, *negative_cache; + PyObject *registry, *cache, *negative_cache, *cache_version; _abc_data *impl = _get_impl(self); if (impl == NULL) { return NULL; @@ -262,13 +260,19 @@ _abc__get_dump(PyObject *module, PyObject *self) Py_DECREF(cache); return NULL; } - Py_INCREF(impl->_abc_negative_cache_version); /* PyTuple_Pack doesn't do this. */ - PyObject *res = PyTuple_Pack(4, - registry, cache, negative_cache, impl->_abc_negative_cache_version); + cache_version = PyLong_FromUnsignedLongLong(impl->_abc_negative_cache_version); + if (cache_version == NULL) { + Py_DECREF(impl); + Py_DECREF(registry); + Py_DECREF(cache); + Py_DECREF(negative_cache); + return NULL; + } + PyObject *res = PyTuple_Pack(4, registry, cache, negative_cache, cache_version); Py_DECREF(registry); Py_DECREF(cache); Py_DECREF(negative_cache); - Py_DECREF(impl->_abc_negative_cache_version); + Py_DECREF(cache_version); Py_DECREF(impl); return res; } @@ -475,16 +479,7 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) Py_DECREF(impl); /* Invalidate negative cache */ - PyObject *one = PyLong_FromLong(1); - if (one == NULL) { - return NULL; - } - PyObject *next_version = PyNumber_Add(abc_invalidation_counter, one); - Py_DECREF(one); - if (next_version == NULL) { - return NULL; - } - Py_SETREF(abc_invalidation_counter, next_version); + abc_invalidation_counter++; Py_INCREF(subclass); return subclass; @@ -525,11 +520,7 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { - int r = PyObject_RichCompareBool( - impl->_abc_negative_cache_version, - abc_invalidation_counter, Py_EQ); - assert(r >= 0); // Both should be PyLong - if (r > 0) { + if (impl->_abc_negative_cache_version == abc_invalidation_counter) { incache = _in_weak_set(impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; @@ -614,19 +605,13 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } /* 2. Check negative cache; may have to invalidate. */ - int r = PyObject_RichCompareBool(impl->_abc_negative_cache_version, - abc_invalidation_counter, Py_LT); - assert(r >= 0); // Both should be PyLong - if (r > 0) { + if (impl->_abc_negative_cache_version < abc_invalidation_counter) { /* Invalidate the negative cache. */ if (impl->_abc_negative_cache != NULL && PySet_Clear(impl->_abc_negative_cache) < 0) { goto end; } - /* INCREF the new value of cache version, - then carefully DECREF the old one. */ - Py_INCREF(abc_invalidation_counter); - Py_SETREF(impl->_abc_negative_cache_version, abc_invalidation_counter); + impl->_abc_negative_cache_version = abc_invalidation_counter; } else { incache = _in_weak_set(impl->_abc_negative_cache, subclass); @@ -811,8 +796,7 @@ static PyObject * _abc_get_cache_token_impl(PyObject *module) /*[clinic end generated code: output=c7d87841e033dacc input=1d49ab7218687f59]*/ { - Py_INCREF(abc_invalidation_counter); - return abc_invalidation_counter; + return PyLong_FromUnsignedLongLong(abc_invalidation_counter); } static struct PyMethodDef module_functions[] = { @@ -848,6 +832,5 @@ PyInit__abc(void) } _abc_data_type.tp_doc = abc_data_doc; - abc_invalidation_counter = PyLong_FromLong(0); return PyModule_Create(&_abcmodule); } From 079e3be4768361f54a9da78cd261a008da2f579e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 17 Feb 2018 21:33:09 +0000 Subject: [PATCH 90/93] The rest of the comments --- Modules/_abc.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index 963ba0eb766234..d93bc2f880fd32 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -93,7 +93,7 @@ _get_impl(PyObject *self) static int _in_weak_set(PyObject *set, PyObject *obj) { - if (set == NULL || PySet_Size(set) == 0) { + if (set == NULL || PySet_GET_SIZE(set) == 0) { return 0; } PyObject *ref = PyWeakref_NewRef(obj, NULL); @@ -148,6 +148,10 @@ _add_to_weak_set(PyObject **pset, PyObject *obj) return -1; } destroy_cb = PyCFunction_NewEx(&_destroy_def, wr, NULL); + if (destroy_cb == NULL) { + Py_DECREF(wr); + return -1; + } ref = PyWeakref_NewRef(obj, destroy_cb); Py_DECREF(destroy_cb); if (ref == NULL) { @@ -374,7 +378,8 @@ compute_abstract_methods(PyObject *self) int is_abstract = _PyObject_IsAbstract(value); Py_DECREF(value); if (is_abstract < 0 || - (is_abstract && PySet_Add(abstracts, key) < 0)) { + (is_abstract && PySet_Add(abstracts, key) < 0)) + { Py_DECREF(key); Py_DECREF(iter); goto error; @@ -406,7 +411,7 @@ _abc._abc_init self: object / -Internal ABC helper for class set-up. Should be never used outside abc module +Internal ABC helper for class set-up. Should be never used outside abc module. [clinic start generated code]*/ static PyObject * @@ -438,7 +443,7 @@ _abc._abc_register subclass: object / -Internal ABC helper for subclasss registration. Should be never used outside abc module +Internal ABC helper for subclasss registration. Should be never used outside abc module. [clinic start generated code]*/ static PyObject * @@ -493,7 +498,7 @@ _abc._abc_instancecheck instance: object / -Internal ABC helper for instance checks. Should be never used outside abc module +Internal ABC helper for instance checks. Should be never used outside abc module. [clinic start generated code]*/ static PyObject * @@ -508,6 +513,9 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, } subclass = _PyObject_GetAttrId(instance, &PyId___class__); + if (subclass == NULL) { + return NULL; + } /* Inline the cache checking. */ int incache = _in_weak_set(impl->_abc_cache, subclass); if (incache < 0) { @@ -578,7 +586,7 @@ _abc._abc_subclasscheck subclass: object / -Internal ABC helper for subclasss checks. Should be never used outside abc module +Internal ABC helper for subclasss checks. Should be never used outside abc module. [clinic start generated code]*/ static PyObject * @@ -657,8 +665,8 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, /* 4. Check if it's a direct subclass. */ mro = ((PyTypeObject *)subclass)->tp_mro; assert(PyTuple_Check(mro)); - for (pos = 0; pos < PyTuple_Size(mro); pos++) { - PyObject *mro_item = PyTuple_GetItem(mro, pos); + for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { + PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); if (mro_item == NULL) { goto end; } @@ -679,7 +687,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, /* 6. Check if it's a subclass of a subclass (recursive). */ subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); - if(!PyList_Check(subclasses)) { + if (!PyList_Check(subclasses)) { PyErr_SetString(PyExc_TypeError, "__subclasses__() must return a list"); goto end; } @@ -734,7 +742,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return 0; } // Weakref callback may remove entry from set. - // Se we take snapshot of registry first. + // So we take snapshot of registry first. PyObject **copy = PyMem_Malloc(sizeof(PyObject*) * registry_size); PyObject *key; Py_ssize_t pos = 0; @@ -789,7 +797,7 @@ Returns the current ABC cache token. The token is an opaque object (supporting equality testing) identifying the current version of the ABC cache for virtual subclasses. The token changes -with every call to ``register()`` on any ABC. +with every call to register() on any ABC. [clinic start generated code]*/ static PyObject * From 41465888ec5a48b1f45ce9d5c4f296c654b3d65b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 17 Feb 2018 21:46:41 +0000 Subject: [PATCH 91/93] Regenerate clinics. --- Modules/_abc.c | 10 +++++----- Modules/clinic/_abc.c.h | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Modules/_abc.c b/Modules/_abc.c index d93bc2f880fd32..dd32876b7bd71f 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -416,7 +416,7 @@ Internal ABC helper for class set-up. Should be never used outside abc module. static PyObject * _abc__abc_init(PyObject *module, PyObject *self) -/*[clinic end generated code: output=594757375714cda1 input=6061a045416da18b]*/ +/*[clinic end generated code: output=594757375714cda1 input=8d7fe470ff77f029]*/ { PyObject *data; if (compute_abstract_methods(self) < 0) { @@ -448,7 +448,7 @@ Internal ABC helper for subclasss registration. Should be never used outside abc static PyObject * _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) -/*[clinic end generated code: output=7851e7668c963524 input=4b94c86e77e9e901]*/ +/*[clinic end generated code: output=7851e7668c963524 input=ca589f8c3080e67f]*/ { if (!PyType_Check(subclass)) { PyErr_SetString(PyExc_TypeError, "Can only register classes"); @@ -504,7 +504,7 @@ Internal ABC helper for instance checks. Should be never used outside abc module static PyObject * _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, PyObject *instance) -/*[clinic end generated code: output=b8b5148f63b6b56f input=5b15699272795a93]*/ +/*[clinic end generated code: output=b8b5148f63b6b56f input=a4f4525679261084]*/ { PyObject *subtype, *result = NULL, *subclass = NULL; _abc_data *impl = _get_impl(self); @@ -592,7 +592,7 @@ Internal ABC helper for subclasss checks. Should be never used outside abc modul static PyObject * _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) -/*[clinic end generated code: output=b56c9e4a530e3894 input=4c87faea511976a8]*/ +/*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ { PyObject *ok, *mro, *subclasses = NULL, *result = NULL; Py_ssize_t pos; @@ -802,7 +802,7 @@ with every call to register() on any ABC. static PyObject * _abc_get_cache_token_impl(PyObject *module) -/*[clinic end generated code: output=c7d87841e033dacc input=1d49ab7218687f59]*/ +/*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ { return PyLong_FromUnsignedLongLong(abc_invalidation_counter); } diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h index 4428dc2ab143d2..b1ec371d155159 100644 --- a/Modules/clinic/_abc.c.h +++ b/Modules/clinic/_abc.c.h @@ -41,7 +41,7 @@ PyDoc_STRVAR(_abc__abc_init__doc__, "_abc_init($module, self, /)\n" "--\n" "\n" -"Internal ABC helper for class set-up. Should be never used outside abc module"); +"Internal ABC helper for class set-up. Should be never used outside abc module."); #define _ABC__ABC_INIT_METHODDEF \ {"_abc_init", (PyCFunction)_abc__abc_init, METH_O, _abc__abc_init__doc__}, @@ -50,7 +50,7 @@ PyDoc_STRVAR(_abc__abc_register__doc__, "_abc_register($module, self, subclass, /)\n" "--\n" "\n" -"Internal ABC helper for subclasss registration. Should be never used outside abc module"); +"Internal ABC helper for subclasss registration. Should be never used outside abc module."); #define _ABC__ABC_REGISTER_METHODDEF \ {"_abc_register", (PyCFunction)_abc__abc_register, METH_FASTCALL, _abc__abc_register__doc__}, @@ -80,7 +80,7 @@ PyDoc_STRVAR(_abc__abc_instancecheck__doc__, "_abc_instancecheck($module, self, instance, /)\n" "--\n" "\n" -"Internal ABC helper for instance checks. Should be never used outside abc module"); +"Internal ABC helper for instance checks. Should be never used outside abc module."); #define _ABC__ABC_INSTANCECHECK_METHODDEF \ {"_abc_instancecheck", (PyCFunction)_abc__abc_instancecheck, METH_FASTCALL, _abc__abc_instancecheck__doc__}, @@ -111,7 +111,7 @@ PyDoc_STRVAR(_abc__abc_subclasscheck__doc__, "_abc_subclasscheck($module, self, subclass, /)\n" "--\n" "\n" -"Internal ABC helper for subclasss checks. Should be never used outside abc module"); +"Internal ABC helper for subclasss checks. Should be never used outside abc module."); #define _ABC__ABC_SUBCLASSCHECK_METHODDEF \ {"_abc_subclasscheck", (PyCFunction)_abc__abc_subclasscheck, METH_FASTCALL, _abc__abc_subclasscheck__doc__}, @@ -146,7 +146,7 @@ PyDoc_STRVAR(_abc_get_cache_token__doc__, "\n" "The token is an opaque object (supporting equality testing) identifying the\n" "current version of the ABC cache for virtual subclasses. The token changes\n" -"with every call to ``register()`` on any ABC."); +"with every call to register() on any ABC."); #define _ABC_GET_CACHE_TOKEN_METHODDEF \ {"get_cache_token", (PyCFunction)_abc_get_cache_token, METH_NOARGS, _abc_get_cache_token__doc__}, @@ -159,4 +159,4 @@ _abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _abc_get_cache_token_impl(module); } -/*[clinic end generated code: output=3e785943d168e371 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d6f861a8f45bc6f input=a9049054013a1b77]*/ From c1336051c4d8351f0556d00f05985ecc9f38eb88 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 17 Feb 2018 22:47:34 +0000 Subject: [PATCH 92/93] Two more comments --- Lib/abc.py | 1 + Modules/_abc.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/abc.py b/Lib/abc.py index a7705281256056..7094141277ae4f 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -107,6 +107,7 @@ def setx(self, value): ... _reset_registry, _reset_caches) except ImportError: from _py_abc import ABCMeta, get_cache_token + ABCMeta.__module__ = 'abc' else: class ABCMeta(type): """Metaclass for defining Abstract Base Classes (ABCs). diff --git a/Modules/_abc.c b/Modules/_abc.c index dd32876b7bd71f..7eaadc7a7cffe6 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -692,7 +692,10 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, goto end; } for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { - int r = PyObject_IsSubclass(subclass, PyList_GET_ITEM(subclasses, pos)); + PyObject *scls = PyList_GET_ITEM(subclasses, pos); + Py_INCREF(scls); + int r = PyObject_IsSubclass(subclass, scls); + Py_DECREF(scls); if (r > 0) { if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { goto end; From f82e04de3374a2729b7b2113f075f3700940a530 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 18 Feb 2018 11:31:07 +0000 Subject: [PATCH 93/93] Few more comments by Serhiy; add Whats New item --- Doc/whatsnew/3.7.rst | 6 ++++++ Modules/_abc.c | 41 ++++++++--------------------------------- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 7c98c7cfa38d54..c35f07c0eabf71 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -853,6 +853,12 @@ Optimizations * Constant folding is moved from peephole optimizer to new AST optimizer. (Contributed by Eugene Toder and INADA Naoki in :issue:`29469`) +* Most functions and methods in :mod:`abc` have been rewrittent in C. + This makes creation of abstract base classes, and calling :func:`isinstance` + and :func:`issubclass` on them 1.5x faster. This also reduces Python + start-up time by up to 10%. (Contributed by Ivan Levkivskyi and INADA Naoki + in :issue:`31333`) + Build and C API Changes ======================= diff --git a/Modules/_abc.c b/Modules/_abc.c index 7eaadc7a7cffe6..504e23d9a74d3e 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -241,42 +241,15 @@ static PyObject * _abc__get_dump(PyObject *module, PyObject *self) /*[clinic end generated code: output=9d9569a8e2c1c443 input=2c5deb1bfe9e3c79]*/ { - PyObject *registry, *cache, *negative_cache, *cache_version; _abc_data *impl = _get_impl(self); if (impl == NULL) { return NULL; } - registry = PySet_New(impl->_abc_registry); - if (registry == NULL) { - Py_DECREF(impl); - return NULL; - } - cache = PySet_New(impl->_abc_cache); - if (cache == NULL) { - Py_DECREF(impl); - Py_DECREF(registry); - return NULL; - } - negative_cache = PySet_New(impl->_abc_negative_cache); - if (negative_cache == NULL) { - Py_DECREF(impl); - Py_DECREF(registry); - Py_DECREF(cache); - return NULL; - } - cache_version = PyLong_FromUnsignedLongLong(impl->_abc_negative_cache_version); - if (cache_version == NULL) { - Py_DECREF(impl); - Py_DECREF(registry); - Py_DECREF(cache); - Py_DECREF(negative_cache); - return NULL; - } - PyObject *res = PyTuple_Pack(4, registry, cache, negative_cache, cache_version); - Py_DECREF(registry); - Py_DECREF(cache); - Py_DECREF(negative_cache); - Py_DECREF(cache_version); + PyObject *res = Py_BuildValue("NNNK", + PySet_New(impl->_abc_registry), + PySet_New(impl->_abc_cache), + PySet_New(impl->_abc_negative_cache), + impl->_abc_negative_cache_version); Py_DECREF(impl); return res; } @@ -514,6 +487,7 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, subclass = _PyObject_GetAttrId(instance, &PyId___class__); if (subclass == NULL) { + Py_DECREF(impl); return NULL; } /* Inline the cache checking. */ @@ -616,7 +590,8 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, if (impl->_abc_negative_cache_version < abc_invalidation_counter) { /* Invalidate the negative cache. */ if (impl->_abc_negative_cache != NULL && - PySet_Clear(impl->_abc_negative_cache) < 0) { + PySet_Clear(impl->_abc_negative_cache) < 0) + { goto end; } impl->_abc_negative_cache_version = abc_invalidation_counter;