gh-111489: Add PyList_FromArrayMoveRef() function

vstinner · vstinner · commit 7b20ab09d01c · 2023-12-11T19:00:13.000+01:00
Add new functions to create tuple and list from arrays:

* PyTuple_FromArray().
* PyTuple_FromArrayMoveRef().
* PyList_FromArrayMoveRef().

Add tests on new functions.
diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst
@@ -43,6 +43,19 @@ List Objects
       setting all items to a real object with :c:func:`PyList_SetItem`.
 
 
+.. c:function:: PyObject* PyList_FromArrayMoveRef(PyObject **array, Py_ssize_t size)
+
+   Create a list of *size* items and move *array* :term:`strong references
+   <strong reference>` to the new list.
+
+   * Return a new reference on success. *array* references become
+     :term:`borrowed references <borrowed reference>`.
+   * Set an exception and return ``NULL`` on error. *array* is left unchanged,
+     the caller should clear *array* strong references.
+
+   .. versionadded:: 3.13
+
+
 .. c:function:: Py_ssize_t PyList_Size(PyObject *list)
 
    .. index:: pair: built-in function; len
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
@@ -36,6 +36,30 @@ Tuple Objects
    Return a new tuple object of size *len*, or ``NULL`` on failure.
 
 
+.. c:function:: PyObject* PyTuple_FromArray(PyObject *const *array, Py_ssize_t size)
+
+   Create a tuple of *size* items and copy references from *array* to the new
+   tuple.
+
+   * Return a new reference on success.
+   * Set an exception and return NULL on error.
+
+   .. versionadded:: 3.13
+
+
+.. c:function:: PyObject* PyTuple_FromArrayMoveRef(PyObject **array, Py_ssize_t size)
+
+   Create a tuple of *size* items and move *array* :term:`strong references
+   <strong reference>` to the new tuple.
+
+   * Return a new reference on success. *array* references become :term:`borrowed
+     references <borrowed reference>`.
+   * Set an exception and return ``NULL`` on error. *array* is left unchanged,
+     the caller should clear *array* strong references.
+
+   .. versionadded:: 3.13
+
+
 .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...)
 
    Return a new tuple object of size *n*, or ``NULL`` on failure. The tuple values
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
@@ -1269,6 +1269,14 @@ New Features
 * Add :c:func:`Py_HashPointer` function to hash a pointer.
   (Contributed by Victor Stinner in :gh:`111545`.)
 
+* Add new functions to create :class:`tuple` and :class:`list` from arrays:
+
+  * :c:func:`PyTuple_FromArray`.
+  * :c:func:`PyTuple_FromArrayMoveRef`.
+  * :c:func:`PyList_FromArrayMoveRef`.
+
+  (Contributed by Victor Stinner in :gh:`111489`.)
+
 
 Porting to Python 3.13
 ----------------------
diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h
@@ -47,3 +47,6 @@ PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
 
 PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable);
 PyAPI_FUNC(int) PyList_Clear(PyObject *self);
+PyAPI_FUNC(PyObject*) PyList_FromArrayMoveRef(
+    PyObject *const *array,
+    Py_ssize_t size);
diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h
@@ -36,3 +36,10 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
 }
 #define PyTuple_SET_ITEM(op, index, value) \
     PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value))
+
+PyAPI_FUNC(PyObject*) PyTuple_FromArray(
+    PyObject *const *array,
+    Py_ssize_t size);
+PyAPI_FUNC(PyObject*) PyTuple_FromArrayMoveRef(
+    PyObject *const *array,
+    Py_ssize_t size);
diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h
@@ -77,7 +77,9 @@ typedef struct {
     PyListObject *it_seq; /* Set to NULL when iterator is exhausted */
 } _PyListIterObject;
 
-extern PyObject *_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n);
+extern PyObject *_PyList_FromArraySteal(
+    PyObject *const *array,
+    Py_ssize_t size);
 
 #ifdef __cplusplus
 }
diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h
@@ -63,8 +63,11 @@ struct _Py_tuple_state {
 
 #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item)
 
-extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t);
-extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t);
+// Alias for backward compatibility
+#define _PyTuple_FromArray PyTuple_FromArray
+extern PyObject *_PyTuple_FromArraySteal(
+    PyObject *const *array,
+    Py_ssize_t size);
 
 
 typedef struct {
diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py
@@ -342,3 +342,11 @@ def test_list_extend(self):
 
         # CRASHES list_extend(NULL, [])
         # CRASHES list_extend([], NULL)
+
+    def test_list_fromarraymoveref(self):
+        # Test PyList_FromArrayMoveRef()
+        list_fromarraymoveref = _testcapi.list_fromarraymoveref
+
+        lst = [object() for _ in range(5)]
+        copy = list_fromarraymoveref(lst)
+        self.assertEqual(copy, lst)
diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py
@@ -0,0 +1,21 @@
+import unittest
+from test.support import import_helper
+_testcapi = import_helper.import_module('_testcapi')
+
+
+class CAPITest(unittest.TestCase):
+    def test_tuple_fromarray(self):
+        # Test PyTuple_FromArray()
+        tuple_fromarray = _testcapi.tuple_fromarraymoveref
+
+        tup = tuple(object() for _ in range(5))
+        copy = tuple_fromarray(tup)
+        self.assertEqual(copy, tup)
+
+    def test_tuple_fromarraymoveref(self):
+        # Test PyTuple_FromArrayMoveRef()
+        tuple_fromarraymoveref = _testcapi.tuple_fromarraymoveref
+
+        tup = tuple(object() for _ in range(5))
+        copy = tuple_fromarraymoveref(tup)
+        self.assertEqual(copy, tup)
diff --git a/Misc/NEWS.d/next/C API/2023-12-11-18-30-51.gh-issue-111489.S78zth.rst b/Misc/NEWS.d/next/C API/2023-12-11-18-30-51.gh-issue-111489.S78zth.rst
@@ -0,0 +1,7 @@
+Add new functions to create :class:`tuple` and :class:`list` from arrays:
+
+* :c:func:`PyTuple_FromArray`.
+* :c:func:`PyTuple_FromArrayMoveRef`.
+* :c:func:`PyList_FromArrayMoveRef`.
+
+Patch by Victor Stinner.
diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c
@@ -183,6 +183,45 @@ list_extend(PyObject* Py_UNUSED(module), PyObject *args)
 }
 
 
+static PyObject *
+list_fromarraymoveref(PyObject* Py_UNUSED(module), PyObject *args)
+{
+    PyObject *src;
+    if (!PyArg_ParseTuple(args, "O!", &PyList_Type, &src)) {
+        return NULL;
+    }
+
+    Py_ssize_t size = PyList_GET_SIZE(src);
+    PyObject **array = PyMem_Malloc(size * sizeof(PyObject**));
+    if (array == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    for (Py_ssize_t i = 0; i < size; i++) {
+        array[i] = Py_NewRef(PyList_GET_ITEM(src, i));
+    }
+
+    PyObject *list = PyList_FromArrayMoveRef(array, size);
+
+    for (Py_ssize_t i = 0; i < size; i++) {
+        // check that the array is not modified
+        assert(array[i] == PyList_GET_ITEM(src, i));
+
+        // check that the array was copied properly
+        assert(PyList_GET_ITEM(list, i) == array[i]);
+    }
+
+    if (list == NULL) {
+        for (Py_ssize_t i = 0; i < size; i++) {
+            Py_DECREF(array[i]);
+        }
+    }
+    PyMem_Free(array);
+
+    return list;
+}
+
+
 static PyMethodDef test_methods[] = {
     {"list_check", list_check, METH_O},
     {"list_check_exact", list_check_exact, METH_O},
@@ -202,6 +241,7 @@ static PyMethodDef test_methods[] = {
     {"list_astuple", list_astuple, METH_O},
     {"list_clear", list_clear, METH_O},
     {"list_extend", list_extend, METH_VARARGS},
+    {"list_fromarraymoveref", list_fromarraymoveref, METH_VARARGS},
     {NULL},
 };
 
diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c
@@ -2,7 +2,68 @@
 #include "util.h"
 
 
+static PyObject *
+tuple_fromarray_impl(PyObject *args, int move_ref)
+{
+    PyObject *src;
+    if (!PyArg_ParseTuple(args, "O!", &PyTuple_Type, &src)) {
+        return NULL;
+    }
+
+    Py_ssize_t size = PyTuple_GET_SIZE(src);
+    PyObject **array = PyMem_Malloc(size * sizeof(PyObject**));
+    if (array == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    for (Py_ssize_t i = 0; i < size; i++) {
+        array[i] = Py_NewRef(PyTuple_GET_ITEM(src, i));
+    }
+
+    PyObject *tuple;
+    if (move_ref) {
+        tuple = PyTuple_FromArrayMoveRef(array, size);
+    }
+    else {
+        tuple = PyTuple_FromArray(array, size);
+    }
+
+    for (Py_ssize_t i = 0; i < size; i++) {
+        // check that the array is not modified
+        assert(array[i] == PyTuple_GET_ITEM(src, i));
+
+        // check that the array was copied properly
+        assert(PyTuple_GET_ITEM(tuple, i) == array[i]);
+    }
+
+    if (!move_ref || tuple == NULL) {
+        for (Py_ssize_t i = 0; i < size; i++) {
+            Py_DECREF(array[i]);
+        }
+    }
+    PyMem_Free(array);
+
+    return tuple;
+}
+
+
+static PyObject *
+tuple_fromarray(PyObject* Py_UNUSED(module), PyObject *args)
+{
+    return tuple_fromarray_impl(args, 0);
+}
+
+
+static PyObject *
+tuple_fromarraymoveref(PyObject* Py_UNUSED(module), PyObject *args)
+{
+    return tuple_fromarray_impl(args, 1);
+}
+
+
 static PyMethodDef test_methods[] = {
+    {"tuple_fromarray", tuple_fromarray, METH_VARARGS},
+    {"tuple_fromarraymoveref", tuple_fromarraymoveref, METH_VARARGS},
     {NULL},
 };
 
diff --git a/Objects/listobject.c b/Objects/listobject.c
@@ -2622,25 +2622,36 @@ PyList_AsTuple(PyObject *v)
     return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v));
 }
 
-PyObject *
-_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n)
+PyObject*
+PyList_FromArrayMoveRef(PyObject *const *array, Py_ssize_t size)
 {
-    if (n == 0) {
+    if (size == 0) {
         return PyList_New(0);
     }
 
-    PyListObject *list = (PyListObject *)PyList_New(n);
-    if (list == NULL) {
-        for (Py_ssize_t i = 0; i < n; i++) {
-            Py_DECREF(src[i]);
-        }
+    PyObject *op = PyList_New(size);
+    if (op == NULL) {
         return NULL;
     }
+    PyListObject *list = (PyListObject *)op;
 
-    PyObject **dst = list->ob_item;
-    memcpy(dst, src, n * sizeof(PyObject *));
+    PyObject **items = list->ob_item;
+    // PyList_New(size) returns NULL is size*sizeof(PyObject*) is greater than
+    // PY_SSIZE_T_MAX. No need to check for integer overflow again.
+    memcpy(items, array, size * sizeof(PyObject *));
+    return op;
+}
 
-    return (PyObject *)list;
+PyObject *
+_PyList_FromArraySteal(PyObject *const *array, Py_ssize_t size)
+{
+    PyObject *list = PyList_FromArrayMoveRef(array, size);
+    if (list == NULL) {
+        for (Py_ssize_t i = 0; i < size; i++) {
+            Py_DECREF(array[i]);
+        }
+    }
+    return list;
 }
 
 /*[clinic input]
diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c

Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,9 @@ typedef struct {`
`77`	`77`	`PyListObject it_seq; / Set to NULL when iterator is exhausted */`
`78`	`78`	`} _PyListIterObject;`
`79`	`79`
`80`		`-extern PyObject _PyList_FromArraySteal(PyObject const *src, Py_ssize_t n);`
	`80`	`+extern PyObject *_PyList_FromArraySteal(`
	`81`	`+ PyObject const array,`
	`82`	`+ Py_ssize_t size);`
`81`	`83`
`82`	`84`	`#ifdef __cplusplus`
`83`	`85`	`}`