Change array_caster in pybind11/stl.h to support value types that are not default-constructible. (#30034)

* Reproducer for `error: call to implicitly-deleted default constructor` triggered by stl.h `array_caster`:

```
clang++ -o pybind11/tests/test_stl_no_default_ctor.os -c -std=c++17 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.10 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_stl_no_default_ctor.cpp
In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_stl_no_default_ctor.cpp:1:
In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/stl.h:12:
In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:13:
In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/detail/class.h:12:
In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/attr.h:14:
In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:18:
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/detail/type_caster_odr_guard.h:111:5: error: call to implicitly-deleted default constructor of 'pybind11::detail::type_caster<std::array<pybind11_tests::stl_no_default_ctor::Node, 2>>'
    type_caster_odr_guard() {
    ^
/usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/tuple:190:9: note: in instantiation of member function 'pybind11::detail::type_caster_odr_guard<std::array<pybind11_tests::stl_no_default_ctor::Node, 2>, pybind11::detail::type_caster<std::array<pybind11_tests::stl_no_default_ctor::Node, 2>>>::type_caster_odr_guard' requested here
      : _M_head_impl() { }
        ^
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:104:9: note: in instantiation of function template specialization 'pybind11::cpp_function::initialize<(lambda at /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1583:29), void, pybind11_tests::stl_no_default_ctor::NodeArray &, const std::array<pybind11_tests::stl_no_default_ctor::Node, 2> &, pybind11::is_method>' requested here
        initialize(
        ^
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1583:16: note: in instantiation of function template specialization 'pybind11::cpp_function::cpp_function<(lambda at /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1583:29), pybind11::is_method, void>' requested here
        return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl));
               ^
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1926:54: note: in instantiation of function template specialization 'pybind11::property_cpp_function<pybind11_tests::stl_no_default_ctor::NodeArray, std::array<pybind11_tests::stl_no_default_ctor::Node, 2>>::write<std::array<pybind11_tests::stl_no_default_ctor::Node, 2> pybind11_tests::stl_no_default_ctor::NodeArray::*, 0>' requested here
                     property_cpp_function<type, D>::write(pm, *this),
                                                     ^
/usr/local/google/home/rwgk/forked/pybind11/tests/test_stl_no_default_ctor.cpp:32:10: note: in instantiation of function template specialization 'pybind11::class_<pybind11_tests::stl_no_default_ctor::NodeArray>::def_readwrite<pybind11_tests::stl_no_default_ctor::NodeArray, std::array<pybind11_tests::stl_no_default_ctor::Node, 2>>' requested here
        .def_readwrite("arr", &NodeArray::arr);
         ^
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/stl.h:288:7: note: default constructor of 'type_caster<std::array<pybind11_tests::stl_no_default_ctor::Node, 2>>' is implicitly deleted because base class 'array_caster<std::array<Node, 2UL>, pybind11_tests::stl_no_default_ctor::Node, false, 2UL>' has a deleted default constructor
    : array_caster<std::array<Type, Size>, Type, false, Size> {};
      ^
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/stl.h:278:5: note: default constructor of 'array_caster<std::array<pybind11_tests::stl_no_default_ctor::Node, 2>, pybind11_tests::stl_no_default_ctor::Node, false, 2>' is implicitly deleted because field 'value' has a deleted default constructor
    PYBIND11_TYPE_CASTER_RVPP(ArrayType,
    ^
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:262:5: note: expanded from macro 'PYBIND11_TYPE_CASTER_RVPP'
    PYBIND11_TYPE_CASTER_IMPL(type, py_name, ::pybind11::return_value_policy_pack)
    ^
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:232:10: note: expanded from macro 'PYBIND11_TYPE_CASTER_IMPL'
    type value;                                                                                   \
         ^
/usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/array:115:56: note: default constructor of 'array<pybind11_tests::stl_no_default_ctor::Node, 2>' is implicitly deleted because field '_M_elems' has no default constructor
      typename _AT_Type::_Type                         _M_elems;
                                                       ^
1 error generated.
```

* Expand `PYBIND11_TYPE_CASTER_RVPP` macro.

* Resolve clang-tidy and clang <= 5 errors.

* Second try: Resolve clang-tidy and clang <= 5 errors.

* First working version of `array_caster` that works for types without a default constructor.

* Try `()` instead of `{}` to see if that resolves clang 3.6 and gcc 4.8.5 compilation errors.

* Add minor comment.

* Avoid `temp` storage if `Resizable` is `true`.

* Merge test_stl_no_default_ctor into test_stl

* Minor move of code blocks (no functional change).

* Remove unused default value.

* IncludeCleaner (IWYU) fixes

* Add two comments: preserving existing behavior

Author: rwgk

include/pybind11/stl.h

@@ -11,11 +11,14 @@
 
 #include "pybind11.h"
 #include "detail/common.h"
+#include "detail/descr.h"
+#include "detail/type_caster_base.h"
 
 #include <deque>
 #include <initializer_list>
 #include <list>
 #include <map>
+#include <memory>
 #include <ostream>
 #include <set>
 #include <unordered_map>
@@ -352,36 +355,65 @@ struct type_caster<std::deque<Type, Alloc>> : list_caster<std::deque<Type, Alloc
 template <typename Type, typename Alloc>
 struct type_caster<std::list<Type, Alloc>> : list_caster<std::list<Type, Alloc>, Type> {};
 
+template <typename ArrayType, typename V, size_t... I>
+ArrayType vector_to_array_impl(V &&v, index_sequence<I...>) {
+    return {{std::move(v[I])...}};
+}
+
+// Based on https://en.cppreference.com/w/cpp/container/array/to_array
+template <typename ArrayType, size_t N, typename V>
+ArrayType vector_to_array(V &&v) {
+    return vector_to_array_impl<ArrayType, V>(std::forward<V>(v), make_index_sequence<N>{});
+}
+
 template <typename ArrayType, typename Value, bool Resizable, size_t Size = 0>
 struct array_caster {
     using value_conv = make_caster<Value>;
 
 private:
-    template <bool R = Resizable>
-    bool require_size(enable_if_t<R, size_t> size) {
-        if (value.size() != size) {
-            value.resize(size);
+    std::unique_ptr<ArrayType> value;
+
+    template <bool R = Resizable, enable_if_t<R, int> = 0>
+    bool convert_elements(handle seq, bool convert) {
+        auto l = reinterpret_borrow<sequence>(seq);
+        value.reset(new ArrayType{});
+        // Using `resize` to preserve the behavior exactly as it was before google/pywrapcc#30034
+        // For the `resize` to work, `Value` must be default constructible.
+        // For `std::valarray`, this is a requirement:
+        // https://en.cppreference.com/w/cpp/named_req/NumericType
+        value->resize(l.size());
+        size_t ctr = 0;
+        for (const auto &it : l) {
+            value_conv conv;
+            if (!conv.load(it, convert)) {
+                return false;
+            }
+            (*value)[ctr++] = cast_op<Value &&>(std::move(conv));
         }
         return true;
     }
-    template <bool R = Resizable>
-    bool require_size(enable_if_t<!R, size_t> size) {
-        return size == Size;
-    }
 
+    template <bool R = Resizable, enable_if_t<!R, int> = 0>
     bool convert_elements(handle seq, bool convert) {
         auto l = reinterpret_borrow<sequence>(seq);
-        if (!require_size(l.size())) {
+        if (l.size() != Size) {
             return false;
         }
-        size_t ctr = 0;
-        for (const auto &it : l) {
+        // The `temp` storage is needed to support `Value` types that are not
+        // default-constructible.
+        // Deliberate choice: no template specializations, for simplicity, and
+        // because the compile time overhead for the specializations is deemed
+        // more significant than the runtime overhead for the `temp` storage.
+        std::vector<Value> temp;
+        temp.reserve(l.size());
+        for (auto it : l) {
             value_conv conv;
             if (!conv.load(it, convert)) {
                 return false;
             }
-            value[ctr++] = cast_op<Value &&>(std::move(conv));
+            temp.emplace_back(cast_op<Value &&>(std::move(conv)));
         }
+        value.reset(new ArrayType(vector_to_array<ArrayType, Size>(std::move(temp))));
         return true;
     }
 
@@ -420,13 +452,36 @@ struct array_caster {
         return l.release();
     }
 
-    PYBIND11_TYPE_CASTER_RVPP(ArrayType,
-                              const_name<Resizable>(const_name(""), const_name("Annotated["))
-                                  + const_name("list[") + value_conv::name + const_name("]")
-                                  + const_name<Resizable>(const_name(""),
-                                                          const_name(", FixedSize(")
-                                                              + const_name<Size>()
-                                                              + const_name(")]")));
+    // Code copied from PYBIND11_TYPE_CASTER macro.
+    // Intentionally preserving the behavior exactly as it was before google/pywrapcc#30034
+    template <typename T_, enable_if_t<std::is_same<ArrayType, remove_cv_t<T_>>::value, int> = 0>
+    static handle cast(T_ *src, const return_value_policy_pack &policy, handle parent) {
+        if (!src) {
+            return none().release();
+        }
+        if (policy == return_value_policy::take_ownership) {
+            auto h = cast(std::move(*src), policy, parent);
+            delete src; // WARNING: Assumes `src` was allocated with `new`.
+            return h;
+        }
+        return cast(*src, policy, parent);
+    }
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator ArrayType *() { return &(*value); }
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator ArrayType &() { return *value; }
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator ArrayType &&() && { return std::move(*value); }
+
+    template <typename T_>
+    using cast_op_type = movable_cast_op_type<T_>;
+
+    static constexpr auto name
+        = const_name<Resizable>(const_name(""), const_name("Annotated[")) + const_name("list[")
+          + value_conv::name + const_name("]")
+          + const_name<Resizable>(
+              const_name(""), const_name(", FixedSize(") + const_name<Size>() + const_name(")]"));
 };
 
 template <typename Type, size_t Size>

tests/test_stl.cpp

@@ -201,6 +201,23 @@ TEST_SUBMODULE(stl, m) {
     m.def("cast_array", []() { return std::array<int, 2>{{1, 2}}; });
     m.def("load_array", [](const std::array<int, 2> &a) { return a[0] == 1 && a[1] == 2; });
 
+    struct NoDefaultCtor {
+        explicit constexpr NoDefaultCtor(int val) : val{val} {}
+        int val;
+    };
+
+    struct NoDefaultCtorArray {
+        explicit constexpr NoDefaultCtorArray(int i)
+            : arr{{NoDefaultCtor(10 + i), NoDefaultCtor(20 + i)}} {}
+        std::array<NoDefaultCtor, 2> arr;
+    };
+
+    // test_array_no_default_ctor
+    py::class_<NoDefaultCtor>(m, "NoDefaultCtor").def_readonly("val", &NoDefaultCtor::val);
+    py::class_<NoDefaultCtorArray>(m, "NoDefaultCtorArray")
+        .def(py::init<int>())
+        .def_readwrite("arr", &NoDefaultCtorArray::arr);
+
     // test_valarray
     m.def("cast_valarray", []() { return std::valarray<int>{1, 4, 9}; });
     m.def("load_valarray", [](const std::valarray<int> &v) {

tests/test_stl.py

@@ -46,6 +46,13 @@ def test_array(doc):
     )
 
 
+def test_array_no_default_ctor():
+    lst = m.NoDefaultCtorArray(3)
+    assert [e.val for e in lst.arr] == [13, 23]
+    lst.arr = m.NoDefaultCtorArray(4).arr
+    assert [e.val for e in lst.arr] == [14, 24]
+
+
 def test_valarray(doc):
     """std::valarray <-> list"""
     lst = m.cast_valarray()