From 772e22a6d4e35e6eb49c67b1c3bf29881e32942d Mon Sep 17 00:00:00 2001 From: Porras Huang Date: Tue, 8 Feb 2022 19:19:41 -0800 Subject: [PATCH 1/8] Add bytearray to string cast, testcase and rename load_bytes to load_raw --- include/pybind11/cast.h | 19 ++++++++++++++----- tests/test_builtin_casters.py | 6 ++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 165102443c..75291dd61f 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -345,10 +345,10 @@ template struct string_caster { } if (!PyUnicode_Check(load_src.ptr())) { #if PY_MAJOR_VERSION >= 3 - return load_bytes(load_src); + return load_raw(load_src); #else if (std::is_same::value) { - return load_bytes(load_src); + return load_raw(load_src); } // The below is a guaranteed failure in Python 3 when PyUnicode_Check returns false @@ -421,11 +421,11 @@ template struct string_caster { #endif } - // When loading into a std::string or char*, accept a bytes object as-is (i.e. + // When loading into a std::string or char*, accept a bytes/bytearray object as-is (i.e. // without any encoding/decoding attempt). For other C++ char sizes this is a no-op. // which supports loading a unicode from a str, doesn't take this path. template - bool load_bytes(enable_if_t::value, handle> src) { + bool load_raw(enable_if_t::value, handle> src) { if (PYBIND11_BYTES_CHECK(src.ptr())) { // We were passed a Python 3 raw bytes; accept it into a std::string or char* // without any encoding attempt. @@ -435,12 +435,21 @@ template struct string_caster { return true; } } + if (PyByteArray_Check(src.ptr())) { + // We were passed a bytearray; accept it into a std::string or char* + // without any encoding attempt. + const char *bytes = PyByteArray_AsString(src.ptr()); + if (bytes) { + value = StringType(bytes, (size_t) PyByteArray_Size(src.ptr())); + return true; + } + } return false; } template - bool load_bytes(enable_if_t::value, handle>) { return false; } + bool load_raw(enable_if_t::value, handle>) { return false; } }; template diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index b1f1e395a7..c61793d5a2 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -136,6 +136,12 @@ def to_bytes(s): # passing in a utf8 encoded string should work assert m.string_length(u"💩".encode("utf8")) == 4 +def test_bytearray_to_string(): + """Tests the ability to pass bytearray to C++ string-accepting functions. Note that this is + one-way: the only way to return bytes to Python is via the pybind11::bytearray class.""" + # Issue #2799 + assert m.string_length(bytearray(b'Hi')) == 2 + assert m.strlen(bytearray(b'bytearray')) == 9 @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no ") def test_string_view(capture): From 521e5700c78e2d78b9206dfa0361a0db5ec17034 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 06:19:58 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_builtin_casters.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index c61793d5a2..df1eb890ef 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -136,12 +136,14 @@ def to_bytes(s): # passing in a utf8 encoded string should work assert m.string_length(u"💩".encode("utf8")) == 4 + def test_bytearray_to_string(): """Tests the ability to pass bytearray to C++ string-accepting functions. Note that this is one-way: the only way to return bytes to Python is via the pybind11::bytearray class.""" - # Issue #2799 - assert m.string_length(bytearray(b'Hi')) == 2 - assert m.strlen(bytearray(b'bytearray')) == 9 + # Issue #2799 + assert m.string_length(bytearray(b"Hi")) == 2 + assert m.strlen(bytearray(b"bytearray")) == 9 + @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no ") def test_string_view(capture): From 4820aefc5ec62a76ce4784b7970f7b8106a29c00 Mon Sep 17 00:00:00 2001 From: Porras Huang Date: Mon, 14 Feb 2022 02:16:20 -0800 Subject: [PATCH 3/8] New bytearray test case and convert failure to pybind11_fail --- include/pybind11/cast.h | 24 ++++++------------------ tests/test_builtin_casters.py | 13 +++++++------ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 75291dd61f..46fbb680c1 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -344,21 +344,7 @@ template struct string_caster { return false; } if (!PyUnicode_Check(load_src.ptr())) { -#if PY_MAJOR_VERSION >= 3 return load_raw(load_src); -#else - if (std::is_same::value) { - return load_raw(load_src); - } - - // The below is a guaranteed failure in Python 3 when PyUnicode_Check returns false - if (!PYBIND11_BYTES_CHECK(load_src.ptr())) - return false; - - temp = reinterpret_steal(PyUnicode_FromObject(load_src.ptr())); - if (!temp) { PyErr_Clear(); return false; } - load_src = temp; -#endif } #if PY_VERSION_HEX >= 0x03030000 @@ -438,9 +424,9 @@ template struct string_caster { if (PyByteArray_Check(src.ptr())) { // We were passed a bytearray; accept it into a std::string or char* // without any encoding attempt. - const char *bytes = PyByteArray_AsString(src.ptr()); - if (bytes) { - value = StringType(bytes, (size_t) PyByteArray_Size(src.ptr())); + const char *bytearray = PyByteArray_AsString(src.ptr()); + if (bytearray) { + value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); return true; } } @@ -449,7 +435,9 @@ template struct string_caster { } template - bool load_raw(enable_if_t::value, handle>) { return false; } + bool load_raw(enable_if_t::value, handle>) { + return false; + } }; template diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index c61793d5a2..bc6c60b3f3 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -134,14 +134,15 @@ def to_bytes(s): assert m.strlen(to_bytes("a\x00b")) == 1 # C-string limitation # passing in a utf8 encoded string should work - assert m.string_length(u"💩".encode("utf8")) == 4 + assert m.string_length(u" ".encode("utf8")) == 4 def test_bytearray_to_string(): - """Tests the ability to pass bytearray to C++ string-accepting functions. Note that this is - one-way: the only way to return bytes to Python is via the pybind11::bytearray class.""" - # Issue #2799 - assert m.string_length(bytearray(b'Hi')) == 2 - assert m.strlen(bytearray(b'bytearray')) == 9 + """Tests the ability to pass bytearray to C++ string-accepting functions""" + assert m.string_length(bytearray(b"Hi")) == 2 + assert m.strlen(bytearray(b"bytearray")) == 9 + assert m.string_length(bytearray()) == 0 + assert m.string_length(bytearray(u"🦜", "utf-8", "strict")) == 4 + assert m.string_length(bytearray(b"\x80")) == 1 @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no ") def test_string_view(capture): From 5d0b3b7845c4034fa83c54c0fe2fb38054a2ddce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:40:15 +0000 Subject: [PATCH 4/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/pybind11/cast.h | 4 ++-- tests/test_builtin_casters.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index e79eaf9df2..2adf494b8c 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -486,8 +486,8 @@ struct string_caster { } template - bool load_raw(enable_if_t::value, handle>) { - return false; + bool load_raw(enable_if_t::value, handle>) { + return false; } }; diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index b65a6beb50..e3f14dee47 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -131,14 +131,14 @@ def test_bytes_to_string(): # passing in a utf8 encoded string should work <<<<<<< HEAD - assert m.string_length(u" ".encode("utf8")) == 4 + assert m.string_length(b" ") == 4 def test_bytearray_to_string(): """Tests the ability to pass bytearray to C++ string-accepting functions""" assert m.string_length(bytearray(b"Hi")) == 2 assert m.strlen(bytearray(b"bytearray")) == 9 assert m.string_length(bytearray()) == 0 - assert m.string_length(bytearray(u"🦜", "utf-8", "strict")) == 4 + assert m.string_length(bytearray("🦜", "utf-8", "strict")) == 4 assert m.string_length(bytearray(b"\x80")) == 1 ======= assert m.string_length("💩".encode()) == 4 From 4e18f829a592d55dc212a24f4d5f0f4b009d860b Mon Sep 17 00:00:00 2001 From: Porras Huang Date: Mon, 14 Feb 2022 02:41:18 -0800 Subject: [PATCH 5/8] Fix merge comments --- tests/test_builtin_casters.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index b65a6beb50..2b3f417d91 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -130,8 +130,7 @@ def test_bytes_to_string(): assert m.strlen("a\x00b".encode()) == 1 # C-string limitation # passing in a utf8 encoded string should work -<<<<<<< HEAD - assert m.string_length(u" ".encode("utf8")) == 4 + assert m.string_length("💩".encode()) == 4 def test_bytearray_to_string(): """Tests the ability to pass bytearray to C++ string-accepting functions""" @@ -140,18 +139,6 @@ def test_bytearray_to_string(): assert m.string_length(bytearray()) == 0 assert m.string_length(bytearray(u"🦜", "utf-8", "strict")) == 4 assert m.string_length(bytearray(b"\x80")) == 1 -======= - assert m.string_length("💩".encode()) == 4 - - -def test_bytearray_to_string(): - """Tests the ability to pass bytearray to C++ string-accepting functions. Note that this is - one-way: the only way to return bytes to Python is via the pybind11::bytearray class.""" - # Issue #2799 - assert m.string_length(bytearray(b"Hi")) == 2 - assert m.strlen(bytearray(b"bytearray")) == 9 - ->>>>>>> ac37829e7b1c12184f95c5633f102cf77b513c94 @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no ") def test_string_view(capture): From c64beb2b801a742ea4a3f3989afd2cda4406c199 Mon Sep 17 00:00:00 2001 From: Porras Huang Date: Mon, 14 Feb 2022 02:56:12 -0800 Subject: [PATCH 6/8] Actually fix merge comments --- tests/test_builtin_casters.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index 270ef4e09b..2b3f417d91 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -130,19 +130,14 @@ def test_bytes_to_string(): assert m.strlen("a\x00b".encode()) == 1 # C-string limitation # passing in a utf8 encoded string should work -<<<<<<< HEAD assert m.string_length("💩".encode()) == 4 -======= -<<<<<<< HEAD - assert m.string_length(b" ") == 4 ->>>>>>> 5d0b3b7845c4034fa83c54c0fe2fb38054a2ddce def test_bytearray_to_string(): """Tests the ability to pass bytearray to C++ string-accepting functions""" assert m.string_length(bytearray(b"Hi")) == 2 assert m.strlen(bytearray(b"bytearray")) == 9 assert m.string_length(bytearray()) == 0 - assert m.string_length(bytearray("🦜", "utf-8", "strict")) == 4 + assert m.string_length(bytearray(u"🦜", "utf-8", "strict")) == 4 assert m.string_length(bytearray(b"\x80")) == 1 @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no ") From b8d3b6ae39ac036220a5f6ce1ed82fb587698536 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:57:08 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_builtin_casters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index 2b3f417d91..d38ae68028 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -132,14 +132,16 @@ def test_bytes_to_string(): # passing in a utf8 encoded string should work assert m.string_length("💩".encode()) == 4 + def test_bytearray_to_string(): """Tests the ability to pass bytearray to C++ string-accepting functions""" assert m.string_length(bytearray(b"Hi")) == 2 assert m.strlen(bytearray(b"bytearray")) == 9 assert m.string_length(bytearray()) == 0 - assert m.string_length(bytearray(u"🦜", "utf-8", "strict")) == 4 + assert m.string_length(bytearray("🦜", "utf-8", "strict")) == 4 assert m.string_length(bytearray(b"\x80")) == 1 + @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no ") def test_string_view(capture): """Tests support for C++17 string_view arguments and return values""" From 19d01241f4e8f91dfead776280fc4e60b186f652 Mon Sep 17 00:00:00 2001 From: Porras Huang Date: Mon, 14 Feb 2022 03:59:18 -0800 Subject: [PATCH 8/8] Assert early if AsString fails --- include/pybind11/cast.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 2adf494b8c..fad65f0fae 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -467,19 +467,21 @@ struct string_caster { // We were passed raw bytes; accept it into a std::string or char* // without any encoding attempt. const char *bytes = PYBIND11_BYTES_AS_STRING(src.ptr()); - if (bytes) { - value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); - return true; + if (!bytes) { + pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); } + value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + return true; } if (PyByteArray_Check(src.ptr())) { // We were passed a bytearray; accept it into a std::string or char* // without any encoding attempt. const char *bytearray = PyByteArray_AsString(src.ptr()); - if (bytearray) { - value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); - return true; + if (!bytearray) { + pybind11_fail("Unexpected PyByteArray_AsString() failure."); } + value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + return true; } return false;