bpo-35134: Add Include/cpython/bytesobject.h file (GH-18494)

Add Include/cpython/bytearrayobject.h and
Include/cpython/bytesobject.h header files.

Move CPython C API from Include/bytesobject.h into a new
Include/cpython/bytesobject.h header file which is included by
Include/bytesobject.h. Do a similar change for
Include/bytearrayobject.h.

Author: vstinner

Include/bytearrayobject.h

@@ -18,17 +18,6 @@ extern "C" {
  * to contain a char pointer, not an unsigned char pointer.
  */
 
-/* Object layout */
-#ifndef Py_LIMITED_API
-typedef struct {
-    PyObject_VAR_HEAD
-    Py_ssize_t ob_alloc;   /* How many bytes allocated in ob_bytes */
-    char *ob_bytes;        /* Physical backing buffer */
-    char *ob_start;        /* Logical start inside ob_bytes */
-    Py_ssize_t ob_exports; /* How many buffer exports */
-} PyByteArrayObject;
-#endif
-
 /* Type object */
 PyAPI_DATA(PyTypeObject) PyByteArray_Type;
 PyAPI_DATA(PyTypeObject) PyByteArrayIter_Type;
@@ -45,14 +34,10 @@ PyAPI_FUNC(Py_ssize_t) PyByteArray_Size(PyObject *);
 PyAPI_FUNC(char *) PyByteArray_AsString(PyObject *);
 PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t);
 
-/* Macros, trading safety for speed */
 #ifndef Py_LIMITED_API
-#define PyByteArray_AS_STRING(self) \
-    (assert(PyByteArray_Check(self)), \
-     Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_start : _PyByteArray_empty_string)
-#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)), Py_SIZE(self))
-
-PyAPI_DATA(char) _PyByteArray_empty_string[];
+#  define Py_CPYTHON_BYTEARRAYOBJECT_H
+#  include  "cpython/bytearrayobject.h"
+#  undef Py_CPYTHON_BYTEARRAYOBJECT_H
 #endif
 
 #ifdef __cplusplus

Include/bytesobject.h

@@ -27,20 +27,6 @@ functions should be applied to nil objects.
 /* Caching the hash (ob_shash) saves recalculation of a string's hash value.
    This significantly speeds up dict lookups. */
 
-#ifndef Py_LIMITED_API
-typedef struct {
-    PyObject_VAR_HEAD
-    Py_hash_t ob_shash;
-    char ob_sval[1];
-
-    /* Invariants:
-     *     ob_sval contains space for 'ob_size+1' elements.
-     *     ob_sval[ob_size] == 0.
-     *     ob_shash is the hash of the string or -1 if not computed yet.
-     */
-} PyBytesObject;
-#endif
-
 PyAPI_DATA(PyTypeObject) PyBytes_Type;
 PyAPI_DATA(PyTypeObject) PyBytesIter_Type;
 
@@ -60,38 +46,9 @@ PyAPI_FUNC(char *) PyBytes_AsString(PyObject *);
 PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int);
 PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *);
 PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t);
-PyAPI_FUNC(PyObject*) _PyBytes_FormatEx(
-    const char *format,
-    Py_ssize_t format_len,
-    PyObject *args,
-    int use_bytearray);
-PyAPI_FUNC(PyObject*) _PyBytes_FromHex(
-    PyObject *string,
-    int use_bytearray);
-#endif
 PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t,
                                             const char *, Py_ssize_t,
                                             const char *);
-#ifndef Py_LIMITED_API
-/* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */
-PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t,
-                                             const char *, const char **);
-#endif
-
-/* Macro, trading safety for speed */
-#ifndef Py_LIMITED_API
-#define PyBytes_AS_STRING(op) (assert(PyBytes_Check(op)), \
-                                (((PyBytesObject *)(op))->ob_sval))
-#define PyBytes_GET_SIZE(op)  (assert(PyBytes_Check(op)),Py_SIZE(op))
-#endif
-
-/* _PyBytes_Join(sep, x) is like sep.join(x).  sep must be PyBytesObject*,
-   x must be an iterable object. */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x);
-#endif
 
 /* Provides access to the internal data buffer and size of a string
    object or the default encoded version of a Unicode object. Passing
@@ -114,85 +71,10 @@ PyAPI_FUNC(int) PyBytes_AsStringAndSize(
 #define F_ZERO  (1<<4)
 
 #ifndef Py_LIMITED_API
-/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer".
-   A _PyBytesWriter variable must be declared at the end of variables in a
-   function to optimize the memory allocation on the stack. */
-typedef struct {
-    /* bytes, bytearray or NULL (when the small buffer is used) */
-    PyObject *buffer;
-
-    /* Number of allocated size. */
-    Py_ssize_t allocated;
-
-    /* Minimum number of allocated bytes,
-       incremented by _PyBytesWriter_Prepare() */
-    Py_ssize_t min_size;
-
-    /* If non-zero, use a bytearray instead of a bytes object for buffer. */
-    int use_bytearray;
-
-    /* If non-zero, overallocate the buffer (default: 0).
-       This flag must be zero if use_bytearray is non-zero. */
-    int overallocate;
-
-    /* Stack buffer */
-    int use_small_buffer;
-    char small_buffer[512];
-} _PyBytesWriter;
-
-/* Initialize a bytes writer
-
-   By default, the overallocation is disabled. Set the overallocate attribute
-   to control the allocation of the buffer. */
-PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer);
-
-/* Get the buffer content and reset the writer.
-   Return a bytes object, or a bytearray object if use_bytearray is non-zero.
-   Raise an exception and return NULL on error. */
-PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer,
-    void *str);
-
-/* Deallocate memory of a writer (clear its internal buffer). */
-PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer);
-
-/* Allocate the buffer to write size bytes.
-   Return the pointer to the beginning of buffer data.
-   Raise an exception and return NULL on error. */
-PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer,
-    Py_ssize_t size);
-
-/* Ensure that the buffer is large enough to write *size* bytes.
-   Add size to the writer minimum size (min_size attribute).
-
-   str is the current pointer inside the buffer.
-   Return the updated current pointer inside the buffer.
-   Raise an exception and return NULL on error. */
-PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer,
-    void *str,
-    Py_ssize_t size);
-
-/* Resize the buffer to make it larger.
-   The new buffer may be larger than size bytes because of overallocation.
-   Return the updated current pointer inside the buffer.
-   Raise an exception and return NULL on error.
-
-   Note: size must be greater than the number of allocated bytes in the writer.
-
-   This function doesn't use the writer minimum size (min_size attribute).
-
-   See also _PyBytesWriter_Prepare().
-   */
-PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer,
-    void *str,
-    Py_ssize_t size);
-
-/* Write bytes.
-   Raise an exception and return NULL on error. */
-PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer,
-    void *str,
-    const void *bytes,
-    Py_ssize_t size);
-#endif   /* Py_LIMITED_API */
+#  define Py_CPYTHON_BYTESOBJECT_H
+#  include  "cpython/bytesobject.h"
+#  undef Py_CPYTHON_BYTESOBJECT_H
+#endif
 
 #ifdef __cplusplus
 }

Include/cpython/bytearrayobject.h

@@ -0,0 +1,20 @@
+#ifndef Py_CPYTHON_BYTEARRAYOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Object layout */
+typedef struct {
+    PyObject_VAR_HEAD
+    Py_ssize_t ob_alloc;   /* How many bytes allocated in ob_bytes */
+    char *ob_bytes;        /* Physical backing buffer */
+    char *ob_start;        /* Logical start inside ob_bytes */
+    Py_ssize_t ob_exports; /* How many buffer exports */
+} PyByteArrayObject;
+
+/* Macros, trading safety for speed */
+#define PyByteArray_AS_STRING(self) \
+    (assert(PyByteArray_Check(self)), \
+     Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_start : _PyByteArray_empty_string)
+#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)), Py_SIZE(self))
+
+PyAPI_DATA(char) _PyByteArray_empty_string[];

Include/cpython/bytesobject.h

@@ -0,0 +1,118 @@
+#ifndef Py_CPYTHON_BYTESOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+typedef struct {
+    PyObject_VAR_HEAD
+    Py_hash_t ob_shash;
+    char ob_sval[1];
+
+    /* Invariants:
+     *     ob_sval contains space for 'ob_size+1' elements.
+     *     ob_sval[ob_size] == 0.
+     *     ob_shash is the hash of the string or -1 if not computed yet.
+     */
+} PyBytesObject;
+
+PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t);
+PyAPI_FUNC(PyObject*) _PyBytes_FormatEx(
+    const char *format,
+    Py_ssize_t format_len,
+    PyObject *args,
+    int use_bytearray);
+PyAPI_FUNC(PyObject*) _PyBytes_FromHex(
+    PyObject *string,
+    int use_bytearray);
+
+/* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */
+PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t,
+                                             const char *, const char **);
+
+/* Macro, trading safety for speed */
+#define PyBytes_AS_STRING(op) (assert(PyBytes_Check(op)), \
+                                (((PyBytesObject *)(op))->ob_sval))
+#define PyBytes_GET_SIZE(op)  (assert(PyBytes_Check(op)),Py_SIZE(op))
+
+/* _PyBytes_Join(sep, x) is like sep.join(x).  sep must be PyBytesObject*,
+   x must be an iterable object. */
+PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x);
+
+
+/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer".
+   A _PyBytesWriter variable must be declared at the end of variables in a
+   function to optimize the memory allocation on the stack. */
+typedef struct {
+    /* bytes, bytearray or NULL (when the small buffer is used) */
+    PyObject *buffer;
+
+    /* Number of allocated size. */
+    Py_ssize_t allocated;
+
+    /* Minimum number of allocated bytes,
+       incremented by _PyBytesWriter_Prepare() */
+    Py_ssize_t min_size;
+
+    /* If non-zero, use a bytearray instead of a bytes object for buffer. */
+    int use_bytearray;
+
+    /* If non-zero, overallocate the buffer (default: 0).
+       This flag must be zero if use_bytearray is non-zero. */
+    int overallocate;
+
+    /* Stack buffer */
+    int use_small_buffer;
+    char small_buffer[512];
+} _PyBytesWriter;
+
+/* Initialize a bytes writer
+
+   By default, the overallocation is disabled. Set the overallocate attribute
+   to control the allocation of the buffer. */
+PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer);
+
+/* Get the buffer content and reset the writer.
+   Return a bytes object, or a bytearray object if use_bytearray is non-zero.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer,
+    void *str);
+
+/* Deallocate memory of a writer (clear its internal buffer). */
+PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer);
+
+/* Allocate the buffer to write size bytes.
+   Return the pointer to the beginning of buffer data.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer,
+    Py_ssize_t size);
+
+/* Ensure that the buffer is large enough to write *size* bytes.
+   Add size to the writer minimum size (min_size attribute).
+
+   str is the current pointer inside the buffer.
+   Return the updated current pointer inside the buffer.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer,
+    void *str,
+    Py_ssize_t size);
+
+/* Resize the buffer to make it larger.
+   The new buffer may be larger than size bytes because of overallocation.
+   Return the updated current pointer inside the buffer.
+   Raise an exception and return NULL on error.
+
+   Note: size must be greater than the number of allocated bytes in the writer.
+
+   This function doesn't use the writer minimum size (min_size attribute).
+
+   See also _PyBytesWriter_Prepare().
+   */
+PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer,
+    void *str,
+    Py_ssize_t size);
+
+/* Write bytes.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer,
+    void *str,
+    const void *bytes,
+    Py_ssize_t size);

Makefile.pre.in

@@ -1055,6 +1055,8 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/Python-ast.h \
 		\
 		$(srcdir)/Include/cpython/abstract.h \
+		$(srcdir)/Include/cpython/bytearrayobject.h \
+		$(srcdir)/Include/cpython/bytesobject.h \
 		$(srcdir)/Include/cpython/ceval.h \
 		$(srcdir)/Include/cpython/dictobject.h \
 		$(srcdir)/Include/cpython/fileobject.h \

PCbuild/pythoncore.vcxproj

@@ -126,6 +126,8 @@
     <ClInclude Include="..\Include\complexobject.h" />
     <ClInclude Include="..\Include\context.h" />
     <ClInclude Include="..\Include\cpython\abstract.h" />
+    <ClInclude Include="..\Include\cpython\bytearrayobject.h" />
+    <ClInclude Include="..\Include\cpython\bytesobject.h" />
     <ClInclude Include="..\Include\cpython\ceval.h" />
     <ClInclude Include="..\Include\cpython\dictobject.h" />
     <ClInclude Include="..\Include\cpython\fileobject.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -81,6 +81,12 @@
     <ClInclude Include="..\Include\cpython\abstract.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\cpython\bytearrayobject.h">
+      <Filter>Include</Filter>
+    </ClInclude>
+    <ClInclude Include="..\Include\cpython\bytesobject.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\cpython\ceval.h">
       <Filter>Include</Filter>
     </ClInclude>