From 703f1e8c21c5bf86d569ee15af137d4297a6d2e2 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 10:21:54 +1000 Subject: [PATCH 01/20] Tidy up extension before adding support for more versions Avoid range checks on PY_MINOR_VERSION. These get confusing the more that structs diverge across versions. Avoid aliasing struct fields across Python versions too. Instead, when structs/enums/accessors/mutators diverge across Python versions, use an explicit chain of branch contains the full copy of the struct/enum at that version, or the code required to access/mutate a struct field. This should hopefully clear up the code as support for new versions are added, and reduce coding errors over time. --- src/dispatch/experimental/durable/frame.c | 156 ++++++++++++++++------ 1 file changed, 116 insertions(+), 40 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 58420a39..43884fac 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -11,39 +11,56 @@ #endif // This is a redefinition of the private/opaque struct _PyInterpreterFrame: -// https://github.com/python/cpython/blob/3.12/Include/cpython/pyframe.h#L23 +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L47 // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L51 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L57 typedef struct InterpreterFrame { #if PY_MINOR_VERSION == 11 PyFunctionObject *f_func; -#elif PY_MINOR_VERSION >= 12 - PyCodeObject *f_code; // 3.13: PyObject *f_executable - struct _PyInterpreterFrame *previous; - PyObject *f_funcobj; -#endif PyObject *f_globals; PyObject *f_builtins; PyObject *f_locals; -#if PY_MINOR_VERSION == 11 PyCodeObject *f_code; PyFrameObject *frame_obj; struct _PyInterpreterFrame *previous; _Py_CODEUNIT *prev_instr; int stacktop; bool is_entry; -#elif PY_MINOR_VERSION >= 12 + char owner; + PyObject *localsplus[1]; +#elif PY_MINOR_VERSION == 12 + PyCodeObject *f_code; + struct _PyInterpreterFrame *previous; + PyObject *f_funcobj; + PyObject *f_globals; + PyObject *f_builtins; + PyObject *f_locals; PyFrameObject *frame_obj; - _Py_CODEUNIT *prev_instr; // 3.13: _Py_CODEUNIT *instr_ptr + _Py_CODEUNIT *prev_instr; int stacktop; uint16_t return_offset; -#endif char owner; PyObject *localsplus[1]; +#elif PY_MINOR_VERSION == 13 + PyObject *f_executable; + struct _PyInterpreterFrame *previous; + PyObject *f_funcobj; + PyObject *f_globals; + PyObject *f_builtins; + PyObject *f_locals; + PyFrameObject *frame_obj; + _Py_CODEUNIT *instr_ptr; + int stacktop; + uint16_t return_offset; + char owner; + PyObject *localsplus[1]; +#endif } InterpreterFrame; -// This is a redefinition of the private/opaque PyFrameObject: -// https://github.com/python/cpython/blob/3.12/Include/pytypedefs.h#L22 +// This is a redefinition of the private/opaque PyFrameObject (aka. struct _frame): +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L15 // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L16 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L20 // The definition is the same for Python 3.11-3.13. typedef struct FrameObject { PyObject_HEAD @@ -58,29 +75,46 @@ typedef struct FrameObject { } FrameObject; // This is a redefinition of frame state constants: +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L33 // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L34 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L38 typedef enum _framestate { -#if PY_MINOR_VERSION == 13 +#if PY_MINOR_VERSION == 11 + FRAME_CREATED = -2, + FRAME_SUSPENDED = -1, + FRAME_EXECUTING = 0, + FRAME_COMPLETED = 1, + FRAME_CLEARED = 4 +#elif PY_MINOR_VERSION == 12 + FRAME_CREATED = -2, + FRAME_SUSPENDED = -1, + FRAME_EXECUTING = 0, + FRAME_COMPLETED = 1, + FRAME_CLEARED = 4 +#elif PY_MINOR_VERSION == 13 FRAME_CREATED = -3, FRAME_SUSPENDED = -2, FRAME_SUSPENDED_YIELD_FROM = -1, -#else - FRAME_CREATED = -2, - FRAME_SUSPENDED = -1, -#endif FRAME_EXECUTING = 0, FRAME_COMPLETED = 1, FRAME_CLEARED = 4 +#endif } FrameState; // This is a redefinition of the private PyCoroWrapper: +// https://github.com/python/cpython/blob/3.11/Objects/genobject.c#L1016 +// https://github.com/python/cpython/blob/3.12/Objects/genobject.c#L1003 +// https://github.com/python/cpython/blob/v3.13.0a5/Objects/genobject.c#L985 typedef struct { PyObject_HEAD PyCoroObject *cw_coroutine; } PyCoroWrapper; // For reference, PyGenObject is defined as follows after expanding top-most macro: -// https://github.com/python/cpython/blob/3.12/Include/cpython/genobject.h +// https://github.com/python/cpython/blob/3.11/Include/cpython/genobject.h#L14 +// https://github.com/python/cpython/blob/3.12/Include/cpython/genobject.h#L14 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/cpython/genobject.h#L14 +// // Note that PyCoroObject and PyAsyncGenObject have the same layout in // Python 3.11-3.13, however the struct fields have a cr_ and ag_ prefix // (respectively) instead of a gi_ prefix. @@ -122,17 +156,22 @@ static PyGenObject *get_generator_like_object(PyObject *obj) { if (PyGen_Check(obj) || PyCoro_CheckExact(obj) || PyAsyncGen_CheckExact(obj)) { // Note: In Python 3.11-3.13, the PyGenObject, PyCoroObject and PyAsyncGenObject // have the same layout, they just have different field prefixes (gi_, cr_, ag_). + // We cast to PyGenObject here so that the remainder of the code can use the gi_ + // prefix for all three cases. return (PyGenObject *)obj; } - // CPython unfortunately does not export any functions that - // check whether an object is a coroutine_wrapper. - // FIXME: improve safety here, e.g. by checking that the obj type matches a known size + // If the object isn't a PyGenObject, PyCoroObject or PyAsyncGenObject, it may + // still be a coroutine, for example a PyCoroWrapper. CPython unfortunately does + // not export functions that check whether an object is a coroutine_wrapper; we + // need to check the type name string. const char *type_name = get_type_name(obj); if (!type_name) { return NULL; } if (strcmp(type_name, "coroutine_wrapper") == 0) { + // FIXME: improve safety here, e.g. by checking that the obj type matches a known size PyCoroWrapper *wrapper = (PyCoroWrapper *)obj; + // Cast the inner PyCoroObject to PyGenObject. See the comment above. return (PyGenObject *)wrapper->cw_coroutine; } PyErr_SetString(PyExc_TypeError, "Input object is not a generator or coroutine"); @@ -145,6 +184,50 @@ static InterpreterFrame *get_interpreter_frame(PyGenObject *gen_like) { return (InterpreterFrame *)frame; } +static PyCodeObject *get_frame_code(InterpreterFrame *frame) { +#if PY_MINOR_VERSION == 11 + PyCodeObject *code = frame->f_code; +#elif PY_MINOR_VERSION == 12 + PyCodeObject *code = frame->f_code; +#elif PY_MINOR_VERSION == 13 + PyCodeObject *code = (PyCodeObject *)frame->f_executable; +#endif + assert(code); + return code; +} + +static _Py_CODEUNIT *get_frame_instr_ptr(InterpreterFrame *frame) { +#if PY_MINOR_VERSION == 11 + _Py_CODEUNIT *instr_ptr = frame->prev_instr; +#elif PY_MINOR_VERSION == 12 + _Py_CODEUNIT *instr_ptr = frame->prev_instr; +#elif PY_MINOR_VERSION == 13 + _Py_CODEUNIT *instr_ptr = frame->instr_ptr; +#endif + assert(instr_ptr); + return instr_ptr; +} + +void set_frame_instr_ptr(InterpreterFrame *frame, _Py_CODEUNIT *instr_ptr) { +#if PY_MINOR_VERSION == 11 + frame->prev_instr = instr_ptr; +#elif PY_MINOR_VERSION == 12 + frame->prev_instr = instr_ptr; +#elif PY_MINOR_VERSION == 13 + frame->instr_ptr = instr_ptr; +#endif +} + +static int valid_frame_state(int fs) { +#if PY_MINOR_VERSION == 11 + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; +#elif PY_MINOR_VERSION == 12 + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; +#elif PY_MINOR_VERSION == 13 + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_SUSPENDED_YIELD_FROM || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; +#endif +} + static PyObject *get_frame_state(PyObject *self, PyObject *args) { PyObject *arg; if (!PyArg_ParseTuple(args, "O", &arg)) { @@ -174,11 +257,11 @@ static PyObject *get_frame_ip(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - assert(frame->f_code); - assert(frame->prev_instr); + _Py_CODEUNIT *instr_ptr = get_frame_instr_ptr(frame); + PyCodeObject *code = get_frame_code(frame); // See _PyInterpreterFrame_LASTI // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 - intptr_t ip = (intptr_t)frame->prev_instr - (intptr_t)_PyCode_CODE(frame->f_code); + intptr_t ip = (intptr_t)instr_ptr - (intptr_t)_PyCode_CODE(code); return PyLong_FromLong((long)ip); } @@ -223,8 +306,8 @@ static PyObject *get_frame_stack_at(PyObject *self, PyObject *args) { return NULL; } assert(frame->stacktop >= 0); - - int limit = frame->f_code->co_stacksize + frame->f_code->co_nlocalsplus; + PyCodeObject *code = get_frame_code(frame); + int limit = code->co_stacksize + code->co_nlocalsplus; if (index < 0 || index >= limit) { PyErr_SetString(PyExc_IndexError, "Index out of bounds"); return NULL; @@ -259,11 +342,10 @@ static PyObject *set_frame_ip(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - assert(frame->f_code); - assert(frame->prev_instr); + PyCodeObject *code = get_frame_code(frame); // See _PyInterpreterFrame_LASTI // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 - frame->prev_instr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(frame->f_code) + (intptr_t)ip); + set_frame_instr_ptr(frame, (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)ip)); Py_RETURN_NONE; } @@ -286,8 +368,8 @@ static PyObject *set_frame_sp(PyObject *self, PyObject *args) { return NULL; } assert(frame->stacktop >= 0); - - int limit = frame->f_code->co_stacksize + frame->f_code->co_nlocalsplus; + PyCodeObject *code = get_frame_code(frame); + int limit = code->co_stacksize + code->co_nlocalsplus; if (sp < 0 || sp >= limit) { PyErr_SetString(PyExc_IndexError, "Stack pointer out of bounds"); return NULL; @@ -325,17 +407,10 @@ static PyObject *set_frame_state(PyObject *self, PyObject *args) { if (!frame) { return NULL; } -#if PY_MINOR_VERSION == 13 - if (fs != FRAME_CREATED && fs != FRAME_SUSPENDED && fs != FRAME_SUSPENDED_YIELD_FROM && fs != FRAME_EXECUTING && fs != FRAME_COMPLETED) { + if (!valid_frame_state(fs)) { PyErr_SetString(PyExc_ValueError, "Invalid frame state"); return NULL; } -#else - if (fs != FRAME_CREATED && fs != FRAME_SUSPENDED && fs != FRAME_EXECUTING && fs != FRAME_COMPLETED) { - PyErr_SetString(PyExc_ValueError, "Invalid frame state"); - return NULL; - } -#endif gen_like->gi_frame_state = (int8_t)fs; // aka. cr_frame_state / ag_frame_state Py_RETURN_NONE; } @@ -365,7 +440,8 @@ static PyObject *set_frame_stack_at(PyObject *self, PyObject *args) { return NULL; } assert(frame->stacktop >= 0); - int limit = frame->f_code->co_stacksize + frame->f_code->co_nlocalsplus; + PyCodeObject *code = get_frame_code(frame); + int limit = code->co_stacksize + code->co_nlocalsplus; if (index < 0 || index >= limit) { PyErr_SetString(PyExc_IndexError, "Index out of bounds"); return NULL; From ac82d8e77ebade35fd03cfbf0ea7e177184c3d63 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 10:58:53 +1000 Subject: [PATCH 02/20] Start on 3.10 support --- src/dispatch/experimental/durable/frame.c | 143 +++++++++++++++++----- 1 file changed, 114 insertions(+), 29 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 43884fac..1befa2c6 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -6,16 +6,37 @@ #define PY_SSIZE_T_CLEAN #include -#if PY_MAJOR_VERSION != 3 || (PY_MINOR_VERSION < 11 || PY_MINOR_VERSION > 13) -# error Python 3.11-3.13 is required +#if PY_MAJOR_VERSION != 3 || (PY_MINOR_VERSION < 10 || PY_MINOR_VERSION > 13) +# error Python 3.10-3.13 is required #endif -// This is a redefinition of the private/opaque struct _PyInterpreterFrame: -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L47 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L51 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L57 +// This is a redefinition of the private/opaque PyInterpreterFrame. +// In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and +// PyInterpreterFrame. From Python 3.11 onwards, the two were split with +// PyFrameObject pointing to PyInterpreterFrame. typedef struct InterpreterFrame { -#if PY_MINOR_VERSION == 11 +#if PY_MINOR_VERSION == 10 +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 + PyObject_VAR_HEAD + struct InterpreterFrame *f_back; // struct _frame + PyCodeObject *f_code; + PyObject *f_builtins; + PyObject *f_globals; + PyObject *f_locals; + PyObject **f_valuestack; + PyObject *f_trace; + int f_stackdepth; + char f_trace_lines; + char f_trace_opcodes; + PyObject *f_gen; + int f_lasti; + int f_lineno; + int f_iblock; + PyFrameState f_state; + PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */ + PyObject *f_localsplus[1]; +#elif PY_MINOR_VERSION == 11 +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L47 PyFunctionObject *f_func; PyObject *f_globals; PyObject *f_builtins; @@ -29,6 +50,7 @@ typedef struct InterpreterFrame { char owner; PyObject *localsplus[1]; #elif PY_MINOR_VERSION == 12 +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L51 PyCodeObject *f_code; struct _PyInterpreterFrame *previous; PyObject *f_funcobj; @@ -42,6 +64,7 @@ typedef struct InterpreterFrame { char owner; PyObject *localsplus[1]; #elif PY_MINOR_VERSION == 13 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L57 PyObject *f_executable; struct _PyInterpreterFrame *previous; PyObject *f_funcobj; @@ -57,11 +80,11 @@ typedef struct InterpreterFrame { #endif } InterpreterFrame; -// This is a redefinition of the private/opaque PyFrameObject (aka. struct _frame): +// This is a redefinition of the private/opaque PyFrameObject: +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 // https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L15 // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L16 // https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L20 -// The definition is the same for Python 3.11-3.13. typedef struct FrameObject { PyObject_HEAD PyFrameObject *f_back; @@ -75,23 +98,32 @@ typedef struct FrameObject { } FrameObject; // This is a redefinition of frame state constants: -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L33 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L34 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L38 typedef enum _framestate { -#if PY_MINOR_VERSION == 11 +#if PY_MINOR_VERSION == 10 +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L10 + FRAME_CREATED = -2, + FRAME_SUSPENDED = -1, + FRAME_EXECUTING = 0, + FRAME_RETURNED = 1, + FRAME_UNWINDING = 2, + FRAME_RAISED = 3, + FRAME_CLEARED = 4 +#elif PY_MINOR_VERSION == 11 +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L33 FRAME_CREATED = -2, FRAME_SUSPENDED = -1, FRAME_EXECUTING = 0, FRAME_COMPLETED = 1, FRAME_CLEARED = 4 #elif PY_MINOR_VERSION == 12 +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L34 FRAME_CREATED = -2, FRAME_SUSPENDED = -1, FRAME_EXECUTING = 0, FRAME_COMPLETED = 1, FRAME_CLEARED = 4 #elif PY_MINOR_VERSION == 13 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L38 FRAME_CREATED = -3, FRAME_SUSPENDED = -2, FRAME_SUSPENDED_YIELD_FROM = -1, @@ -102,6 +134,7 @@ typedef enum _framestate { } FrameState; // This is a redefinition of the private PyCoroWrapper: +// https://github.com/python/cpython/blob/3.10/Objects/genobject.c#L884 // https://github.com/python/cpython/blob/3.11/Objects/genobject.c#L1016 // https://github.com/python/cpython/blob/3.12/Objects/genobject.c#L1003 // https://github.com/python/cpython/blob/v3.13.0a5/Objects/genobject.c#L985 @@ -110,20 +143,31 @@ typedef struct { PyCoroObject *cw_coroutine; } PyCoroWrapper; -// For reference, PyGenObject is defined as follows after expanding top-most macro: -// https://github.com/python/cpython/blob/3.11/Include/cpython/genobject.h#L14 -// https://github.com/python/cpython/blob/3.12/Include/cpython/genobject.h#L14 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/cpython/genobject.h#L14 -// -// Note that PyCoroObject and PyAsyncGenObject have the same layout in -// Python 3.11-3.13, however the struct fields have a cr_ and ag_ prefix -// (respectively) instead of a gi_ prefix. /* +// This is the definition of PyGenObject for reference to developers +// working on the extension. +// +// Note that PyCoroObject and PyAsyncGenObject have the same layout as +// PyGenObject, however the struct fields have a cr_ and ag_ prefix +// (respectively) rather than a gi_ prefix. In Python 3.10, PyCoroObject +// and PyAsyncGenObject have extra fields compared to PyGenObject. In Python +// 3.11 onwards, the three objects are identical (except for field name +// prefixes). Note that the extra fields in Python 3.10 are not applicable to +// this extension at this time. +// typedef struct { PyObject_HEAD -#if PY_MINOR_VERSION == 11 +#if PY_MINOR_VERSION == 10 +// https://github.com/python/cpython/blob/3.10/Include/genobject.h#L16 + PyFrameObject *gi_frame; + PyObject *gi_code; + PyObject *gi_weakreflist; + PyObject *gi_name; + PyObject *gi_qualname; + _PyErr_StackItem gi_exc_state; +#elif PY_MINOR_VERSION == 11 +// https://github.com/python/cpython/blob/3.11/Include/cpython/genobject.h#L14 PyCodeObject *gi_code; -#endif PyObject *gi_weakreflist; PyObject *gi_name; PyObject *gi_qualname; @@ -134,6 +178,31 @@ typedef struct { char gi_running_async; int8_t gi_frame_state; PyObject *gi_iframe[1]; +#elif PY_MINOR_VERSION == 12 +// https://github.com/python/cpython/blob/3.12/Include/cpython/genobject.h#L14 + PyObject *gi_weakreflist; + PyObject *gi_name; + PyObject *gi_qualname; + _PyErr_StackItem gi_exc_state; + PyObject *gi_origin_or_finalizer; + char gi_hooks_inited; + char gi_closed; + char gi_running_async; + int8_t gi_frame_state; + PyObject *gi_iframe[1]; +#elif PY_MINOR_VERSION == 13 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/cpython/genobject.h#L14 + PyObject *gi_weakreflist; + PyObject *gi_name; + PyObject *gi_qualname; + _PyErr_StackItem gi_exc_state; + PyObject *gi_origin_or_finalizer; + char gi_hooks_inited; + char gi_closed; + char gi_running_async; + int8_t gi_frame_state; + PyObject *gi_iframe[1]; +#endif } PyGenObject; */ @@ -179,13 +248,23 @@ static PyGenObject *get_generator_like_object(PyObject *obj) { } static InterpreterFrame *get_interpreter_frame(PyGenObject *gen_like) { - struct _PyInterpreterFrame *frame = (struct _PyInterpreterFrame *)(gen_like->gi_iframe); +#if PY_MINOR_VERSION == 10 + InterpreterFrame *frame = (InterpreterFrame *)(gen_like->gi_iframe); +#elif PY_MINOR_VERSION == 11 + InterpreterFrame *frame = (InterpreterFrame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); +#elif PY_MINOR_VERSION == 12 + InterpreterFrame *frame = (InterpreterFrame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); +#elif PY_MINOR_VERSION == 13 + InterpreterFrame *frame = (InterpreterFrame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); +#endif assert(frame); - return (InterpreterFrame *)frame; + return frame; } static PyCodeObject *get_frame_code(InterpreterFrame *frame) { -#if PY_MINOR_VERSION == 11 +#if PY_MINOR_VERSION == 10 + PyCodeObject *code = frame->f_code; +#elif PY_MINOR_VERSION == 11 PyCodeObject *code = frame->f_code; #elif PY_MINOR_VERSION == 12 PyCodeObject *code = frame->f_code; @@ -197,7 +276,9 @@ static PyCodeObject *get_frame_code(InterpreterFrame *frame) { } static _Py_CODEUNIT *get_frame_instr_ptr(InterpreterFrame *frame) { -#if PY_MINOR_VERSION == 11 +#if PY_MINOR_VERSION == 10 +# error TODO +#elif PY_MINOR_VERSION == 11 _Py_CODEUNIT *instr_ptr = frame->prev_instr; #elif PY_MINOR_VERSION == 12 _Py_CODEUNIT *instr_ptr = frame->prev_instr; @@ -209,7 +290,9 @@ static _Py_CODEUNIT *get_frame_instr_ptr(InterpreterFrame *frame) { } void set_frame_instr_ptr(InterpreterFrame *frame, _Py_CODEUNIT *instr_ptr) { -#if PY_MINOR_VERSION == 11 +#if PY_MINOR_VERSION == 10 +# error TODO +#elif PY_MINOR_VERSION == 11 frame->prev_instr = instr_ptr; #elif PY_MINOR_VERSION == 12 frame->prev_instr = instr_ptr; @@ -219,7 +302,9 @@ void set_frame_instr_ptr(InterpreterFrame *frame, _Py_CODEUNIT *instr_ptr) { } static int valid_frame_state(int fs) { -#if PY_MINOR_VERSION == 11 +#if PY_MINOR_VERSION == 10 + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_RETURNED || fs == FRAME_UNWINDING || fs == FRAME_RAISED || fs == FRAME_CLEARED; +#elif PY_MINOR_VERSION == 11 return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; #elif PY_MINOR_VERSION == 12 return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; From 417dd508bcb42152e24e14123788f3c209b6337f Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 11:19:49 +1000 Subject: [PATCH 03/20] Handle differences in getting/setting lasti --- src/dispatch/experimental/durable/frame.c | 62 +++++++++++++---------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 1befa2c6..015554d8 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -33,7 +33,7 @@ typedef struct InterpreterFrame { int f_lineno; int f_iblock; PyFrameState f_state; - PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */ + PyTryBlock f_blockstack[CO_MAXBLOCKS]; PyObject *f_localsplus[1]; #elif PY_MINOR_VERSION == 11 // https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L47 @@ -80,7 +80,7 @@ typedef struct InterpreterFrame { #endif } InterpreterFrame; -// This is a redefinition of the private/opaque PyFrameObject: +// This is a redefinition of the private PyFrameObject (aka. struct _frame): // https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 // https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L15 // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L16 @@ -152,8 +152,8 @@ typedef struct { // (respectively) rather than a gi_ prefix. In Python 3.10, PyCoroObject // and PyAsyncGenObject have extra fields compared to PyGenObject. In Python // 3.11 onwards, the three objects are identical (except for field name -// prefixes). Note that the extra fields in Python 3.10 are not applicable to -// this extension at this time. +// prefixes). The extra fields in Python 3.10 are not applicable to this +// extension at this time. // typedef struct { PyObject_HEAD @@ -223,7 +223,7 @@ static const char *get_type_name(PyObject *obj) { static PyGenObject *get_generator_like_object(PyObject *obj) { if (PyGen_Check(obj) || PyCoro_CheckExact(obj) || PyAsyncGen_CheckExact(obj)) { - // Note: In Python 3.11-3.13, the PyGenObject, PyCoroObject and PyAsyncGenObject + // Note: In Python 3.10-3.13, the PyGenObject, PyCoroObject and PyAsyncGenObject // have the same layout, they just have different field prefixes (gi_, cr_, ag_). // We cast to PyGenObject here so that the remainder of the code can use the gi_ // prefix for all three cases. @@ -231,7 +231,7 @@ static PyGenObject *get_generator_like_object(PyObject *obj) { } // If the object isn't a PyGenObject, PyCoroObject or PyAsyncGenObject, it may // still be a coroutine, for example a PyCoroWrapper. CPython unfortunately does - // not export functions that check whether an object is a coroutine_wrapper; we + // not export a function that checks whether a PyObject is a PyCoroWrapper. We // need to check the type name string. const char *type_name = get_type_name(obj); if (!type_name) { @@ -240,7 +240,7 @@ static PyGenObject *get_generator_like_object(PyObject *obj) { if (strcmp(type_name, "coroutine_wrapper") == 0) { // FIXME: improve safety here, e.g. by checking that the obj type matches a known size PyCoroWrapper *wrapper = (PyCoroWrapper *)obj; - // Cast the inner PyCoroObject to PyGenObject. See the comment above. + // Cast the inner PyCoroObject to a PyGenObject. See the comment above. return (PyGenObject *)wrapper->cw_coroutine; } PyErr_SetString(PyExc_TypeError, "Input object is not a generator or coroutine"); @@ -275,29 +275,42 @@ static PyCodeObject *get_frame_code(InterpreterFrame *frame) { return code; } -static _Py_CODEUNIT *get_frame_instr_ptr(InterpreterFrame *frame) { +static int get_frame_lasti(InterpreterFrame *frame) { #if PY_MINOR_VERSION == 10 -# error TODO + return frame->f_lasti; #elif PY_MINOR_VERSION == 11 - _Py_CODEUNIT *instr_ptr = frame->prev_instr; +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L69 + PyCodeObject *code = get_frame_code(frame); + assert(frame->prev_instr); + return (int)((intptr_t)frame->prev_instr - (intptr_t)_PyCode_CODE(code)); #elif PY_MINOR_VERSION == 12 - _Py_CODEUNIT *instr_ptr = frame->prev_instr; +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 + PyCodeObject *code = get_frame_code(frame); + assert(frame->prev_instr); + return (int)((intptr_t)frame->prev_instr - (intptr_t)_PyCode_CODE(code)); #elif PY_MINOR_VERSION == 13 - _Py_CODEUNIT *instr_ptr = frame->instr_ptr; +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L73 + PyCodeObject *code = get_frame_code(frame); + assert(frame->instr_ptr); + return (int)((intptr_t)frame->instr_ptr - (intptr_t)_PyCode_CODE(code)); #endif - assert(instr_ptr); - return instr_ptr; } -void set_frame_instr_ptr(InterpreterFrame *frame, _Py_CODEUNIT *instr_ptr) { +void set_frame_lasti(InterpreterFrame *frame, int lasti) { #if PY_MINOR_VERSION == 10 -# error TODO + frame->f_lasti = lasti; #elif PY_MINOR_VERSION == 11 - frame->prev_instr = instr_ptr; +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L69 + PyCodeObject *code = get_frame_code(frame); + frame->prev_instr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); #elif PY_MINOR_VERSION == 12 - frame->prev_instr = instr_ptr; +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 + PyCodeObject *code = get_frame_code(frame); + frame->prev_instr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); #elif PY_MINOR_VERSION == 13 - frame->instr_ptr = instr_ptr; +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L73 + PyCodeObject *code = get_frame_code(frame); + frame->instr_ptr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); #endif } @@ -342,11 +355,7 @@ static PyObject *get_frame_ip(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - _Py_CODEUNIT *instr_ptr = get_frame_instr_ptr(frame); - PyCodeObject *code = get_frame_code(frame); - // See _PyInterpreterFrame_LASTI - // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 - intptr_t ip = (intptr_t)instr_ptr - (intptr_t)_PyCode_CODE(code); + int ip = get_frame_lasti(frame); return PyLong_FromLong((long)ip); } @@ -427,10 +436,7 @@ static PyObject *set_frame_ip(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - PyCodeObject *code = get_frame_code(frame); - // See _PyInterpreterFrame_LASTI - // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 - set_frame_instr_ptr(frame, (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)ip)); + set_frame_lasti(frame, ip); Py_RETURN_NONE; } From 39d418efa97ce35809bb3b834d21856194658bb9 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 11:25:26 +1000 Subject: [PATCH 04/20] Include the definition of relevant structs --- src/dispatch/experimental/durable/frame.c | 81 ++++++++++++++++++----- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 015554d8..7983b411 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -80,24 +80,7 @@ typedef struct InterpreterFrame { #endif } InterpreterFrame; -// This is a redefinition of the private PyFrameObject (aka. struct _frame): -// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L15 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L16 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L20 -typedef struct FrameObject { - PyObject_HEAD - PyFrameObject *f_back; - struct _PyInterpreterFrame *f_frame; - PyObject *f_trace; - int f_lineno; - char f_trace_lines; - char f_trace_opcodes; - char f_fast_as_locals; - PyObject *_f_frame_data[1]; -} FrameObject; - -// This is a redefinition of frame state constants: +// This is a redefinition of private frame state constants: typedef enum _framestate { #if PY_MINOR_VERSION == 10 // https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L10 @@ -143,6 +126,68 @@ typedef struct { PyCoroObject *cw_coroutine; } PyCoroWrapper; +/* +// This is the definition of PyFrameObject (aka. struct _frame) for reference +// to developers working on the extension. +// +typedef struct { +#if PY_MINOR_VERSION == 10 +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 + PyObject_VAR_HEAD + struct InterpreterFrame *f_back; // struct _frame + PyCodeObject *f_code; + PyObject *f_builtins; + PyObject *f_globals; + PyObject *f_locals; + PyObject **f_valuestack; + PyObject *f_trace; + int f_stackdepth; + char f_trace_lines; + char f_trace_opcodes; + PyObject *f_gen; + int f_lasti; + int f_lineno; + int f_iblock; + PyFrameState f_state; + PyTryBlock f_blockstack[CO_MAXBLOCKS]; + PyObject *f_localsplus[1]; +#elif PY_MINOR_VERSION == 11 +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L15 + PyObject_HEAD + PyFrameObject *f_back; + struct _PyInterpreterFrame *f_frame; + PyObject *f_trace; + int f_lineno; + char f_trace_lines; + char f_trace_opcodes; + char f_fast_as_locals; + PyObject *_f_frame_data[1]; +#elif PY_MINOR_VERSION == 12 +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L16 + PyObject_HEAD + PyFrameObject *f_back; + struct _PyInterpreterFrame *f_frame; + PyObject *f_trace; + int f_lineno; + char f_trace_lines; + char f_trace_opcodes; + char f_fast_as_locals; + PyObject *_f_frame_data[1]; +#elif PY_MINOR_VERSION == 13 +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L20 + PyObject_HEAD + PyFrameObject *f_back; + struct _PyInterpreterFrame *f_frame; + PyObject *f_trace; + int f_lineno; + char f_trace_lines; + char f_trace_opcodes; + char f_fast_as_locals; + PyObject *_f_frame_data[1]; +#endif +} PyFrameObject; +*/ + /* // This is the definition of PyGenObject for reference to developers // working on the extension. From e673f0eded936312102e2b30b9a60266cc23cf77 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 11:40:23 +1000 Subject: [PATCH 05/20] Relax build constraint --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ace78758..21796365 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "dispatch-py" description = "Develop reliable distributed systems on the Dispatch platform." readme = "README.md" dynamic = ["version"] -requires-python = ">= 3.11" +requires-python = ">= 3.10" dependencies = [ "grpcio >= 1.60.0", "protobuf >= 4.24.0", From 7cca2e5caa98fa4b97c51ddc6543bebbd7925769 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 11:52:56 +1000 Subject: [PATCH 06/20] Handle differences in frame state --- src/dispatch/experimental/durable/frame.c | 92 ++++++++++++++++------- 1 file changed, 64 insertions(+), 28 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 7983b411..8c819429 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -10,6 +10,16 @@ # error Python 3.10-3.13 is required #endif +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L20 +typedef int8_t PyFrameState; + +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L22 +typedef struct _PyTryBlock { + int b_type; + int b_handler; + int b_level; +} PyTryBlock; + // This is a redefinition of the private/opaque PyInterpreterFrame. // In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and // PyInterpreterFrame. From Python 3.11 onwards, the two were split with @@ -294,7 +304,7 @@ static PyGenObject *get_generator_like_object(PyObject *obj) { static InterpreterFrame *get_interpreter_frame(PyGenObject *gen_like) { #if PY_MINOR_VERSION == 10 - InterpreterFrame *frame = (InterpreterFrame *)(gen_like->gi_iframe); + InterpreterFrame *frame = (InterpreterFrame *)(gen_like->gi_frame); #elif PY_MINOR_VERSION == 11 InterpreterFrame *frame = (InterpreterFrame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); #elif PY_MINOR_VERSION == 12 @@ -359,6 +369,31 @@ void set_frame_lasti(InterpreterFrame *frame, int lasti) { #endif } +static int get_frame_state(PyGenObject *gen_like) { +#if PY_MINOR_VERSION == 10 + return get_interpreter_frame(gen_like)->f_state; +#elif PY_MINOR_VERSION == 11 + return gen_like->gi_frame_state; +#elif PY_MINOR_VERSION == 12 + return gen_like->gi_frame_state; +#elif PY_MINOR_VERSION == 13 + return gen_like->gi_frame_state; +#endif +} + +static void set_frame_state(PyGenObject *gen_like, int fs) { +#if PY_MINOR_VERSION == 10 + InterpreterFrame *frame = get_interpreter_frame(gen_like); + frame->f_state = (PyFrameState)fs; +#elif PY_MINOR_VERSION == 11 + gen_like->gi_frame_state = (int8_t)fs; +#elif PY_MINOR_VERSION == 12 + gen_like->gi_frame_state = (int8_t)fs; +#elif PY_MINOR_VERSION == 13 + gen_like->gi_frame_state = (int8_t)fs; +#endif +} + static int valid_frame_state(int fs) { #if PY_MINOR_VERSION == 10 return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_RETURNED || fs == FRAME_UNWINDING || fs == FRAME_RAISED || fs == FRAME_CLEARED; @@ -371,19 +406,20 @@ static int valid_frame_state(int fs) { #endif } -static PyObject *get_frame_state(PyObject *self, PyObject *args) { +static PyObject *ext_get_frame_state(PyObject *self, PyObject *args) { PyObject *arg; if (!PyArg_ParseTuple(args, "O", &arg)) { return NULL; } - PyGenObject *gen = get_generator_like_object(arg); - if (!gen) { + PyGenObject *gen_like = get_generator_like_object(arg); + if (!gen_like) { return NULL; } - return PyLong_FromLong((long)gen->gi_frame_state); // aka. cr_frame_state / ag_frame_state + int fs = get_frame_state(gen_like); + return PyLong_FromLong((long)fs); } -static PyObject *get_frame_ip(PyObject *self, PyObject *args) { +static PyObject *ext_get_frame_ip(PyObject *self, PyObject *args) { PyObject *obj; if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; @@ -392,7 +428,7 @@ static PyObject *get_frame_ip(PyObject *self, PyObject *args) { if (!gen_like) { return NULL; } - if (gen_like->gi_frame_state >= FRAME_CLEARED) { + if (get_frame_state(gen_like) >= FRAME_CLEARED) { PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); return NULL; } @@ -404,7 +440,7 @@ static PyObject *get_frame_ip(PyObject *self, PyObject *args) { return PyLong_FromLong((long)ip); } -static PyObject *get_frame_sp(PyObject *self, PyObject *args) { +static PyObject *ext_get_frame_sp(PyObject *self, PyObject *args) { PyObject *obj; if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; @@ -413,7 +449,7 @@ static PyObject *get_frame_sp(PyObject *self, PyObject *args) { if (!gen_like) { return NULL; } - if (gen_like->gi_frame_state >= FRAME_CLEARED) { + if (get_frame_state(gen_like) >= FRAME_CLEARED) { PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); return NULL; } @@ -426,7 +462,7 @@ static PyObject *get_frame_sp(PyObject *self, PyObject *args) { return PyLong_FromLong((long)sp); } -static PyObject *get_frame_stack_at(PyObject *self, PyObject *args) { +static PyObject *ext_get_frame_stack_at(PyObject *self, PyObject *args) { PyObject *obj; int index; if (!PyArg_ParseTuple(args, "Oi", &obj, &index)) { @@ -436,7 +472,7 @@ static PyObject *get_frame_stack_at(PyObject *self, PyObject *args) { if (!gen_like) { return NULL; } - if (gen_like->gi_frame_state >= FRAME_CLEARED) { + if (get_frame_state(gen_like) >= FRAME_CLEARED) { PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); return NULL; } @@ -463,7 +499,7 @@ static PyObject *get_frame_stack_at(PyObject *self, PyObject *args) { return PyTuple_Pack(2, is_null, stack_obj); } -static PyObject *set_frame_ip(PyObject *self, PyObject *args) { +static PyObject *ext_set_frame_ip(PyObject *self, PyObject *args) { PyObject *obj; int ip; if (!PyArg_ParseTuple(args, "Oi", &obj, &ip)) { @@ -473,7 +509,7 @@ static PyObject *set_frame_ip(PyObject *self, PyObject *args) { if (!gen_like) { return NULL; } - if (gen_like->gi_frame_state >= FRAME_CLEARED) { + if (get_frame_state(gen_like) >= FRAME_CLEARED) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } @@ -485,7 +521,7 @@ static PyObject *set_frame_ip(PyObject *self, PyObject *args) { Py_RETURN_NONE; } -static PyObject *set_frame_sp(PyObject *self, PyObject *args) { +static PyObject *ext_set_frame_sp(PyObject *self, PyObject *args) { PyObject *obj; int sp; if (!PyArg_ParseTuple(args, "Oi", &obj, &sp)) { @@ -495,7 +531,7 @@ static PyObject *set_frame_sp(PyObject *self, PyObject *args) { if (!gen_like) { return NULL; } - if (gen_like->gi_frame_state >= FRAME_CLEARED) { + if (get_frame_state(gen_like) >= FRAME_CLEARED) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } @@ -521,7 +557,7 @@ static PyObject *set_frame_sp(PyObject *self, PyObject *args) { Py_RETURN_NONE; } -static PyObject *set_frame_state(PyObject *self, PyObject *args) { +static PyObject *ext_set_frame_state(PyObject *self, PyObject *args) { PyObject *obj; int fs; if (!PyArg_ParseTuple(args, "Oi", &obj, &fs)) { @@ -535,7 +571,7 @@ static PyObject *set_frame_state(PyObject *self, PyObject *args) { if (!gen_like) { return NULL; } - if (gen_like->gi_frame_state >= FRAME_CLEARED) { + if (get_frame_state(gen_like) >= FRAME_CLEARED) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } @@ -547,11 +583,11 @@ static PyObject *set_frame_state(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_ValueError, "Invalid frame state"); return NULL; } - gen_like->gi_frame_state = (int8_t)fs; // aka. cr_frame_state / ag_frame_state + set_frame_state(gen_like, fs); Py_RETURN_NONE; } -static PyObject *set_frame_stack_at(PyObject *self, PyObject *args) { +static PyObject *ext_set_frame_stack_at(PyObject *self, PyObject *args) { PyObject *obj; int index; PyObject *unset; @@ -567,7 +603,7 @@ static PyObject *set_frame_stack_at(PyObject *self, PyObject *args) { if (!gen_like) { return NULL; } - if (gen_like->gi_frame_state >= FRAME_CLEARED) { + if (get_frame_state(gen_like) >= FRAME_CLEARED) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } @@ -599,14 +635,14 @@ static PyObject *set_frame_stack_at(PyObject *self, PyObject *args) { } static PyMethodDef methods[] = { - {"get_frame_ip", get_frame_ip, METH_VARARGS, "Get instruction pointer of a generator or coroutine."}, - {"set_frame_ip", set_frame_ip, METH_VARARGS, "Set instruction pointer of a generator or coroutine."}, - {"get_frame_sp", get_frame_sp, METH_VARARGS, "Get stack pointer of a generator or coroutine."}, - {"set_frame_sp", set_frame_sp, METH_VARARGS, "Set stack pointer of a generator or coroutine."}, - {"get_frame_stack_at", get_frame_stack_at, METH_VARARGS, "Get an object from a generator or coroutine's stack, as an (is_null, obj) tuple."}, - {"set_frame_stack_at", set_frame_stack_at, METH_VARARGS, "Set or unset an object on the stack of a generator or coroutine."}, - {"get_frame_state", get_frame_state, METH_VARARGS, "Get frame state of a generator or coroutine."}, - {"set_frame_state", set_frame_state, METH_VARARGS, "Set frame state of a generator or coroutine."}, + {"get_frame_ip", ext_get_frame_ip, METH_VARARGS, "Get instruction pointer of a generator or coroutine."}, + {"set_frame_ip", ext_set_frame_ip, METH_VARARGS, "Set instruction pointer of a generator or coroutine."}, + {"get_frame_sp", ext_get_frame_sp, METH_VARARGS, "Get stack pointer of a generator or coroutine."}, + {"set_frame_sp", ext_set_frame_sp, METH_VARARGS, "Set stack pointer of a generator or coroutine."}, + {"get_frame_stack_at", ext_get_frame_stack_at, METH_VARARGS, "Get an object from a generator or coroutine's stack, as an (is_null, obj) tuple."}, + {"set_frame_stack_at", ext_set_frame_stack_at, METH_VARARGS, "Set or unset an object on the stack of a generator or coroutine."}, + {"get_frame_state", ext_get_frame_state, METH_VARARGS, "Get frame state of a generator or coroutine."}, + {"set_frame_state", ext_set_frame_state, METH_VARARGS, "Set frame state of a generator or coroutine."}, {NULL, NULL, 0, NULL} }; From abafe74c04a6e0e6c73120fd38572343695e0933 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 12:00:31 +1000 Subject: [PATCH 07/20] Rename the frame object, to make it clear it's neither struct _frame nor struct _PyInterpreterFrame --- src/dispatch/experimental/durable/frame.c | 58 ++++++++++++----------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 8c819429..daca9356 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -20,15 +20,15 @@ typedef struct _PyTryBlock { int b_level; } PyTryBlock; -// This is a redefinition of the private/opaque PyInterpreterFrame. +// This is a redefinition of the private/opaque frame object. // In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and -// PyInterpreterFrame. From Python 3.11 onwards, the two were split with -// PyFrameObject pointing to PyInterpreterFrame. -typedef struct InterpreterFrame { +// PyFrame. From Python 3.11 onwards, the two were split with the +// PyFrameObject (struct _frame) pointing to struct _PyInterpreterFrame. +typedef struct Frame { #if PY_MINOR_VERSION == 10 // https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 PyObject_VAR_HEAD - struct InterpreterFrame *f_back; // struct _frame + struct Frame *f_back; // struct _frame PyCodeObject *f_code; PyObject *f_builtins; PyObject *f_globals; @@ -53,7 +53,7 @@ typedef struct InterpreterFrame { PyObject *f_locals; PyCodeObject *f_code; PyFrameObject *frame_obj; - struct _PyInterpreterFrame *previous; + struct Frame *previous; // struct _PyInterpreterFrame _Py_CODEUNIT *prev_instr; int stacktop; bool is_entry; @@ -62,7 +62,7 @@ typedef struct InterpreterFrame { #elif PY_MINOR_VERSION == 12 // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L51 PyCodeObject *f_code; - struct _PyInterpreterFrame *previous; + struct Frame *previous; // struct _PyInterpreterFrame PyObject *f_funcobj; PyObject *f_globals; PyObject *f_builtins; @@ -76,7 +76,7 @@ typedef struct InterpreterFrame { #elif PY_MINOR_VERSION == 13 // https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L57 PyObject *f_executable; - struct _PyInterpreterFrame *previous; + struct Frame *previous; // struct _PyInterpreterFrame PyObject *f_funcobj; PyObject *f_globals; PyObject *f_builtins; @@ -88,7 +88,7 @@ typedef struct InterpreterFrame { char owner; PyObject *localsplus[1]; #endif -} InterpreterFrame; +} Frame; // This is a redefinition of private frame state constants: typedef enum _framestate { @@ -138,13 +138,15 @@ typedef struct { /* // This is the definition of PyFrameObject (aka. struct _frame) for reference -// to developers working on the extension. +// to developers working on the extension. As noted above, the contents of +// struct _frame changed over time, with some fields moving in Python >= 3.11 +// to a separate struct _PyInterpreterFrame. // typedef struct { #if PY_MINOR_VERSION == 10 // https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 PyObject_VAR_HEAD - struct InterpreterFrame *f_back; // struct _frame + struct _frame *f_back; PyCodeObject *f_code; PyObject *f_builtins; PyObject *f_globals; @@ -302,21 +304,21 @@ static PyGenObject *get_generator_like_object(PyObject *obj) { return NULL; } -static InterpreterFrame *get_interpreter_frame(PyGenObject *gen_like) { +static Frame *get_frame(PyGenObject *gen_like) { #if PY_MINOR_VERSION == 10 - InterpreterFrame *frame = (InterpreterFrame *)(gen_like->gi_frame); + Frame *frame = (Frame *)(gen_like->gi_frame); #elif PY_MINOR_VERSION == 11 - InterpreterFrame *frame = (InterpreterFrame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); + Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); #elif PY_MINOR_VERSION == 12 - InterpreterFrame *frame = (InterpreterFrame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); + Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); #elif PY_MINOR_VERSION == 13 - InterpreterFrame *frame = (InterpreterFrame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); + Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); #endif assert(frame); return frame; } -static PyCodeObject *get_frame_code(InterpreterFrame *frame) { +static PyCodeObject *get_frame_code(Frame *frame) { #if PY_MINOR_VERSION == 10 PyCodeObject *code = frame->f_code; #elif PY_MINOR_VERSION == 11 @@ -330,7 +332,7 @@ static PyCodeObject *get_frame_code(InterpreterFrame *frame) { return code; } -static int get_frame_lasti(InterpreterFrame *frame) { +static int get_frame_lasti(Frame *frame) { #if PY_MINOR_VERSION == 10 return frame->f_lasti; #elif PY_MINOR_VERSION == 11 @@ -351,7 +353,7 @@ static int get_frame_lasti(InterpreterFrame *frame) { #endif } -void set_frame_lasti(InterpreterFrame *frame, int lasti) { +void set_frame_lasti(Frame *frame, int lasti) { #if PY_MINOR_VERSION == 10 frame->f_lasti = lasti; #elif PY_MINOR_VERSION == 11 @@ -371,7 +373,7 @@ void set_frame_lasti(InterpreterFrame *frame, int lasti) { static int get_frame_state(PyGenObject *gen_like) { #if PY_MINOR_VERSION == 10 - return get_interpreter_frame(gen_like)->f_state; + return get_frame(gen_like)->f_state; #elif PY_MINOR_VERSION == 11 return gen_like->gi_frame_state; #elif PY_MINOR_VERSION == 12 @@ -383,7 +385,7 @@ static int get_frame_state(PyGenObject *gen_like) { static void set_frame_state(PyGenObject *gen_like, int fs) { #if PY_MINOR_VERSION == 10 - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); frame->f_state = (PyFrameState)fs; #elif PY_MINOR_VERSION == 11 gen_like->gi_frame_state = (int8_t)fs; @@ -432,7 +434,7 @@ static PyObject *ext_get_frame_ip(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); return NULL; } - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); if (!frame) { return NULL; } @@ -453,7 +455,7 @@ static PyObject *ext_get_frame_sp(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); return NULL; } - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); if (!frame) { return NULL; } @@ -476,7 +478,7 @@ static PyObject *ext_get_frame_stack_at(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); return NULL; } - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); if (!frame) { return NULL; } @@ -513,7 +515,7 @@ static PyObject *ext_set_frame_ip(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); if (!frame) { return NULL; } @@ -535,7 +537,7 @@ static PyObject *ext_set_frame_sp(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); if (!frame) { return NULL; } @@ -575,7 +577,7 @@ static PyObject *ext_set_frame_state(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); if (!frame) { return NULL; } @@ -607,7 +609,7 @@ static PyObject *ext_set_frame_stack_at(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); return NULL; } - InterpreterFrame *frame = get_interpreter_frame(gen_like); + Frame *frame = get_frame(gen_like); if (!frame) { return NULL; } From 6e43d62562adf44436430e2471da7ad5429c3858 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 12:34:25 +1000 Subject: [PATCH 08/20] Extract helpers for getting/setting stacktop --- src/dispatch/experimental/durable/frame.c | 61 +++++++++++++++-------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index daca9356..f70cfcf2 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -408,6 +408,35 @@ static int valid_frame_state(int fs) { #endif } +static int get_frame_stacktop(Frame *frame) { +#if PY_MINOR_VERSION == 10 +# error TODO +#elif PY_MINOR_VERSION == 11 + int stacktop = frame->stacktop; +#elif PY_MINOR_VERSION == 12 + int stacktop = frame->stacktop; +#elif PY_MINOR_VERSION == 13 + int stacktop = frame->stacktop; +#endif + PyCodeObject *code = get_frame_code(frame); + int limit = code->co_stacksize + code->co_nlocalsplus; + (void)limit; // if NDEBUG + assert(stacktop >= 0 && stacktop < limit); + return stacktop; +} + +static void set_frame_stacktop(Frame *frame, int stacktop) { +#if PY_MINOR_VERSION == 10 +# error TODO +#elif PY_MINOR_VERSION == 11 + frame->stacktop = stacktop; +#elif PY_MINOR_VERSION == 12 + frame->stacktop = stacktop; +#elif PY_MINOR_VERSION == 13 + frame->stacktop = stacktop; +#endif +} + static PyObject *ext_get_frame_state(PyObject *self, PyObject *args) { PyObject *arg; if (!PyArg_ParseTuple(args, "O", &arg)) { @@ -459,8 +488,7 @@ static PyObject *ext_get_frame_sp(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - assert(frame->stacktop >= 0); - int sp = frame->stacktop; + int sp = get_frame_stacktop(frame); return PyLong_FromLong((long)sp); } @@ -482,10 +510,8 @@ static PyObject *ext_get_frame_stack_at(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - assert(frame->stacktop >= 0); - PyCodeObject *code = get_frame_code(frame); - int limit = code->co_stacksize + code->co_nlocalsplus; - if (index < 0 || index >= limit) { + int sp = get_frame_stacktop(frame); + if (index < 0 || index >= sp) { PyErr_SetString(PyExc_IndexError, "Index out of bounds"); return NULL; } @@ -541,21 +567,19 @@ static PyObject *ext_set_frame_sp(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - assert(frame->stacktop >= 0); PyCodeObject *code = get_frame_code(frame); int limit = code->co_stacksize + code->co_nlocalsplus; if (sp < 0 || sp >= limit) { PyErr_SetString(PyExc_IndexError, "Stack pointer out of bounds"); return NULL; } - - if (sp > frame->stacktop) { - for (int i = frame->stacktop; i < sp; i++) { + int current_sp = get_frame_stacktop(frame); + if (sp > current_sp) { + for (int i = current_sp; i < sp; i++) { frame->localsplus[i] = NULL; } } - - frame->stacktop = sp; + set_frame_stacktop(frame, sp); Py_RETURN_NONE; } @@ -613,14 +637,11 @@ static PyObject *ext_set_frame_stack_at(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - assert(frame->stacktop >= 0); - PyCodeObject *code = get_frame_code(frame); - int limit = code->co_stacksize + code->co_nlocalsplus; - if (index < 0 || index >= limit) { + int sp = get_frame_stacktop(frame); + if (index < 0 || index >= sp) { PyErr_SetString(PyExc_IndexError, "Index out of bounds"); return NULL; } - PyObject *prev = frame->localsplus[index]; if (Py_IsTrue(unset)) { frame->localsplus[index] = NULL; @@ -628,11 +649,7 @@ static PyObject *ext_set_frame_stack_at(PyObject *self, PyObject *args) { Py_INCREF(stack_obj); frame->localsplus[index] = stack_obj; } - - if (index < frame->stacktop) { - Py_XDECREF(prev); - } - + Py_XDECREF(prev); Py_RETURN_NONE; } From ea9c26e28522d9e149fb103ebb462b2c4f2754fc Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 12:37:14 +1000 Subject: [PATCH 09/20] Extract helper to get localsplus --- src/dispatch/experimental/durable/frame.c | 25 ++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index f70cfcf2..61ba05ca 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -437,6 +437,18 @@ static void set_frame_stacktop(Frame *frame, int stacktop) { #endif } +static PyObject **get_frame_localsplus(Frame *frame) { +#if PY_MINOR_VERSION == 10 + return frame->f_localsplus; +#elif PY_MINOR_VERSION == 11 + return frame->localsplus; +#elif PY_MINOR_VERSION == 12 + return frame->localsplus; +#elif PY_MINOR_VERSION == 13 + return frame->localsplus; +#endif +} + static PyObject *ext_get_frame_state(PyObject *self, PyObject *args) { PyObject *arg; if (!PyArg_ParseTuple(args, "O", &arg)) { @@ -519,7 +531,8 @@ static PyObject *ext_get_frame_stack_at(PyObject *self, PyObject *args) { // NULL in C != None in Python. We need to preserve the fact that some items // on the stack are NULL (not yet available). PyObject *is_null = Py_False; - PyObject *stack_obj = frame->localsplus[index]; + PyObject **localsplus = get_frame_localsplus(frame); + PyObject *stack_obj = localsplus[index]; if (!stack_obj) { is_null = Py_True; stack_obj = Py_None; @@ -573,10 +586,11 @@ static PyObject *ext_set_frame_sp(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_IndexError, "Stack pointer out of bounds"); return NULL; } + PyObject **localsplus = get_frame_localsplus(frame); int current_sp = get_frame_stacktop(frame); if (sp > current_sp) { for (int i = current_sp; i < sp; i++) { - frame->localsplus[i] = NULL; + localsplus[i] = NULL; } } set_frame_stacktop(frame, sp); @@ -642,12 +656,13 @@ static PyObject *ext_set_frame_stack_at(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_IndexError, "Index out of bounds"); return NULL; } - PyObject *prev = frame->localsplus[index]; + PyObject **localsplus = get_frame_localsplus(frame); + PyObject *prev = localsplus[index]; if (Py_IsTrue(unset)) { - frame->localsplus[index] = NULL; + localsplus[index] = NULL; } else { Py_INCREF(stack_obj); - frame->localsplus[index] = stack_obj; + localsplus[index] = stack_obj; } Py_XDECREF(prev); Py_RETURN_NONE; From 8497ac628eef9335e1e454e132490c479484b694 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 12:43:47 +1000 Subject: [PATCH 10/20] Set SP on 3.10 --- src/dispatch/experimental/durable/frame.c | 41 ++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 61ba05ca..f4cad3eb 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -408,9 +408,24 @@ static int valid_frame_state(int fs) { #endif } +static int get_frame_stacktop_limit(Frame *frame) { + PyCodeObject *code = get_frame_code(frame); +#if PY_MINOR_VERSION == 10 + return code->co_stacksize + code->co_nlocals; +#elif PY_MINOR_VERSION == 11 + return code->co_stacksize + code->co_nlocalsplus; +#elif PY_MINOR_VERSION == 12 + return code->co_stacksize + code->co_nlocalsplus; +#elif PY_MINOR_VERSION == 13 + return code->co_stacksize + code->co_nlocalsplus; +#endif +} + static int get_frame_stacktop(Frame *frame) { #if PY_MINOR_VERSION == 10 -# error TODO + assert(frame->f_localsplus); + assert(frame->f_valuestack); + int stacktop = (int)(frame->f_valuestack - frame->f_localsplus) + frame->f_stackdepth; #elif PY_MINOR_VERSION == 11 int stacktop = frame->stacktop; #elif PY_MINOR_VERSION == 12 @@ -418,16 +433,17 @@ static int get_frame_stacktop(Frame *frame) { #elif PY_MINOR_VERSION == 13 int stacktop = frame->stacktop; #endif - PyCodeObject *code = get_frame_code(frame); - int limit = code->co_stacksize + code->co_nlocalsplus; - (void)limit; // if NDEBUG - assert(stacktop >= 0 && stacktop < limit); + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); return stacktop; } static void set_frame_stacktop(Frame *frame, int stacktop) { #if PY_MINOR_VERSION == 10 -# error TODO + assert(frame->f_localsplus); + assert(frame->f_valuestack); + int base = (int)(frame->f_valuestack - frame->f_localsplus); + assert(stacktop >= base); + frame->f_stackdepth = stacktop - base; #elif PY_MINOR_VERSION == 11 frame->stacktop = stacktop; #elif PY_MINOR_VERSION == 12 @@ -439,14 +455,16 @@ static void set_frame_stacktop(Frame *frame, int stacktop) { static PyObject **get_frame_localsplus(Frame *frame) { #if PY_MINOR_VERSION == 10 - return frame->f_localsplus; + PyObject **localsplus = frame->f_localsplus; #elif PY_MINOR_VERSION == 11 - return frame->localsplus; + PyObject **localsplus = frame->localsplus; #elif PY_MINOR_VERSION == 12 - return frame->localsplus; + PyObject **localsplus = frame->localsplus; #elif PY_MINOR_VERSION == 13 - return frame->localsplus; + PyObject **localsplus = frame->localsplus; #endif + assert(localsplus); + return localsplus; } static PyObject *ext_get_frame_state(PyObject *self, PyObject *args) { @@ -580,8 +598,7 @@ static PyObject *ext_set_frame_sp(PyObject *self, PyObject *args) { if (!frame) { return NULL; } - PyCodeObject *code = get_frame_code(frame); - int limit = code->co_stacksize + code->co_nlocalsplus; + int limit = get_frame_stacktop_limit(frame); if (sp < 0 || sp >= limit) { PyErr_SetString(PyExc_IndexError, "Stack pointer out of bounds"); return NULL; From e7729302696d5923dc919fbf7815c8831a599c93 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 12:49:10 +1000 Subject: [PATCH 11/20] Python 3.10 doesn't have these properties --- src/dispatch/experimental/durable/function.py | 4 ++-- tests/dispatch/experimental/durable/test_coroutine.py | 5 ++++- tests/dispatch/experimental/durable/test_generator.py | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/dispatch/experimental/durable/function.py b/src/dispatch/experimental/durable/function.py index 069689e7..e988ef61 100644 --- a/src/dispatch/experimental/durable/function.py +++ b/src/dispatch/experimental/durable/function.py @@ -249,7 +249,7 @@ def cr_running(self) -> bool: @property def cr_suspended(self) -> bool: - return self.coroutine.cr_suspended + return getattr(self.coroutine, "cr_suspended", False) @property def cr_code(self) -> CodeType: @@ -315,7 +315,7 @@ def gi_running(self) -> bool: @property def gi_suspended(self) -> bool: - return self.generator.gi_suspended + return getattr(self.generator, "gi_suspended", False) @property def gi_code(self) -> CodeType: diff --git a/tests/dispatch/experimental/durable/test_coroutine.py b/tests/dispatch/experimental/durable/test_coroutine.py index 92263f00..d27db033 100644 --- a/tests/dispatch/experimental/durable/test_coroutine.py +++ b/tests/dispatch/experimental/durable/test_coroutine.py @@ -97,7 +97,10 @@ def test_export_cr_fields(self): def check(): self.assertEqual(c.cr_running, underlying.cr_running) - self.assertEqual(c.cr_suspended, underlying.cr_suspended) + try: + self.assertEqual(c.cr_suspended, underlying.cr_suspended) + except AttributeError: + pass self.assertEqual(c.cr_origin, underlying.cr_origin) self.assertIs(c.cr_await, underlying.cr_await) diff --git a/tests/dispatch/experimental/durable/test_generator.py b/tests/dispatch/experimental/durable/test_generator.py index e5018b78..ff0424be 100644 --- a/tests/dispatch/experimental/durable/test_generator.py +++ b/tests/dispatch/experimental/durable/test_generator.py @@ -99,7 +99,10 @@ def test_export_gi_fields(self): def check(): self.assertEqual(g.gi_running, underlying.gi_running) - self.assertEqual(g.gi_suspended, underlying.gi_suspended) + try: + self.assertEqual(g.gi_suspended, underlying.gi_suspended) + except AttributeError: + pass self.assertIs(g.gi_yieldfrom, underlying.gi_yieldfrom) check() From cd4838c2501c4153c843c5bb644345480207529c Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 16:26:50 +1000 Subject: [PATCH 12/20] Save/restore the block pointer --- src/dispatch/experimental/durable/frame.c | 72 +++++++++++++++++++ src/dispatch/experimental/durable/frame.pyi | 6 ++ src/dispatch/experimental/durable/function.py | 8 ++- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index f4cad3eb..7da9e652 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -438,6 +438,7 @@ static int get_frame_stacktop(Frame *frame) { } static void set_frame_stacktop(Frame *frame, int stacktop) { + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); #if PY_MINOR_VERSION == 10 assert(frame->f_localsplus); assert(frame->f_valuestack); @@ -467,6 +468,27 @@ static PyObject **get_frame_localsplus(Frame *frame) { return localsplus; } +static int get_frame_iblock_limit(Frame *frame) { +#if PY_MINOR_VERSION == 10 + return CO_MAXBLOCKS; +#endif + return 1; // not applicable >= 3.11 +} + +static int get_frame_iblock(Frame *frame) { +#if PY_MINOR_VERSION == 10 + return frame->f_iblock; +#endif + return 0; // not applicable >= 3.11 +} + +void set_frame_iblock(Frame *frame, int iblock) { + assert(iblock >= 0 && iblock < get_frame_iblock_limit(frame)); +#if PY_MINOR_VERSION == 10 + frame->f_iblock = iblock; +#endif +} + static PyObject *ext_get_frame_state(PyObject *self, PyObject *args) { PyObject *arg; if (!PyArg_ParseTuple(args, "O", &arg)) { @@ -522,6 +544,27 @@ static PyObject *ext_get_frame_sp(PyObject *self, PyObject *args) { return PyLong_FromLong((long)sp); } +static PyObject *ext_get_frame_bp(PyObject *self, PyObject *args) { + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + PyGenObject *gen_like = get_generator_like_object(obj); + if (!gen_like) { + return NULL; + } + if (get_frame_state(gen_like) >= FRAME_CLEARED) { + PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); + return NULL; + } + Frame *frame = get_frame(gen_like); + if (!frame) { + return NULL; + } + int bp = get_frame_iblock(frame); + return PyLong_FromLong((long)bp); +} + static PyObject *ext_get_frame_stack_at(PyObject *self, PyObject *args) { PyObject *obj; int index; @@ -614,6 +657,33 @@ static PyObject *ext_set_frame_sp(PyObject *self, PyObject *args) { Py_RETURN_NONE; } +static PyObject *ext_set_frame_bp(PyObject *self, PyObject *args) { + PyObject *obj; + int bp; + if (!PyArg_ParseTuple(args, "Oi", &obj, &bp)) { + return NULL; + } + PyGenObject *gen_like = get_generator_like_object(obj); + if (!gen_like) { + return NULL; + } + if (get_frame_state(gen_like) >= FRAME_CLEARED) { + PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); + return NULL; + } + Frame *frame = get_frame(gen_like); + if (!frame) { + return NULL; + } + int limit = get_frame_iblock_limit(frame); + if (bp < 0 || bp >= limit) { + PyErr_SetString(PyExc_IndexError, "Block pointer out of bounds"); + return NULL; + } + set_frame_iblock(frame, bp); + Py_RETURN_NONE; +} + static PyObject *ext_set_frame_state(PyObject *self, PyObject *args) { PyObject *obj; int fs; @@ -690,6 +760,8 @@ static PyMethodDef methods[] = { {"set_frame_ip", ext_set_frame_ip, METH_VARARGS, "Set instruction pointer of a generator or coroutine."}, {"get_frame_sp", ext_get_frame_sp, METH_VARARGS, "Get stack pointer of a generator or coroutine."}, {"set_frame_sp", ext_set_frame_sp, METH_VARARGS, "Set stack pointer of a generator or coroutine."}, + {"get_frame_bp", ext_get_frame_bp, METH_VARARGS, "Get block pointer of a generator or coroutine."}, + {"set_frame_bp", ext_set_frame_bp, METH_VARARGS, "Set block pointer of a generator or coroutine."}, {"get_frame_stack_at", ext_get_frame_stack_at, METH_VARARGS, "Get an object from a generator or coroutine's stack, as an (is_null, obj) tuple."}, {"set_frame_stack_at", ext_set_frame_stack_at, METH_VARARGS, "Set or unset an object on the stack of a generator or coroutine."}, {"get_frame_state", ext_get_frame_state, METH_VARARGS, "Get frame state of a generator or coroutine."}, diff --git a/src/dispatch/experimental/durable/frame.pyi b/src/dispatch/experimental/durable/frame.pyi index 5a2ab953..0a094c3c 100644 --- a/src/dispatch/experimental/durable/frame.pyi +++ b/src/dispatch/experimental/durable/frame.pyi @@ -13,6 +13,12 @@ def get_frame_sp(frame: FrameType | Coroutine | Generator | AsyncGenerator) -> i def set_frame_sp(frame: FrameType | Coroutine | Generator | AsyncGenerator, sp: int): """Set stack pointer of a generator or coroutine.""" +def get_frame_bp(frame: FrameType | Coroutine | Generator | AsyncGenerator) -> int: + """Get block pointer of a generator or coroutine.""" + +def set_frame_bp(frame: FrameType | Coroutine | Generator | AsyncGenerator, bp: int): + """Set block pointer of a generator or coroutine.""" + def get_frame_stack_at( frame: FrameType | Coroutine | Generator | AsyncGenerator, index: int ) -> Tuple[bool, Any]: diff --git a/src/dispatch/experimental/durable/function.py b/src/dispatch/experimental/durable/function.py index e988ef61..e4ddc6f1 100644 --- a/src/dispatch/experimental/durable/function.py +++ b/src/dispatch/experimental/durable/function.py @@ -109,9 +109,10 @@ def __getstate__(self): if frame_state < FRAME_CLEARED: ip = ext.get_frame_ip(g) sp = ext.get_frame_sp(g) + bp = ext.get_frame_bp(g) stack = [ext.get_frame_stack_at(g, i) for i in range(ext.get_frame_sp(g))] else: - ip, sp, stack = None, None, None + ip, sp, bp, stack = None, None, None, None if TRACE: print(f"\n[DISPATCH] Serializing {self}:") @@ -124,6 +125,7 @@ def __getstate__(self): if frame_state < FRAME_CLEARED: print(f"IP = {ip}") print(f"SP = {sp}") + print(f"BP = {bp}") for i, (is_null, value) in enumerate(stack): if is_null: print(f"stack[{i}] = NULL") @@ -144,6 +146,7 @@ def __getstate__(self): "frame": { "ip": ip, "sp": sp, + "bp": bp, "stack": stack, "state": frame_state, }, @@ -182,9 +185,10 @@ def __setstate__(self, state): else: g = rfn.fn(*args, **kwargs) - # Restore the frame state (stack + stack pointer + instruction pointer). + # Restore the frame. ext.set_frame_ip(g, frame_state["ip"]) ext.set_frame_sp(g, frame_state["sp"]) + ext.set_frame_bp(g, frame_state["bp"]) for i, (is_null, obj) in enumerate(frame_state["stack"]): ext.set_frame_stack_at(g, i, is_null, obj) ext.set_frame_state(g, frame_state["state"]) From c853b9a93eecd8e717f7f6fa9cf62c43ccf3a9b1 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Mon, 25 Mar 2024 16:43:43 +1000 Subject: [PATCH 13/20] Save/restore blocks --- src/dispatch/experimental/durable/frame.c | 75 +++++++++++++++++++ src/dispatch/experimental/durable/frame.pyi | 12 +++ src/dispatch/experimental/durable/function.py | 14 +++- 3 files changed, 97 insertions(+), 4 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 7da9e652..4f067268 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -487,6 +487,17 @@ void set_frame_iblock(Frame *frame, int iblock) { #if PY_MINOR_VERSION == 10 frame->f_iblock = iblock; #endif + assert(!iblock); // not applicable >= 3.11 +} + +static PyTryBlock *get_frame_blockstack(Frame *frame) { +#if PY_MINOR_VERSION == 10 + PyTryBlock *blockstack = frame->f_blockstack; +#else + PyTryBlock *blockstack = NULL; // not applicable >= 3.11 +#endif + assert(blockstack); + return blockstack; } static PyObject *ext_get_frame_state(PyObject *self, PyObject *args) { @@ -601,6 +612,34 @@ static PyObject *ext_get_frame_stack_at(PyObject *self, PyObject *args) { return PyTuple_Pack(2, is_null, stack_obj); } +static PyObject *ext_get_frame_block_at(PyObject *self, PyObject *args) { + PyObject *obj; + int index; + if (!PyArg_ParseTuple(args, "Oi", &obj, &index)) { + return NULL; + } + PyGenObject *gen_like = get_generator_like_object(obj); + if (!gen_like) { + return NULL; + } + if (get_frame_state(gen_like) >= FRAME_CLEARED) { + PyErr_SetString(PyExc_RuntimeError, "Cannot access cleared frame"); + return NULL; + } + Frame *frame = get_frame(gen_like); + if (!frame) { + return NULL; + } + int bp = get_frame_iblock(frame); + if (index < 0 || index >= bp) { + PyErr_SetString(PyExc_IndexError, "Index out of bounds"); + return NULL; + } + PyTryBlock *blockstack = get_frame_blockstack(frame); + PyTryBlock *block = &blockstack[index]; + return PyTuple_Pack(3, block->b_type, block->b_handler, block->b_level); +} + static PyObject *ext_set_frame_ip(PyObject *self, PyObject *args) { PyObject *obj; int ip; @@ -755,6 +794,40 @@ static PyObject *ext_set_frame_stack_at(PyObject *self, PyObject *args) { Py_RETURN_NONE; } +static PyObject *ext_set_frame_block_at(PyObject *self, PyObject *args) { + PyObject *obj; + int index; + int b_type; + int b_handler; + int b_level; + if (!PyArg_ParseTuple(args, "Oi(iii)", &obj, &index, &b_type, &b_handler, &b_level)) { + return NULL; + } + PyGenObject *gen_like = get_generator_like_object(obj); + if (!gen_like) { + return NULL; + } + if (get_frame_state(gen_like) >= FRAME_CLEARED) { + PyErr_SetString(PyExc_RuntimeError, "Cannot mutate cleared frame"); + return NULL; + } + Frame *frame = get_frame(gen_like); + if (!frame) { + return NULL; + } + int bp = get_frame_iblock(frame); + if (index < 0 || index >= bp) { + PyErr_SetString(PyExc_IndexError, "Index out of bounds"); + return NULL; + } + PyTryBlock *blockstack = get_frame_blockstack(frame); + PyTryBlock *block = &blockstack[index]; + block->b_type = b_type; + block->b_handler = b_handler; + block->b_level = b_level; + Py_RETURN_NONE; +} + static PyMethodDef methods[] = { {"get_frame_ip", ext_get_frame_ip, METH_VARARGS, "Get instruction pointer of a generator or coroutine."}, {"set_frame_ip", ext_set_frame_ip, METH_VARARGS, "Set instruction pointer of a generator or coroutine."}, @@ -764,6 +837,8 @@ static PyMethodDef methods[] = { {"set_frame_bp", ext_set_frame_bp, METH_VARARGS, "Set block pointer of a generator or coroutine."}, {"get_frame_stack_at", ext_get_frame_stack_at, METH_VARARGS, "Get an object from a generator or coroutine's stack, as an (is_null, obj) tuple."}, {"set_frame_stack_at", ext_set_frame_stack_at, METH_VARARGS, "Set or unset an object on the stack of a generator or coroutine."}, + {"get_frame_block_at", ext_get_frame_block_at, METH_VARARGS, "Get a block from a generator or coroutine."}, + {"set_frame_block_at", ext_set_frame_block_at, METH_VARARGS, "Restore a block of a generator or coroutine."}, {"get_frame_state", ext_get_frame_state, METH_VARARGS, "Get frame state of a generator or coroutine."}, {"set_frame_state", ext_set_frame_state, METH_VARARGS, "Set frame state of a generator or coroutine."}, {NULL, NULL, 0, NULL} diff --git a/src/dispatch/experimental/durable/frame.pyi b/src/dispatch/experimental/durable/frame.pyi index 0a094c3c..e701afd0 100644 --- a/src/dispatch/experimental/durable/frame.pyi +++ b/src/dispatch/experimental/durable/frame.pyi @@ -32,6 +32,18 @@ def set_frame_stack_at( ): """Set or unset an object on the stack of a generator or coroutine.""" +def get_frame_block_at( + frame: FrameType | Coroutine | Generator | AsyncGenerator, index: int +) -> Tuple[int, int, int]: + """Get a block from a generator or coroutine.""" + +def set_frame_block_at( + frame: FrameType | Coroutine | Generator | AsyncGenerator, + index: int, + value: Tuple[int, int, int], +): + """Restore a block of a generator or coroutine.""" + def get_frame_state( frame: FrameType | Coroutine | Generator | AsyncGenerator, ) -> int: diff --git a/src/dispatch/experimental/durable/function.py b/src/dispatch/experimental/durable/function.py index e4ddc6f1..a740be41 100644 --- a/src/dispatch/experimental/durable/function.py +++ b/src/dispatch/experimental/durable/function.py @@ -110,9 +110,10 @@ def __getstate__(self): ip = ext.get_frame_ip(g) sp = ext.get_frame_sp(g) bp = ext.get_frame_bp(g) - stack = [ext.get_frame_stack_at(g, i) for i in range(ext.get_frame_sp(g))] + stack = [ext.get_frame_stack_at(g, i) for i in range(sp)] + blocks = [ext.get_frame_block_at(g, i) for i in range(bp)] else: - ip, sp, bp, stack = None, None, None, None + ip, sp, bp, stack, blocks = None, None, None, None, None if TRACE: print(f"\n[DISPATCH] Serializing {self}:") @@ -125,12 +126,14 @@ def __getstate__(self): if frame_state < FRAME_CLEARED: print(f"IP = {ip}") print(f"SP = {sp}") - print(f"BP = {bp}") for i, (is_null, value) in enumerate(stack): if is_null: print(f"stack[{i}] = NULL") else: print(f"stack[{i}] = {value}") + print(f"BP = {bp}") + for i, block in enumerate(blocks): + print(f"block[{i}] = {block}") print() state = { @@ -148,6 +151,7 @@ def __getstate__(self): "sp": sp, "bp": bp, "stack": stack, + "blocks": blocks, "state": frame_state, }, } @@ -188,9 +192,11 @@ def __setstate__(self, state): # Restore the frame. ext.set_frame_ip(g, frame_state["ip"]) ext.set_frame_sp(g, frame_state["sp"]) - ext.set_frame_bp(g, frame_state["bp"]) for i, (is_null, obj) in enumerate(frame_state["stack"]): ext.set_frame_stack_at(g, i, is_null, obj) + ext.set_frame_bp(g, frame_state["bp"]) + for i, block in enumerate(frame_state["blocks"]): + ext.set_frame_block_at(g, i, block) ext.set_frame_state(g, frame_state["state"]) else: g = None From f40efa33e8dc8f6768d84a35a00e9db46c95c6ee Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 27 Mar 2024 10:46:28 +1000 Subject: [PATCH 14/20] Frame object isn't available once frame is cleared --- src/dispatch/experimental/durable/frame.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 4f067268..6ddc3459 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -373,7 +373,11 @@ void set_frame_lasti(Frame *frame, int lasti) { static int get_frame_state(PyGenObject *gen_like) { #if PY_MINOR_VERSION == 10 - return get_frame(gen_like)->f_state; + Frame *frame = (Frame *)(gen_like->gi_frame); + if (!frame) { + return FRAME_CLEARED; + } + return frame->f_state; #elif PY_MINOR_VERSION == 11 return gen_like->gi_frame_state; #elif PY_MINOR_VERSION == 12 From 28a84466b8ca201541eb1e5b859ab3e8690dddc1 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 27 Mar 2024 10:55:56 +1000 Subject: [PATCH 15/20] Fix construction of tuple --- src/dispatch/experimental/durable/frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 6ddc3459..394a1df1 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -641,7 +641,7 @@ static PyObject *ext_get_frame_block_at(PyObject *self, PyObject *args) { } PyTryBlock *blockstack = get_frame_blockstack(frame); PyTryBlock *block = &blockstack[index]; - return PyTuple_Pack(3, block->b_type, block->b_handler, block->b_level); + return Py_BuildValue("(iii)", block->b_type, block->b_handler, block->b_level); } static PyObject *ext_set_frame_ip(PyObject *self, PyObject *args) { From 5e5165e8588b302fddc9633314f8a2ba979a09be Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 27 Mar 2024 10:57:09 +1000 Subject: [PATCH 16/20] Disable assertion on 3.10 --- src/dispatch/experimental/durable/frame.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 394a1df1..18508030 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -490,8 +490,9 @@ void set_frame_iblock(Frame *frame, int iblock) { assert(iblock >= 0 && iblock < get_frame_iblock_limit(frame)); #if PY_MINOR_VERSION == 10 frame->f_iblock = iblock; -#endif +#else assert(!iblock); // not applicable >= 3.11 +#endif } static PyTryBlock *get_frame_blockstack(Frame *frame) { From c2fb7bcecfb1fd5db20e0b1c5c328719cc11dca9 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 27 Mar 2024 11:05:54 +1000 Subject: [PATCH 17/20] Add 3.10 to the test matrix --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce8bf76d..1acf51d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.11', '3.12'] + python: ['3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} From 932a6efdbbeea910f7673fdf7e7709be90dd3b84 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 27 Mar 2024 11:12:45 +1000 Subject: [PATCH 18/20] Fix comment --- src/dispatch/experimental/durable/frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 18508030..21aa4f32 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -22,7 +22,7 @@ typedef struct _PyTryBlock { // This is a redefinition of the private/opaque frame object. // In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and -// PyFrame. From Python 3.11 onwards, the two were split with the +// PyInterpreterFrame. From Python 3.11 onwards, the two were split with the // PyFrameObject (struct _frame) pointing to struct _PyInterpreterFrame. typedef struct Frame { #if PY_MINOR_VERSION == 10 From a1b32243c74336fddff05c86ae2d05cbcc3bb56c Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 27 Mar 2024 11:43:36 +1000 Subject: [PATCH 19/20] Mark internal functions as static --- src/dispatch/experimental/durable/frame.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 21aa4f32..3a7db9ef 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -353,7 +353,7 @@ static int get_frame_lasti(Frame *frame) { #endif } -void set_frame_lasti(Frame *frame, int lasti) { +static void set_frame_lasti(Frame *frame, int lasti) { #if PY_MINOR_VERSION == 10 frame->f_lasti = lasti; #elif PY_MINOR_VERSION == 11 @@ -486,7 +486,7 @@ static int get_frame_iblock(Frame *frame) { return 0; // not applicable >= 3.11 } -void set_frame_iblock(Frame *frame, int iblock) { +static void set_frame_iblock(Frame *frame, int iblock) { assert(iblock >= 0 && iblock < get_frame_iblock_limit(frame)); #if PY_MINOR_VERSION == 10 frame->f_iblock = iblock; From aaf9c26bfdd782c66be3d95a96f87e94f76f7264 Mon Sep 17 00:00:00 2001 From: Chris O'Hara Date: Wed, 27 Mar 2024 12:14:59 +1000 Subject: [PATCH 20/20] Split differences across version headers --- src/dispatch/experimental/durable/frame.c | 461 ++----------------- src/dispatch/experimental/durable/frame310.h | 151 ++++++ src/dispatch/experimental/durable/frame311.h | 148 ++++++ src/dispatch/experimental/durable/frame312.h | 148 ++++++ src/dispatch/experimental/durable/frame313.h | 148 ++++++ 5 files changed, 624 insertions(+), 432 deletions(-) create mode 100644 src/dispatch/experimental/durable/frame310.h create mode 100644 src/dispatch/experimental/durable/frame311.h create mode 100644 src/dispatch/experimental/durable/frame312.h create mode 100644 src/dispatch/experimental/durable/frame313.h diff --git a/src/dispatch/experimental/durable/frame.c b/src/dispatch/experimental/durable/frame.c index 3a7db9ef..1cfb6c6a 100644 --- a/src/dispatch/experimental/durable/frame.c +++ b/src/dispatch/experimental/durable/frame.c @@ -10,123 +10,15 @@ # error Python 3.10-3.13 is required #endif -// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L20 -typedef int8_t PyFrameState; - +// This is a redefinition of the private PyTryBlock from 3.10. // https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L22 -typedef struct _PyTryBlock { +typedef struct { int b_type; int b_handler; int b_level; } PyTryBlock; -// This is a redefinition of the private/opaque frame object. -// In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and -// PyInterpreterFrame. From Python 3.11 onwards, the two were split with the -// PyFrameObject (struct _frame) pointing to struct _PyInterpreterFrame. -typedef struct Frame { -#if PY_MINOR_VERSION == 10 -// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 - PyObject_VAR_HEAD - struct Frame *f_back; // struct _frame - PyCodeObject *f_code; - PyObject *f_builtins; - PyObject *f_globals; - PyObject *f_locals; - PyObject **f_valuestack; - PyObject *f_trace; - int f_stackdepth; - char f_trace_lines; - char f_trace_opcodes; - PyObject *f_gen; - int f_lasti; - int f_lineno; - int f_iblock; - PyFrameState f_state; - PyTryBlock f_blockstack[CO_MAXBLOCKS]; - PyObject *f_localsplus[1]; -#elif PY_MINOR_VERSION == 11 -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L47 - PyFunctionObject *f_func; - PyObject *f_globals; - PyObject *f_builtins; - PyObject *f_locals; - PyCodeObject *f_code; - PyFrameObject *frame_obj; - struct Frame *previous; // struct _PyInterpreterFrame - _Py_CODEUNIT *prev_instr; - int stacktop; - bool is_entry; - char owner; - PyObject *localsplus[1]; -#elif PY_MINOR_VERSION == 12 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L51 - PyCodeObject *f_code; - struct Frame *previous; // struct _PyInterpreterFrame - PyObject *f_funcobj; - PyObject *f_globals; - PyObject *f_builtins; - PyObject *f_locals; - PyFrameObject *frame_obj; - _Py_CODEUNIT *prev_instr; - int stacktop; - uint16_t return_offset; - char owner; - PyObject *localsplus[1]; -#elif PY_MINOR_VERSION == 13 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L57 - PyObject *f_executable; - struct Frame *previous; // struct _PyInterpreterFrame - PyObject *f_funcobj; - PyObject *f_globals; - PyObject *f_builtins; - PyObject *f_locals; - PyFrameObject *frame_obj; - _Py_CODEUNIT *instr_ptr; - int stacktop; - uint16_t return_offset; - char owner; - PyObject *localsplus[1]; -#endif -} Frame; - -// This is a redefinition of private frame state constants: -typedef enum _framestate { -#if PY_MINOR_VERSION == 10 -// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L10 - FRAME_CREATED = -2, - FRAME_SUSPENDED = -1, - FRAME_EXECUTING = 0, - FRAME_RETURNED = 1, - FRAME_UNWINDING = 2, - FRAME_RAISED = 3, - FRAME_CLEARED = 4 -#elif PY_MINOR_VERSION == 11 -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L33 - FRAME_CREATED = -2, - FRAME_SUSPENDED = -1, - FRAME_EXECUTING = 0, - FRAME_COMPLETED = 1, - FRAME_CLEARED = 4 -#elif PY_MINOR_VERSION == 12 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L34 - FRAME_CREATED = -2, - FRAME_SUSPENDED = -1, - FRAME_EXECUTING = 0, - FRAME_COMPLETED = 1, - FRAME_CLEARED = 4 -#elif PY_MINOR_VERSION == 13 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L38 - FRAME_CREATED = -3, - FRAME_SUSPENDED = -2, - FRAME_SUSPENDED_YIELD_FROM = -1, - FRAME_EXECUTING = 0, - FRAME_COMPLETED = 1, - FRAME_CLEARED = 4 -#endif -} FrameState; - -// This is a redefinition of the private PyCoroWrapper: +// This is a redefinition of the private PyCoroWrapper from 3.10-3.13. // https://github.com/python/cpython/blob/3.10/Objects/genobject.c#L884 // https://github.com/python/cpython/blob/3.11/Objects/genobject.c#L1016 // https://github.com/python/cpython/blob/3.12/Objects/genobject.c#L1003 @@ -136,132 +28,38 @@ typedef struct { PyCoroObject *cw_coroutine; } PyCoroWrapper; -/* -// This is the definition of PyFrameObject (aka. struct _frame) for reference -// to developers working on the extension. As noted above, the contents of -// struct _frame changed over time, with some fields moving in Python >= 3.11 -// to a separate struct _PyInterpreterFrame. -// -typedef struct { -#if PY_MINOR_VERSION == 10 -// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 - PyObject_VAR_HEAD - struct _frame *f_back; - PyCodeObject *f_code; - PyObject *f_builtins; - PyObject *f_globals; - PyObject *f_locals; - PyObject **f_valuestack; - PyObject *f_trace; - int f_stackdepth; - char f_trace_lines; - char f_trace_opcodes; - PyObject *f_gen; - int f_lasti; - int f_lineno; - int f_iblock; - PyFrameState f_state; - PyTryBlock f_blockstack[CO_MAXBLOCKS]; - PyObject *f_localsplus[1]; -#elif PY_MINOR_VERSION == 11 -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L15 - PyObject_HEAD - PyFrameObject *f_back; - struct _PyInterpreterFrame *f_frame; - PyObject *f_trace; - int f_lineno; - char f_trace_lines; - char f_trace_opcodes; - char f_fast_as_locals; - PyObject *_f_frame_data[1]; -#elif PY_MINOR_VERSION == 12 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L16 - PyObject_HEAD - PyFrameObject *f_back; - struct _PyInterpreterFrame *f_frame; - PyObject *f_trace; - int f_lineno; - char f_trace_lines; - char f_trace_opcodes; - char f_fast_as_locals; - PyObject *_f_frame_data[1]; -#elif PY_MINOR_VERSION == 13 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L20 - PyObject_HEAD - PyFrameObject *f_back; - struct _PyInterpreterFrame *f_frame; - PyObject *f_trace; - int f_lineno; - char f_trace_lines; - char f_trace_opcodes; - char f_fast_as_locals; - PyObject *_f_frame_data[1]; -#endif -} PyFrameObject; -*/ +typedef struct Frame Frame; + +static Frame *get_frame(PyGenObject *gen_like); + +static PyCodeObject *get_frame_code(Frame *frame); + +static int get_frame_lasti(Frame *frame); +static void set_frame_lasti(Frame *frame, int lasti); + +static int get_frame_state(PyGenObject *gen_like); +static void set_frame_state(PyGenObject *gen_like, int fs); +static int valid_frame_state(int fs); + +static int get_frame_stacktop_limit(Frame *frame); +static int get_frame_stacktop(Frame *frame); +static void set_frame_stacktop(Frame *frame, int stacktop); +static PyObject **get_frame_localsplus(Frame *frame); + +static int get_frame_iblock_limit(Frame *frame); +static int get_frame_iblock(Frame *frame); +static void set_frame_iblock(Frame *frame, int iblock); +static PyTryBlock *get_frame_blockstack(Frame *frame); -/* -// This is the definition of PyGenObject for reference to developers -// working on the extension. -// -// Note that PyCoroObject and PyAsyncGenObject have the same layout as -// PyGenObject, however the struct fields have a cr_ and ag_ prefix -// (respectively) rather than a gi_ prefix. In Python 3.10, PyCoroObject -// and PyAsyncGenObject have extra fields compared to PyGenObject. In Python -// 3.11 onwards, the three objects are identical (except for field name -// prefixes). The extra fields in Python 3.10 are not applicable to this -// extension at this time. -// -typedef struct { - PyObject_HEAD #if PY_MINOR_VERSION == 10 -// https://github.com/python/cpython/blob/3.10/Include/genobject.h#L16 - PyFrameObject *gi_frame; - PyObject *gi_code; - PyObject *gi_weakreflist; - PyObject *gi_name; - PyObject *gi_qualname; - _PyErr_StackItem gi_exc_state; +#include "frame310.h" #elif PY_MINOR_VERSION == 11 -// https://github.com/python/cpython/blob/3.11/Include/cpython/genobject.h#L14 - PyCodeObject *gi_code; - PyObject *gi_weakreflist; - PyObject *gi_name; - PyObject *gi_qualname; - _PyErr_StackItem gi_exc_state; - PyObject *gi_origin_or_finalizer; - char gi_hooks_inited; - char gi_closed; - char gi_running_async; - int8_t gi_frame_state; - PyObject *gi_iframe[1]; +#include "frame311.h" #elif PY_MINOR_VERSION == 12 -// https://github.com/python/cpython/blob/3.12/Include/cpython/genobject.h#L14 - PyObject *gi_weakreflist; - PyObject *gi_name; - PyObject *gi_qualname; - _PyErr_StackItem gi_exc_state; - PyObject *gi_origin_or_finalizer; - char gi_hooks_inited; - char gi_closed; - char gi_running_async; - int8_t gi_frame_state; - PyObject *gi_iframe[1]; +#include "frame312.h" #elif PY_MINOR_VERSION == 13 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/cpython/genobject.h#L14 - PyObject *gi_weakreflist; - PyObject *gi_name; - PyObject *gi_qualname; - _PyErr_StackItem gi_exc_state; - PyObject *gi_origin_or_finalizer; - char gi_hooks_inited; - char gi_closed; - char gi_running_async; - int8_t gi_frame_state; - PyObject *gi_iframe[1]; +#include "frame313.h" #endif -} PyGenObject; -*/ static const char *get_type_name(PyObject *obj) { PyObject* type = PyObject_Type(obj); @@ -304,207 +102,6 @@ static PyGenObject *get_generator_like_object(PyObject *obj) { return NULL; } -static Frame *get_frame(PyGenObject *gen_like) { -#if PY_MINOR_VERSION == 10 - Frame *frame = (Frame *)(gen_like->gi_frame); -#elif PY_MINOR_VERSION == 11 - Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); -#elif PY_MINOR_VERSION == 12 - Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); -#elif PY_MINOR_VERSION == 13 - Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); -#endif - assert(frame); - return frame; -} - -static PyCodeObject *get_frame_code(Frame *frame) { -#if PY_MINOR_VERSION == 10 - PyCodeObject *code = frame->f_code; -#elif PY_MINOR_VERSION == 11 - PyCodeObject *code = frame->f_code; -#elif PY_MINOR_VERSION == 12 - PyCodeObject *code = frame->f_code; -#elif PY_MINOR_VERSION == 13 - PyCodeObject *code = (PyCodeObject *)frame->f_executable; -#endif - assert(code); - return code; -} - -static int get_frame_lasti(Frame *frame) { -#if PY_MINOR_VERSION == 10 - return frame->f_lasti; -#elif PY_MINOR_VERSION == 11 -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L69 - PyCodeObject *code = get_frame_code(frame); - assert(frame->prev_instr); - return (int)((intptr_t)frame->prev_instr - (intptr_t)_PyCode_CODE(code)); -#elif PY_MINOR_VERSION == 12 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 - PyCodeObject *code = get_frame_code(frame); - assert(frame->prev_instr); - return (int)((intptr_t)frame->prev_instr - (intptr_t)_PyCode_CODE(code)); -#elif PY_MINOR_VERSION == 13 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L73 - PyCodeObject *code = get_frame_code(frame); - assert(frame->instr_ptr); - return (int)((intptr_t)frame->instr_ptr - (intptr_t)_PyCode_CODE(code)); -#endif -} - -static void set_frame_lasti(Frame *frame, int lasti) { -#if PY_MINOR_VERSION == 10 - frame->f_lasti = lasti; -#elif PY_MINOR_VERSION == 11 -// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L69 - PyCodeObject *code = get_frame_code(frame); - frame->prev_instr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); -#elif PY_MINOR_VERSION == 12 -// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 - PyCodeObject *code = get_frame_code(frame); - frame->prev_instr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); -#elif PY_MINOR_VERSION == 13 -// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L73 - PyCodeObject *code = get_frame_code(frame); - frame->instr_ptr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); -#endif -} - -static int get_frame_state(PyGenObject *gen_like) { -#if PY_MINOR_VERSION == 10 - Frame *frame = (Frame *)(gen_like->gi_frame); - if (!frame) { - return FRAME_CLEARED; - } - return frame->f_state; -#elif PY_MINOR_VERSION == 11 - return gen_like->gi_frame_state; -#elif PY_MINOR_VERSION == 12 - return gen_like->gi_frame_state; -#elif PY_MINOR_VERSION == 13 - return gen_like->gi_frame_state; -#endif -} - -static void set_frame_state(PyGenObject *gen_like, int fs) { -#if PY_MINOR_VERSION == 10 - Frame *frame = get_frame(gen_like); - frame->f_state = (PyFrameState)fs; -#elif PY_MINOR_VERSION == 11 - gen_like->gi_frame_state = (int8_t)fs; -#elif PY_MINOR_VERSION == 12 - gen_like->gi_frame_state = (int8_t)fs; -#elif PY_MINOR_VERSION == 13 - gen_like->gi_frame_state = (int8_t)fs; -#endif -} - -static int valid_frame_state(int fs) { -#if PY_MINOR_VERSION == 10 - return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_RETURNED || fs == FRAME_UNWINDING || fs == FRAME_RAISED || fs == FRAME_CLEARED; -#elif PY_MINOR_VERSION == 11 - return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; -#elif PY_MINOR_VERSION == 12 - return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; -#elif PY_MINOR_VERSION == 13 - return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_SUSPENDED_YIELD_FROM || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; -#endif -} - -static int get_frame_stacktop_limit(Frame *frame) { - PyCodeObject *code = get_frame_code(frame); -#if PY_MINOR_VERSION == 10 - return code->co_stacksize + code->co_nlocals; -#elif PY_MINOR_VERSION == 11 - return code->co_stacksize + code->co_nlocalsplus; -#elif PY_MINOR_VERSION == 12 - return code->co_stacksize + code->co_nlocalsplus; -#elif PY_MINOR_VERSION == 13 - return code->co_stacksize + code->co_nlocalsplus; -#endif -} - -static int get_frame_stacktop(Frame *frame) { -#if PY_MINOR_VERSION == 10 - assert(frame->f_localsplus); - assert(frame->f_valuestack); - int stacktop = (int)(frame->f_valuestack - frame->f_localsplus) + frame->f_stackdepth; -#elif PY_MINOR_VERSION == 11 - int stacktop = frame->stacktop; -#elif PY_MINOR_VERSION == 12 - int stacktop = frame->stacktop; -#elif PY_MINOR_VERSION == 13 - int stacktop = frame->stacktop; -#endif - assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); - return stacktop; -} - -static void set_frame_stacktop(Frame *frame, int stacktop) { - assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); -#if PY_MINOR_VERSION == 10 - assert(frame->f_localsplus); - assert(frame->f_valuestack); - int base = (int)(frame->f_valuestack - frame->f_localsplus); - assert(stacktop >= base); - frame->f_stackdepth = stacktop - base; -#elif PY_MINOR_VERSION == 11 - frame->stacktop = stacktop; -#elif PY_MINOR_VERSION == 12 - frame->stacktop = stacktop; -#elif PY_MINOR_VERSION == 13 - frame->stacktop = stacktop; -#endif -} - -static PyObject **get_frame_localsplus(Frame *frame) { -#if PY_MINOR_VERSION == 10 - PyObject **localsplus = frame->f_localsplus; -#elif PY_MINOR_VERSION == 11 - PyObject **localsplus = frame->localsplus; -#elif PY_MINOR_VERSION == 12 - PyObject **localsplus = frame->localsplus; -#elif PY_MINOR_VERSION == 13 - PyObject **localsplus = frame->localsplus; -#endif - assert(localsplus); - return localsplus; -} - -static int get_frame_iblock_limit(Frame *frame) { -#if PY_MINOR_VERSION == 10 - return CO_MAXBLOCKS; -#endif - return 1; // not applicable >= 3.11 -} - -static int get_frame_iblock(Frame *frame) { -#if PY_MINOR_VERSION == 10 - return frame->f_iblock; -#endif - return 0; // not applicable >= 3.11 -} - -static void set_frame_iblock(Frame *frame, int iblock) { - assert(iblock >= 0 && iblock < get_frame_iblock_limit(frame)); -#if PY_MINOR_VERSION == 10 - frame->f_iblock = iblock; -#else - assert(!iblock); // not applicable >= 3.11 -#endif -} - -static PyTryBlock *get_frame_blockstack(Frame *frame) { -#if PY_MINOR_VERSION == 10 - PyTryBlock *blockstack = frame->f_blockstack; -#else - PyTryBlock *blockstack = NULL; // not applicable >= 3.11 -#endif - assert(blockstack); - return blockstack; -} - static PyObject *ext_get_frame_state(PyObject *self, PyObject *args) { PyObject *arg; if (!PyArg_ParseTuple(args, "O", &arg)) { diff --git a/src/dispatch/experimental/durable/frame310.h b/src/dispatch/experimental/durable/frame310.h new file mode 100644 index 00000000..30baebec --- /dev/null +++ b/src/dispatch/experimental/durable/frame310.h @@ -0,0 +1,151 @@ +// This is a redefinition of the private PyFrameState. +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L20 +typedef int8_t PyFrameState; + +// This is a redefinition of the private/opaque frame object. +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L28 +// +// In Python 3.10, `struct _frame` is both the PyFrameObject and +// PyInterpreterFrame. From Python 3.11 onwards, the two were split with the +// PyFrameObject (struct _frame) pointing to struct _PyInterpreterFrame. +struct Frame { + PyObject_VAR_HEAD + struct Frame *f_back; // struct _frame + PyCodeObject *f_code; + PyObject *f_builtins; + PyObject *f_globals; + PyObject *f_locals; + PyObject **f_valuestack; + PyObject *f_trace; + int f_stackdepth; + char f_trace_lines; + char f_trace_opcodes; + PyObject *f_gen; + int f_lasti; + int f_lineno; + int f_iblock; + PyFrameState f_state; + PyTryBlock f_blockstack[CO_MAXBLOCKS]; + PyObject *f_localsplus[1]; +}; + +// This is a redefinition of private frame state constants. +// https://github.com/python/cpython/blob/3.10/Include/cpython/frameobject.h#L10 +typedef enum _framestate { + FRAME_CREATED = -2, + FRAME_SUSPENDED = -1, + FRAME_EXECUTING = 0, + FRAME_RETURNED = 1, + FRAME_UNWINDING = 2, + FRAME_RAISED = 3, + FRAME_CLEARED = 4 +} FrameState; + +/* +// This is the definition of PyGenObject for reference to developers +// working on the extension. +// +// Note that PyCoroObject and PyAsyncGenObject have the same layout as +// PyGenObject, however the struct fields have a cr_ and ag_ prefix +// (respectively) rather than a gi_ prefix. In Python 3.10, PyCoroObject +// and PyAsyncGenObject have extra fields compared to PyGenObject. In Python +// 3.11 onwards, the three objects are identical (except for field name +// prefixes). The extra fields in Python 3.10 are not applicable to this +// extension at this time. +// +// https://github.com/python/cpython/blob/3.10/Include/genobject.h#L16 +typedef struct { + PyObject_HEAD + PyFrameObject *gi_frame; + PyObject *gi_code; + PyObject *gi_weakreflist; + PyObject *gi_name; + PyObject *gi_qualname; + _PyErr_StackItem gi_exc_state; +} PyGenObject; +*/ + +static Frame *get_frame(PyGenObject *gen_like) { + Frame *frame = (Frame *)(gen_like->gi_frame); + assert(frame); + return frame; +} + +static PyCodeObject *get_frame_code(Frame *frame) { + PyCodeObject *code = frame->f_code; + assert(code); + return code; +} + +static int get_frame_lasti(Frame *frame) { + return frame->f_lasti; +} + +static void set_frame_lasti(Frame *frame, int lasti) { + frame->f_lasti = lasti; +} + +static int get_frame_state(PyGenObject *gen_like) { + Frame *frame = (Frame *)(gen_like->gi_frame); + if (!frame) { + return FRAME_CLEARED; + } + return frame->f_state; +} + +static void set_frame_state(PyGenObject *gen_like, int fs) { + Frame *frame = get_frame(gen_like); + frame->f_state = (PyFrameState)fs; +} + +static int valid_frame_state(int fs) { + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_RETURNED || fs == FRAME_UNWINDING || fs == FRAME_RAISED || fs == FRAME_CLEARED; +} + +static int get_frame_stacktop_limit(Frame *frame) { + PyCodeObject *code = get_frame_code(frame); + return code->co_stacksize + code->co_nlocals; +} + +static int get_frame_stacktop(Frame *frame) { + assert(frame->f_localsplus); + assert(frame->f_valuestack); + int stacktop = (int)(frame->f_valuestack - frame->f_localsplus) + frame->f_stackdepth; + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + return stacktop; +} + +static void set_frame_stacktop(Frame *frame, int stacktop) { + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + assert(frame->f_localsplus); + assert(frame->f_valuestack); + int base = (int)(frame->f_valuestack - frame->f_localsplus); + assert(stacktop >= base); + frame->f_stackdepth = stacktop - base; +} + +static PyObject **get_frame_localsplus(Frame *frame) { + PyObject **localsplus = frame->f_localsplus; + assert(localsplus); + return localsplus; +} + +static int get_frame_iblock_limit(Frame *frame) { + return CO_MAXBLOCKS; +} + +static int get_frame_iblock(Frame *frame) { + return frame->f_iblock; +} + +static void set_frame_iblock(Frame *frame, int iblock) { + assert(iblock >= 0 && iblock < get_frame_iblock_limit(frame)); + frame->f_iblock = iblock; +} + +static PyTryBlock *get_frame_blockstack(Frame *frame) { + PyTryBlock *blockstack = frame->f_blockstack; + assert(blockstack); + return blockstack; +} + diff --git a/src/dispatch/experimental/durable/frame311.h b/src/dispatch/experimental/durable/frame311.h new file mode 100644 index 00000000..e75d8549 --- /dev/null +++ b/src/dispatch/experimental/durable/frame311.h @@ -0,0 +1,148 @@ +// This is a redefinition of the private/opaque frame object. +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L47 +// +// In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and +// PyInterpreterFrame. From Python 3.11 onwards, the two were split with the +// PyFrameObject (struct _frame) pointing to struct _PyInterpreterFrame. +struct Frame { + PyFunctionObject *f_func; + PyObject *f_globals; + PyObject *f_builtins; + PyObject *f_locals; + PyCodeObject *f_code; + PyFrameObject *frame_obj; + struct Frame *previous; // struct _PyInterpreterFrame + _Py_CODEUNIT *prev_instr; + int stacktop; + bool is_entry; + char owner; + PyObject *localsplus[1]; +}; + +// This is a redefinition of private frame state constants. +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L33 +typedef enum _framestate { + FRAME_CREATED = -2, + FRAME_SUSPENDED = -1, + FRAME_EXECUTING = 0, + FRAME_COMPLETED = 1, + FRAME_CLEARED = 4 +} FrameState; + +/* +// This is the definition of PyFrameObject (aka. struct _frame) for reference +// to developers working on the extension. +// +// https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L15 +typedef struct { + PyObject_HEAD + PyFrameObject *f_back; + struct _PyInterpreterFrame *f_frame; + PyObject *f_trace; + int f_lineno; + char f_trace_lines; + char f_trace_opcodes; + char f_fast_as_locals; + PyObject *_f_frame_data[1]; +} PyFrameObject; +*/ + +/* +// This is the definition of PyGenObject for reference to developers +// working on the extension. +// +// Note that PyCoroObject and PyAsyncGenObject have the same layout as +// PyGenObject, however the struct fields have a cr_ and ag_ prefix +// (respectively) rather than a gi_ prefix. +// +// https://github.com/python/cpython/blob/3.11/Include/cpython/genobject.h#L14 +typedef struct { + PyObject_HEAD + PyCodeObject *gi_code; + PyObject *gi_weakreflist; + PyObject *gi_name; + PyObject *gi_qualname; + _PyErr_StackItem gi_exc_state; + PyObject *gi_origin_or_finalizer; + char gi_hooks_inited; + char gi_closed; + char gi_running_async; + int8_t gi_frame_state; + PyObject *gi_iframe[1]; +} PyGenObject; +*/ + +static Frame *get_frame(PyGenObject *gen_like) { + Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); + assert(frame); + return frame; +} + +static PyCodeObject *get_frame_code(Frame *frame) { + PyCodeObject *code = frame->f_code; + assert(code); + return code; +} + +static int get_frame_lasti(Frame *frame) { + // https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L69 + PyCodeObject *code = get_frame_code(frame); + assert(frame->prev_instr); + return (int)((intptr_t)frame->prev_instr - (intptr_t)_PyCode_CODE(code)); +} + +static void set_frame_lasti(Frame *frame, int lasti) { + // https://github.com/python/cpython/blob/3.11/Include/internal/pycore_frame.h#L69 + PyCodeObject *code = get_frame_code(frame); + frame->prev_instr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); +} + +static int get_frame_state(PyGenObject *gen_like) { + return gen_like->gi_frame_state; +} + +static void set_frame_state(PyGenObject *gen_like, int fs) { + gen_like->gi_frame_state = (int8_t)fs; +} + +static int valid_frame_state(int fs) { + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; +} + +static int get_frame_stacktop_limit(Frame *frame) { + PyCodeObject *code = get_frame_code(frame); + return code->co_stacksize + code->co_nlocalsplus; +} + +static int get_frame_stacktop(Frame *frame) { + int stacktop = frame->stacktop; + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + return stacktop; +} + +static void set_frame_stacktop(Frame *frame, int stacktop) { + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + frame->stacktop = stacktop; +} + +static PyObject **get_frame_localsplus(Frame *frame) { + PyObject **localsplus = frame->localsplus; + assert(localsplus); + return localsplus; +} + +static int get_frame_iblock_limit(Frame *frame) { + return 1; // not applicable >= 3.11 +} + +static int get_frame_iblock(Frame *frame) { + return 0; // not applicable >= 3.11 +} + +static void set_frame_iblock(Frame *frame, int iblock) { + assert(!iblock); // not applicable >= 3.11 +} + +static PyTryBlock *get_frame_blockstack(Frame *frame) { + return NULL; // not applicable >= 3.11 +} diff --git a/src/dispatch/experimental/durable/frame312.h b/src/dispatch/experimental/durable/frame312.h new file mode 100644 index 00000000..6ca69044 --- /dev/null +++ b/src/dispatch/experimental/durable/frame312.h @@ -0,0 +1,148 @@ +// This is a redefinition of the private/opaque frame object. +// +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L51 +// +// In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and +// PyInterpreterFrame. From Python 3.11 onwards, the two were split with the +// PyFrameObject (struct _frame) pointing to struct _PyInterpreterFrame. +struct Frame { + PyCodeObject *f_code; + struct Frame *previous; // struct _PyInterpreterFrame + PyObject *f_funcobj; + PyObject *f_globals; + PyObject *f_builtins; + PyObject *f_locals; + PyFrameObject *frame_obj; + _Py_CODEUNIT *prev_instr; + int stacktop; + uint16_t return_offset; + char owner; + PyObject *localsplus[1]; +}; + +// This is a redefinition of private frame state constants. +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L34 +typedef enum _framestate { + FRAME_CREATED = -2, + FRAME_SUSPENDED = -1, + FRAME_EXECUTING = 0, + FRAME_COMPLETED = 1, + FRAME_CLEARED = 4 +} FrameState; + +/* +// This is the definition of PyFrameObject (aka. struct _frame) for reference +// to developers working on the extension. +// +// https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L16 +typedef struct { + PyObject_HEAD + PyFrameObject *f_back; + struct _PyInterpreterFrame *f_frame; + PyObject *f_trace; + int f_lineno; + char f_trace_lines; + char f_trace_opcodes; + char f_fast_as_locals; + PyObject *_f_frame_data[1]; +} PyFrameObject; +*/ + +/* +// This is the definition of PyGenObject for reference to developers +// working on the extension. +// +// Note that PyCoroObject and PyAsyncGenObject have the same layout as +// PyGenObject, however the struct fields have a cr_ and ag_ prefix +// (respectively) rather than a gi_ prefix. +// +// https://github.com/python/cpython/blob/3.12/Include/cpython/genobject.h#L14 +typedef struct { + PyObject_HEAD + PyObject *gi_weakreflist; + PyObject *gi_name; + PyObject *gi_qualname; + _PyErr_StackItem gi_exc_state; + PyObject *gi_origin_or_finalizer; + char gi_hooks_inited; + char gi_closed; + char gi_running_async; + int8_t gi_frame_state; + PyObject *gi_iframe[1]; +} PyGenObject; +*/ + +static Frame *get_frame(PyGenObject *gen_like) { + Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); + assert(frame); + return frame; +} + +static PyCodeObject *get_frame_code(Frame *frame) { + PyCodeObject *code = frame->f_code; + assert(code); + return code; +} + +static int get_frame_lasti(Frame *frame) { + // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 + PyCodeObject *code = get_frame_code(frame); + assert(frame->prev_instr); + return (int)((intptr_t)frame->prev_instr - (intptr_t)_PyCode_CODE(code)); +} + +static void set_frame_lasti(Frame *frame, int lasti) { + // https://github.com/python/cpython/blob/3.12/Include/internal/pycore_frame.h#L77 + PyCodeObject *code = get_frame_code(frame); + frame->prev_instr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); +} + +static int get_frame_state(PyGenObject *gen_like) { + return gen_like->gi_frame_state; +} + +static void set_frame_state(PyGenObject *gen_like, int fs) { + gen_like->gi_frame_state = (int8_t)fs; +} + +static int valid_frame_state(int fs) { + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; +} + +static int get_frame_stacktop_limit(Frame *frame) { + PyCodeObject *code = get_frame_code(frame); + return code->co_stacksize + code->co_nlocalsplus; +} + +static int get_frame_stacktop(Frame *frame) { + int stacktop = frame->stacktop; + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + return stacktop; +} + +static void set_frame_stacktop(Frame *frame, int stacktop) { + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + frame->stacktop = stacktop; +} + +static PyObject **get_frame_localsplus(Frame *frame) { + PyObject **localsplus = frame->localsplus; + assert(localsplus); + return localsplus; +} + +static int get_frame_iblock_limit(Frame *frame) { + return 1; // not applicable >= 3.11 +} + +static int get_frame_iblock(Frame *frame) { + return 0; // not applicable >= 3.11 +} + +static void set_frame_iblock(Frame *frame, int iblock) { + assert(!iblock); // not applicable >= 3.11 +} + +static PyTryBlock *get_frame_blockstack(Frame *frame) { + return NULL; // not applicable >= 3.11 +} diff --git a/src/dispatch/experimental/durable/frame313.h b/src/dispatch/experimental/durable/frame313.h new file mode 100644 index 00000000..1126dfd3 --- /dev/null +++ b/src/dispatch/experimental/durable/frame313.h @@ -0,0 +1,148 @@ +// This is a redefinition of the private/opaque frame object. +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L57 +// +// In Python 3.10 and prior, `struct _frame` is both the PyFrameObject and +// PyInterpreterFrame. From Python 3.11 onwards, the two were split with the +// PyFrameObject (struct _frame) pointing to struct _PyInterpreterFrame. +struct Frame { + PyObject *f_executable; + struct Frame *previous; // struct _PyInterpreterFrame + PyObject *f_funcobj; + PyObject *f_globals; + PyObject *f_builtins; + PyObject *f_locals; + PyFrameObject *frame_obj; + _Py_CODEUNIT *instr_ptr; + int stacktop; + uint16_t return_offset; + char owner; + PyObject *localsplus[1]; +}; + +// This is a redefinition of private frame state constants. +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L38 +typedef enum _framestate { + FRAME_CREATED = -3, + FRAME_SUSPENDED = -2, + FRAME_SUSPENDED_YIELD_FROM = -1, + FRAME_EXECUTING = 0, + FRAME_COMPLETED = 1, + FRAME_CLEARED = 4 +} FrameState; + +/* +// This is the definition of PyFrameObject (aka. struct _frame) for reference +// to developers working on the extension. +// +// https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L20 +typedef struct { + PyObject_HEAD + PyFrameObject *f_back; + struct _PyInterpreterFrame *f_frame; + PyObject *f_trace; + int f_lineno; + char f_trace_lines; + char f_trace_opcodes; + char f_fast_as_locals; + PyObject *_f_frame_data[1]; +} PyFrameObject; +*/ + +/* +// This is the definition of PyGenObject for reference to developers +// working on the extension. +// +// Note that PyCoroObject and PyAsyncGenObject have the same layout as +// PyGenObject, however the struct fields have a cr_ and ag_ prefix +// (respectively) rather than a gi_ prefix. +// +// https://github.com/python/cpython/blob/v3.13.0a5/Include/cpython/genobject.h#L14 +typedef struct { + PyObject_HEAD + PyObject *gi_weakreflist; + PyObject *gi_name; + PyObject *gi_qualname; + _PyErr_StackItem gi_exc_state; + PyObject *gi_origin_or_finalizer; + char gi_hooks_inited; + char gi_closed; + char gi_running_async; + int8_t gi_frame_state; + PyObject *gi_iframe[1]; +} PyGenObject; +*/ + +static Frame *get_frame(PyGenObject *gen_like) { + Frame *frame = (Frame *)(struct _PyInterpreterFrame *)(gen_like->gi_iframe); + assert(frame); + return frame; +} + +static PyCodeObject *get_frame_code(Frame *frame) { + PyCodeObject *code = (PyCodeObject *)frame->f_executable; + assert(code); + return code; +} + +static int get_frame_lasti(Frame *frame) { + // https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L73 + PyCodeObject *code = get_frame_code(frame); + assert(frame->instr_ptr); + return (int)((intptr_t)frame->instr_ptr - (intptr_t)_PyCode_CODE(code)); +} + +static void set_frame_lasti(Frame *frame, int lasti) { + // https://github.com/python/cpython/blob/v3.13.0a5/Include/internal/pycore_frame.h#L73 + PyCodeObject *code = get_frame_code(frame); + frame->instr_ptr = (_Py_CODEUNIT *)((intptr_t)_PyCode_CODE(code) + (intptr_t)lasti); +} + +static int get_frame_state(PyGenObject *gen_like) { + return gen_like->gi_frame_state; +} + +static void set_frame_state(PyGenObject *gen_like, int fs) { + gen_like->gi_frame_state = (int8_t)fs; +} + +static int valid_frame_state(int fs) { + return fs == FRAME_CREATED || fs == FRAME_SUSPENDED || fs == FRAME_SUSPENDED_YIELD_FROM || fs == FRAME_EXECUTING || fs == FRAME_COMPLETED || fs == FRAME_CLEARED; +} + +static int get_frame_stacktop_limit(Frame *frame) { + PyCodeObject *code = get_frame_code(frame); + return code->co_stacksize + code->co_nlocalsplus; +} + +static int get_frame_stacktop(Frame *frame) { + int stacktop = frame->stacktop; + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + return stacktop; +} + +static void set_frame_stacktop(Frame *frame, int stacktop) { + assert(stacktop >= 0 && stacktop < get_frame_stacktop_limit(frame)); + frame->stacktop = stacktop; +} + +static PyObject **get_frame_localsplus(Frame *frame) { + PyObject **localsplus = frame->localsplus; + assert(localsplus); + return localsplus; +} + +static int get_frame_iblock_limit(Frame *frame) { + return 1; // not applicable >= 3.11 +} + +static int get_frame_iblock(Frame *frame) { + return 0; // not applicable >= 3.11 +} + +static void set_frame_iblock(Frame *frame, int iblock) { + assert(!iblock); // not applicable >= 3.11 +} + +static PyTryBlock *get_frame_blockstack(Frame *frame) { + return NULL; // not applicable >= 3.11 +}