bpo-43950: Implement fine grained error locations for interactive mode

Include/cpython/code.h

@@ -93,6 +93,8 @@ typedef uint16_t _Py_CODEUNIT;
        Type is a void* to keep the format private in codeobject.c to force     \
        people to go through the proper APIs. */                                \
     void *co_extra;                                                            \
+    PyObject *co_source;          /* unicode (source code of the code, used    \
+                                     for tracebacks) */                        \
     char co_code_adaptive[(SIZE)];                                             \
 }
 

Include/internal/pycore_compile.h

@@ -14,6 +14,7 @@ struct _mod;     // Type defined in pycore_ast.h
 // Export the symbol for test_peg_generator (built as a library)
 PyAPI_FUNC(PyCodeObject*) _PyAST_Compile(
     struct _mod *mod,
+    PyObject *src,
     PyObject *filename,
     PyCompilerFlags *flags,
     int optimize,

Include/internal/pycore_parser.h

@@ -24,6 +24,17 @@ extern struct _mod* _PyParser_ASTFromFile(
     PyCompilerFlags *flags,
     int *errcode,
     PyArena *arena);
+extern struct _mod* _PyParser_InteractiveASTFromFile(
+    FILE *fp,
+    PyObject *filename_ob,
+    const char *enc,
+    int mode,
+    const char *ps1,
+    const char *ps2,
+    PyCompilerFlags *flags,
+    int *errcode,
+    PyObject **interactive_src,
+    PyArena *arena);
 
 #ifdef __cplusplus
 }

Lib/test/test_repl.py

@@ -145,6 +145,17 @@ def f():
         ]
         self.assertEqual(traceback_lines, expected_lines)
 
+    def test_interactive_traceback_reporting(self):
+        output = run_on_interactive_mode("1 / 0 / 3 / 4")
+        traceback_lines = output.splitlines()[-6:-1]
+        expected_lines = [
+            "Traceback (most recent call last):",
+            "  File \"<stdin>\", line 1, in <module>",
+            "    1 / 0 / 3 / 4",
+            "    ~~^~~",
+            "ZeroDivisionError: division by zero",
+        ]
+        self.assertEqual(traceback_lines, expected_lines)
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core and Builtins/2022-05-16-00-00-53.gh-issue-88116.Q7lnyq.rst

@@ -0,0 +1 @@
+Implement fine grained error tracebacks for interactive mode. Patch by Jeremiah Vivian.

Objects/codeobject.c

@@ -335,6 +335,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
     co->co_weakreflist = NULL;
     co->co_extra = NULL;
     co->_co_code = NULL;
+    co->co_source = NULL;
 
     co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
     memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code),
@@ -1537,7 +1538,9 @@ code_dealloc(PyCodeObject *co)
     Py_XDECREF(co->co_qualname);
     Py_XDECREF(co->co_linetable);
     Py_XDECREF(co->co_exceptiontable);
+    Py_XDECREF(co->co_source);
     Py_XDECREF(co->_co_code);
+    Py_XDECREF(co->co_source);
     if (co->co_weakreflist != NULL) {
         PyObject_ClearWeakRefs((PyObject*)co);
     }
@@ -1687,6 +1690,7 @@ static PyMemberDef code_memberlist[] = {
     {"co_firstlineno",     T_INT,    OFF(co_firstlineno),     READONLY},
     {"co_linetable",       T_OBJECT, OFF(co_linetable),       READONLY},
     {"co_exceptiontable",  T_OBJECT, OFF(co_exceptiontable),  READONLY},
+    {"co_source",          T_OBJECT, OFF(co_source),          READONLY},
     {NULL}      /* Sentinel */
 };
 

Parser/peg_api.c

@@ -24,5 +24,18 @@ _PyParser_ASTFromFile(FILE *fp, PyObject *filename_ob, const char *enc,
         return NULL;
     }
     return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2,
-                                        flags, errcode, arena);
+                                                 flags, errcode, NULL, arena);
+}
+
+mod_ty
+_PyParser_InteractiveASTFromFile(FILE *fp, PyObject *filename_ob, const char *enc,
+                                 int mode, const char *ps1, const char* ps2,
+                                 PyCompilerFlags *flags, int *errcode,
+                                 PyObject **interactive_src, PyArena *arena)
+{
+    if (PySys_Audit("compile", "OO", Py_None, filename_ob) < 0) {
+        return NULL;
+    }
+    return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2,
+                                                 flags, errcode, interactive_src, arena);
 }

Parser/pegen.c

@@ -859,7 +859,8 @@ _PyPegen_run_parser(Parser *p)
 mod_ty
 _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
                              const char *enc, const char *ps1, const char *ps2,
-                             PyCompilerFlags *flags, int *errcode, PyArena *arena)
+                             PyCompilerFlags *flags, int *errcode,
+                             PyObject **interactive_src, PyArena *arena)
 {
     struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
     if (tok == NULL) {
@@ -890,6 +891,17 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
     result = _PyPegen_run_parser(p);
     _PyPegen_Parser_Free(p);
 
+    if (tok->fp_interactive && tok->interactive_src_start &&
+            result && interactive_src != NULL)
+    {
+        *interactive_src = PyUnicode_FromString(tok->interactive_src_start);
+        if (!interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) {
+            Py_XDECREF(interactive_src);
+            result = NULL;
+            goto error;
+        }
+    }
+
 error:
     _PyTokenizer_Free(tok);
     return result;

Parser/pegen.h

@@ -333,7 +333,8 @@ void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehensi
 Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
 void _PyPegen_Parser_Free(Parser *);
 mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
-                                    const char *, const char *, PyCompilerFlags *, int *, PyArena *);
+                                             const char *, const char *, PyCompilerFlags *,
+                                             int *, PyObject **, PyArena *);
 void *_PyPegen_run_parser(Parser *);
 mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
 asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);

Python/bltinmodule.c

@@ -816,7 +816,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
                 _PyArena_Free(arena);
                 goto error;
             }
-            result = (PyObject*)_PyAST_Compile(mod, filename,
+            result = (PyObject*)_PyAST_Compile(mod, source, filename,
                                                &cf, optimize, arena);
             _PyArena_Free(arena);
         }

Python/compile.c

@@ -400,6 +400,7 @@ struct compiler {
     struct compiler_unit *u; /* compiler state for current block */
     PyObject *c_stack;           /* Python list holding compiler_unit ptrs */
     PyArena *c_arena;            /* pointer to memory allocation arena */
+    PyObject *c_source;          /* source code of a file/interactive line */
 };
 
 typedef struct {
@@ -577,8 +578,8 @@ compiler_init(struct compiler *c)
 }
 
 PyCodeObject *
-_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
-               int optimize, PyArena *arena)
+_PyAST_Compile(mod_ty mod, PyObject *source, PyObject *filename,
+               PyCompilerFlags *flags, int optimize, PyArena *arena)
 {
     struct compiler c;
     PyCodeObject *co = NULL;
@@ -601,6 +602,8 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
     c.c_flags = flags;
     c.c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize;
     c.c_nestlevel = 0;
+    Py_XINCREF(source);
+    c.c_source = source;
 
     _PyASTOptimizeState state;
     state.optimize = c.c_optimize;
@@ -635,6 +638,7 @@ compiler_free(struct compiler *c)
     Py_XDECREF(c->c_filename);
     Py_DECREF(c->c_const_cache);
     Py_DECREF(c->c_stack);
+    Py_XDECREF(c->c_source);
 }
 
 static PyObject *
@@ -8746,6 +8750,10 @@ assemble(struct compiler *c, int addNone)
     }
 
     co = makecode(c, &a, consts, maxdepth, nlocalsplus, code_flags);
+    if (co) {
+        Py_XINCREF(c->c_source);
+        co->co_source = c->c_source;
+    }
  error:
     Py_XDECREF(consts);
     assemble_free(&a);

Python/pythonrun.c

@@ -44,7 +44,7 @@ extern "C" {
 /* Forward */
 static void flush_io(void);
 static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *,
-                          PyCompilerFlags *, PyArena *);
+                          PyCompilerFlags *, PyObject *, PyArena *);
 static PyObject *run_pyc_file(FILE *, PyObject *, PyObject *,
                               PyCompilerFlags *);
 static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *);
@@ -237,8 +237,9 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
         return -1;
     }
 
-    mod = _PyParser_ASTFromFile(fp, filename, enc, Py_single_input,
-                                ps1, ps2, flags, &errcode, arena);
+    PyObject *interactive_src = NULL;
+    mod = _PyParser_InteractiveASTFromFile(fp, filename, enc, Py_single_input,
+                                           ps1, ps2, flags, &errcode, &interactive_src, arena);
 
     Py_XDECREF(v);
     Py_XDECREF(w);
@@ -257,7 +258,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
         return -1;
     }
     d = PyModule_GetDict(m);
-    v = run_mod(mod, filename, d, d, flags, arena);
+    v = run_mod(mod, filename, d, d, flags, interactive_src, arena);
     _PyArena_Free(arena);
     if (v == NULL) {
         return -1;
@@ -1600,8 +1601,14 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
     mod = _PyParser_ASTFromString(
             str, &_Py_STR(anon_string), start, flags, arena);
 
-    if (mod != NULL)
-        ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, arena);
+    if (mod != NULL) {
+        PyObject *string = PyUnicode_FromString(str);
+        if (string == NULL) {
+            _PyArena_Free(arena);
+            return NULL;
+        }
+        ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, string, arena);
+    }
     _PyArena_Free(arena);
     return ret;
 }
@@ -1626,11 +1633,12 @@ pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals,
 
     PyObject *ret;
     if (mod != NULL) {
-        ret = run_mod(mod, filename, globals, locals, flags, arena);
+        ret = run_mod(mod, filename, globals, locals, flags, NULL, arena);
     }
     else {
         ret = NULL;
     }
+
     _PyArena_Free(arena);
 
     return ret;
@@ -1719,10 +1727,10 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py
 
 static PyObject *
 run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals,
-            PyCompilerFlags *flags, PyArena *arena)
+            PyCompilerFlags *flags, PyObject *src, PyArena *arena)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    PyCodeObject *co = _PyAST_Compile(mod, filename, flags, -1, arena);
+    PyCodeObject *co = _PyAST_Compile(mod, src, filename, flags, -1, arena);
     if (co == NULL)
         return NULL;
 
@@ -1799,7 +1807,12 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start,
         _PyArena_Free(arena);
         return result;
     }
-    co = _PyAST_Compile(mod, filename, flags, optimize, arena);
+    PyObject *string = PyUnicode_FromString(str);
+    if (string == NULL) {
+        _PyArena_Free(arena);
+        return NULL;
+    }
+    co = _PyAST_Compile(mod, string, filename, flags, optimize, arena);
     _PyArena_Free(arena);
     return (PyObject *)co;
 }