Skip to content

Commit 3d2919c

Browse files
isidenticalisidentical
committed
bpo-43950: implement on-the-fly source tracking for interactive mode
1 parent 3b5b99d commit 3d2919c

File tree

8 files changed

+202
-50
lines changed

8 files changed

+202
-50
lines changed

‎Include/internal/pycore_parser.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,17 @@ extern struct _mod* _PyParser_ASTFromFile(
2424
PyCompilerFlags *flags,
2525
int *errcode,
2626
PyArena *arena);
27+
extern struct _mod* _PyParser_InteractiveASTFromFile(
28+
FILE *fp,
29+
PyObject *filename_ob,
30+
const char *enc,
31+
int mode,
32+
const char *ps1,
33+
const char *ps2,
34+
PyCompilerFlags *flags,
35+
int *errcode,
36+
PyObject **interactive_src,
37+
PyArena *arena);
2738

2839
#ifdef __cplusplus
2940
}

‎Lib/test/test_repl.py

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,72 @@ def test_close_stdin(self):
107107
self.assertEqual(process.returncode, 0)
108108
self.assertIn('before close', output)
109109

110+
def test_interactive_source_tracking(self):
111+
import ast
112+
113+
p = spawn_repl()
114+
p.stdin.write(dedent("""\
115+
import sys
116+
117+
sources = []
118+
def source_tracker(frame, event, arg):
119+
global sources
120+
if source := frame.f_globals.get("__source__"):
121+
sources.append(source)
122+
123+
sys.settrace(source_tracker)
124+
125+
# basic statement
126+
a = 1
127+
128+
# multiline statement
129+
if a:
130+
pass
131+
else:
132+
...
133+
134+
# multiline expression
135+
maybe = [
136+
1,
137+
2,
138+
[
139+
3,
140+
4
141+
]
142+
][2][
143+
0
144+
]
145+
146+
# basic expression
147+
sys.settrace(None)
148+
149+
print("\\n" + repr(sources) + "\\n")
150+
"""))
151+
output = kill_python(p)
152+
result_line = output.splitlines()[-3]
153+
tracing_records = ast.literal_eval(result_line)
154+
self.assertIn("a = 1\n", tracing_records)
155+
self.assertIn("sys.settrace(None)\n", tracing_records)
156+
self.assertIn("if a:\n pass\nelse:\n ...\n\n", tracing_records)
157+
self.assertIn("maybe = [\n 1,\n 2,\n [\n 3,\n"
158+
" 4\n ]\n][2][\n 0\n]\n", tracing_records)
159+
160+
def test_interactive_traceback_reporting(self):
161+
user_input = "1 / 0 / 3 / 4"
162+
p = spawn_repl()
163+
p.stdin.write(user_input)
164+
output = kill_python(p)
165+
self.assertEqual(p.returncode, 0)
166+
167+
traceback_lines = output.splitlines()[-6:-1]
168+
expected_lines = [
169+
"Traceback (most recent call last):",
170+
" File \"<stdin>\", line 1, in <module>",
171+
" 1 / 0 / 3 / 4",
172+
" ~~^~~",
173+
"ZeroDivisionError: division by zero",
174+
]
175+
self.assertEqual(traceback_lines, expected_lines)
110176

111177
if __name__ == "__main__":
112178
unittest.main()

‎Parser/peg_api.c

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,19 @@ _PyParser_ASTFromFile(FILE *fp, PyObject *filename_ob, const char *enc,
2424
return NULL;
2525
}
2626
return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2,
27-
flags, errcode, arena);
27+
flags, errcode, NULL, arena);
28+
}
29+
30+
31+
mod_ty
32+
_PyParser_InteractiveASTFromFile(FILE *fp, PyObject *filename_ob, const char *enc,
33+
int mode, const char *ps1, const char* ps2,
34+
PyCompilerFlags *flags, int *errcode,
35+
PyObject **interactive_src, PyArena *arena)
36+
{
37+
if (PySys_Audit("compile", "OO", Py_None, filename_ob) < 0) {
38+
return NULL;
39+
}
40+
return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2,
41+
flags, errcode, interactive_src, arena);
2842
}

‎Parser/pegen.c

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1373,7 +1373,8 @@ _PyPegen_run_parser(Parser *p)
13731373
mod_ty
13741374
_PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
13751375
const char *enc, const char *ps1, const char *ps2,
1376-
PyCompilerFlags *flags, int *errcode, PyArena *arena)
1376+
PyCompilerFlags *flags, int *errcode,
1377+
PyObject **interactive_src, PyArena *arena)
13771378
{
13781379
struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
13791380
if (tok == NULL) {
@@ -1404,6 +1405,15 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
14041405
result = _PyPegen_run_parser(p);
14051406
_PyPegen_Parser_Free(p);
14061407

1408+
if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) {
1409+
*interactive_src = PyUnicode_FromString(tok->interactive_src_start);
1410+
if (!interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) {
1411+
Py_XDECREF(interactive_src);
1412+
result = NULL;
1413+
goto error;
1414+
}
1415+
}
1416+
14071417
error:
14081418
PyTokenizer_Free(tok);
14091419
return result;

‎Parser/pegen.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,8 @@ PyObject *_PyPegen_new_identifier(Parser *, const char *);
250250
Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
251251
void _PyPegen_Parser_Free(Parser *);
252252
mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
253-
const char *, const char *, PyCompilerFlags *, int *, PyArena *);
253+
const char *, const char *, PyCompilerFlags *, int *, PyObject **,
254+
PyArena *);
254255
void *_PyPegen_run_parser(Parser *);
255256
mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
256257
asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);

‎Python/pythonrun.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,9 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
253253
return -1;
254254
}
255255

256-
mod = _PyParser_ASTFromFile(fp, filename, enc, Py_single_input,
257-
ps1, ps2, flags, &errcode, arena);
256+
PyObject *interactive_src = NULL;
257+
mod = _PyParser_InteractiveASTFromFile(fp, filename, enc, Py_single_input,
258+
ps1, ps2, flags, &errcode, &interactive_src, arena);
258259

259260
Py_XDECREF(v);
260261
Py_XDECREF(w);
@@ -273,6 +274,12 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
273274
return -1;
274275
}
275276
d = PyModule_GetDict(m);
277+
if (interactive_src) {
278+
if (PyDict_SetItemString(d, "__source__", interactive_src) < 0) {
279+
_PyArena_Free(arena);
280+
return -1;
281+
}
282+
}
276283
v = run_mod(mod, filename, d, d, flags, arena);
277284
_PyArena_Free(arena);
278285
if (v == NULL) {

‎Python/traceback.c

Lines changed: 87 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,62 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject *
375375
return result;
376376
}
377377

378+
int
379+
_Py_DisplayLine(PyObject *f, PyObject *line_obj, PyObject **line,
380+
int indent, int *truncation)
381+
{
382+
if (line) {
383+
Py_INCREF(line_obj);
384+
*line = line_obj;
385+
}
386+
387+
int i, kind, err = -1;
388+
const void *data;
389+
char buf[MAXPATHLEN+1];
390+
391+
/* remove the indentation of the line */
392+
kind = PyUnicode_KIND(line_obj);
393+
data = PyUnicode_DATA(line_obj);
394+
for (i=0; i < PyUnicode_GET_LENGTH(line_obj); i++) {
395+
Py_UCS4 ch = PyUnicode_READ(kind, data, i);
396+
if (ch != ' ' && ch != '\t' && ch != '\014')
397+
break;
398+
}
399+
if (i) {
400+
PyObject *truncated;
401+
truncated = PyUnicode_Substring(line_obj, i, PyUnicode_GET_LENGTH(line_obj));
402+
if (truncated) {
403+
Py_DECREF(line_obj);
404+
line_obj = truncated;
405+
} else {
406+
PyErr_Clear();
407+
}
408+
}
409+
410+
if (truncation != NULL) {
411+
*truncation = i - indent;
412+
}
413+
414+
/* Write some spaces before the line */
415+
strcpy(buf, " ");
416+
assert (strlen(buf) == 10);
417+
while (indent > 0) {
418+
if (indent < 10)
419+
buf[indent] = '\0';
420+
err = PyFile_WriteString(buf, f);
421+
if (err != 0)
422+
break;
423+
indent -= 10;
424+
}
425+
426+
/* finally display the line */
427+
if (err == 0)
428+
err = PyFile_WriteObject(line_obj, f, Py_PRINT_RAW);
429+
Py_DECREF(line_obj);
430+
if (err == 0)
431+
err = PyFile_WriteString("\n", f);
432+
return err;
433+
}
378434
int
379435
_Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, int *truncation, PyObject **line)
380436
{
@@ -389,8 +445,6 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, i
389445
PyObject *lineobj = NULL;
390446
PyObject *res;
391447
char buf[MAXPATHLEN+1];
392-
int kind;
393-
const void *data;
394448

395449
/* open the file */
396450
if (filename == NULL)
@@ -467,53 +521,34 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, i
467521
return err;
468522
}
469523

470-
if (line) {
471-
Py_INCREF(lineobj);
472-
*line = lineobj;
473-
}
524+
return _Py_DisplayLine(f, lineobj, line, indent, truncation);
525+
}
474526

475-
/* remove the indentation of the line */
476-
kind = PyUnicode_KIND(lineobj);
477-
data = PyUnicode_DATA(lineobj);
478-
for (i=0; i < PyUnicode_GET_LENGTH(lineobj); i++) {
479-
Py_UCS4 ch = PyUnicode_READ(kind, data, i);
480-
if (ch != ' ' && ch != '\t' && ch != '\014')
481-
break;
482-
}
483-
if (i) {
484-
PyObject *truncated;
485-
truncated = PyUnicode_Substring(lineobj, i, PyUnicode_GET_LENGTH(lineobj));
486-
if (truncated) {
487-
Py_DECREF(lineobj);
488-
lineobj = truncated;
489-
} else {
490-
PyErr_Clear();
491-
}
527+
int
528+
_Py_DisplayInteractiveSourceLine(PyObject *f, PyFrameObject *frame, int lineno, int indent,
529+
int *truncation, PyObject **line)
530+
{
531+
PyObject *globals = _PyFrame_GetGlobals(frame);
532+
PyObject *source = PyDict_GetItemString(globals, "__source__");
533+
if (!source) {
534+
return -1;
492535
}
493536

494-
if (truncation != NULL) {
495-
*truncation = i - indent;
537+
PyObject *lines = PyUnicode_Splitlines(source, 0);
538+
if (!lines || PyList_GET_SIZE(lines) < lineno) {
539+
Py_XDECREF(lines);
540+
return -1;
496541
}
497542

498-
/* Write some spaces before the line */
499-
strcpy(buf, " ");
500-
assert (strlen(buf) == 10);
501-
while (indent > 0) {
502-
if (indent < 10)
503-
buf[indent] = '\0';
504-
err = PyFile_WriteString(buf, f);
505-
if (err != 0)
506-
break;
507-
indent -= 10;
543+
PyObject *lineobj = PyList_GetItem(lines, lineno - 1);
544+
if (!lineobj) {
545+
Py_DECREF(lines);
546+
return -1;
508547
}
548+
Py_INCREF(lineobj);
549+
Py_DECREF(lines);
509550

510-
/* finally display the line */
511-
if (err == 0)
512-
err = PyFile_WriteObject(lineobj, f, Py_PRINT_RAW);
513-
Py_DECREF(lineobj);
514-
if (err == 0)
515-
err = PyFile_WriteString("\n", f);
516-
return err;
551+
return _Py_DisplayLine(f, lineobj, line, indent, truncation);
517552
}
518553

519554
/* AST based Traceback Specialization
@@ -702,8 +737,16 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen
702737
int truncation = _TRACEBACK_SOURCE_LINE_INDENT;
703738
PyObject* source_line = NULL;
704739

705-
if (_Py_DisplaySourceLine(f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT,
706-
&truncation, &source_line) != 0) {
740+
if (PyUnicode_CompareWithASCIIString(filename, "<stdin>") == 0) {
741+
err = _Py_DisplayInteractiveSourceLine(f, frame, lineno, _TRACEBACK_SOURCE_LINE_INDENT,
742+
&truncation, &source_line);
743+
}
744+
else {
745+
err = _Py_DisplaySourceLine(f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT,
746+
&truncation, &source_line);
747+
}
748+
749+
if (err != 0) {
707750
/* ignore errors since we can't report them, can we? */
708751
err = ignore_source_errors();
709752
goto done;

‎Tools/peg_generator/peg_extension/peg_extension.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ parse_file(PyObject *self, PyObject *args, PyObject *kwds)
5353
PyCompilerFlags flags = _PyCompilerFlags_INIT;
5454
mod_ty res = _PyPegen_run_parser_from_file_pointer(
5555
fp, Py_file_input, filename_ob,
56-
NULL, NULL, NULL, &flags, NULL, arena);
56+
NULL, NULL, NULL, &flags, NULL, NULL, arena);
5757
fclose(fp);
5858
if (res == NULL) {
5959
goto error;

0 commit comments

Comments
 (0)