gh-112343: pdb: Use tokenize to replace convenience variables

Lib/pdb.py

@@ -76,6 +76,7 @@
 import dis
 import code
 import glob
+import token
 import codeop
 import pprint
 import signal
@@ -590,6 +591,45 @@ def default(self, line):
         except:
             self._error_exc()
 
+    def _replace_convenience_variables(self, line):
+        """Replace the convenience variables in line"""
+
+        if "$" not in line:
+            return line
+
+        last_token_is_dollar = False
+        dollar_start = None
+        dollar_end = None
+        replace_variables = []
+        try:
+            for t in tokenize.generate_tokens(io.StringIO(line).readline):
+                token_type, token_string, start, end, _ = t
+                if token_type == token.OP and token_string == '$':
+                    last_token_is_dollar = True
+                    dollar_end = end
+                    dollar_start = start
+                else:
+                    if (last_token_is_dollar and
+                        token_type == token.NAME and
+                        start == dollar_end):
+                        # line is a one line command so we only care about column
+                        replace_variables.append((dollar_start[1], end[1], token_string))
+                    last_token_is_dollar = False
+        except tokenize.TokenError:
+            return line
+
+        if not replace_variables:
+            return line
+
+        last_end = 0
+        new_line = ''
+        for start, end, name in replace_variables:
+            new_line += line[last_end:start] + f'__pdb_convenience_variables["{name}"]'
+            last_end = end
+        new_line += line[last_end:]
+
+        return new_line
+
     def precmd(self, line):
         """Handle alias expansion and ';;' separator."""
         if not line.strip():
@@ -624,7 +664,7 @@ def precmd(self, line):
                 line = line[:marker].rstrip()
 
         # Replace all the convenience variables
-        line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line)
+        line = self._replace_convenience_variables(line)
 
         return line
 

Lib/test/test_pdb.py

@@ -794,9 +794,12 @@ def test_convenience_variables():
 
     >>> with PdbTestInput([  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
     ...     '$_frame.f_lineno', # Check frame convenience variable
+    ...     '$ _frame',         # This should be a syntax error
     ...     '$a = 10',          # Set a convenience variable
     ...     '$a',               # Print its value
+    ...     'p "$a"',           # Print the string $a
     ...     'p $a + 2',         # Do some calculation
+    ...     'p f"$a = {$a}"',   # Make sure $ in string is not converted and f-string works
     ...     'u',                # Switch frame
     ...     '$_frame.f_lineno', # Make sure the frame changed
     ...     '$a',               # Make sure the value persists
@@ -816,11 +819,17 @@ def test_convenience_variables():
     -> try:
     (Pdb) $_frame.f_lineno
     3
+    (Pdb) $ _frame
+    *** SyntaxError: invalid syntax
     (Pdb) $a = 10
     (Pdb) $a
     10
+    (Pdb) p "$a"
+    '$a'
     (Pdb) p $a + 2
     12
+    (Pdb) p f"$a = {$a}"
+    '$a = 10'
     (Pdb) u
     > <doctest test.test_pdb.test_convenience_variables[1]>(2)test_function()
     -> util_function()

Misc/NEWS.d/next/Library/2023-11-24-19-08-50.gh-issue-112343.RarGFC.rst

@@ -0,0 +1 @@
+Improve handling of pdb convenience variables to avoid replacing string contents.