Moar tests and fix location error

Author: pablogsal

Lib/test/test_tokenize.py

@@ -421,12 +421,6 @@ def test_string(self):
     FSTRING_MIDDLE '}'           (1, 9) (1, 10)
     FSTRING_END '"'           (1, 11) (1, 12)
     """)
-    # TODO: I don't think is is correct now (ERRORTOKEN)
-    #     self.check_tokenize('f"{1+1"', """\
-    # FSTRING_START 'f"'          (1, 0) (1, 2)
-    # ERRORTOKEN '{'           (1, 2) (1, 3)
-    # FSTRING_END '"'           (1, 3) (1, 4)
-    # """)
         self.check_tokenize('f"""{f\'\'\'{f\'{f"{1+1}"}\'}\'\'\'}"""', """\
     FSTRING_START 'f\"""'        (1, 0) (1, 4)
     OP         '{'           (1, 4) (1, 5)
@@ -447,6 +441,26 @@ def test_string(self):
     FSTRING_END "'''"         (1, 24) (1, 27)
     OP         '}'           (1, 27) (1, 28)
     FSTRING_END '\"""'         (1, 28) (1, 31)
+    """)
+        self.check_tokenize('f"""     x\nstr(data, encoding={invalid!r})\n"""', """\
+    FSTRING_START 'f\"""'        (1, 0) (1, 4)
+    FSTRING_MIDDLE '     x\\nstr(data, encoding=' (1, 4) (2, 19)
+    OP         '{'           (2, 19) (2, 20)
+    NAME       'invalid'     (2, 20) (2, 27)
+    OP         '!'           (2, 27) (2, 28)
+    NAME       'r'           (2, 28) (2, 29)
+    OP         '}'           (2, 29) (2, 30)
+    FSTRING_MIDDLE ')\\n'         (2, 30) (3, 0)
+    FSTRING_END '\"""'         (3, 0) (3, 3)
+    """)
+        self.check_tokenize('f"""123456789\nsomething{None}bad"""', """\
+    FSTRING_START 'f\"""'        (1, 0) (1, 4)
+    FSTRING_MIDDLE '123456789\\nsomething' (1, 4) (2, 9)
+    OP         '{'           (2, 9) (2, 10)
+    NAME       'None'        (2, 10) (2, 14)
+    OP         '}'           (2, 14) (2, 15)
+    FSTRING_MIDDLE 'bad'         (2, 15) (2, 18)
+    FSTRING_END '\"""'         (2, 18) (2, 21)
     """)
         self.check_tokenize('f"""abc"""', """\
     FSTRING_START 'f\"""'        (1, 0) (1, 4)

Parser/tokenizer.c

@@ -1126,7 +1126,7 @@ tok_underflow_interactive(struct tok_state *tok) {
 
 static int
 tok_underflow_file(struct tok_state *tok) {
-    if (tok->start == NULL) {
+    if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
         tok->cur = tok->inp = tok->buf;
     }
     if (tok->decoding_state == STATE_INIT) {