Parser: recover on unfinished identifiers

src/Compiler/FSComp.txt

@@ -1683,4 +1683,5 @@ featureEscapeBracesInFormattableString,"Escapes curly braces before calling Form
 3559,typrelNeverRefinedAwayFromTop,"A type has been implicitly inferred as 'obj', which may be unintended. Consider adding explicit type annotations. You can disable this warning by using '#nowarn \"3559\"' or '--nowarn:3559'."
 3560,tcCopyAndUpdateRecordChangesAllFields,"This copy-and-update record expression changes all fields of record type '%s'. Consider using the record construction syntax instead."
 3561,chkAutoOpenAttributeInTypeAbbrev,"FSharp.Core.AutoOpenAttribute should not be aliased."
-3562,parsUnexpectedEndOfFileElif,"Unexpected end of input in 'else if' or 'elif' branch of conditional expression. Expected 'elif <expr> then <expr>' or 'else if <expr> then <expr>'."
+3562,parsUnexpectedEndOfFileElif,"Unexpected end of input in 'else if' or 'elif' branch of conditional expression. Expected 'elif <expr> then <expr>' or 'else if <expr> then <expr>'."
+3563,lexInvalidIdentifier,"This is not a valid identifier"

src/Compiler/lex.fsl

@@ -701,9 +701,21 @@ rule token args skip = parse
        if not skip then WHITESPACE (LexCont.Token(args.ifdefStack, args.stringNest))
        else token args skip lexbuf }
 
- | '`' '`' ([^'`' '\n' '\r' '\t'] | '`' [^'`''\n' '\r' '\t']) + '`' '`'
+ | '`' '`' ([^'`' '\n' '\r' '\t'] | '`' [^'`''\n' '\r' '\t'])+ '`' '`'
      { Keywords.IdentifierToken args lexbuf (lexemeTrimBoth lexbuf 2 2) }
 
+ | '`' '`' (([^'`' '\n' '\r' '\t'] | ('`' [^'`' '\n' '\r' '\t']))+)'`'
+     { errorR(Error(FSComp.SR.lexInvalidIdentifier(), lexbuf.LexemeRange))
+       Keywords.IdentifierToken args lexbuf (lexemeTrimBoth lexbuf 2 1) }
+
+ | '`' '`' (([^'`' '\n' '\r' '\t'] | ('`'[^'`' '\n' '\r' '\t']))+)
+     { errorR(Error(FSComp.SR.lexInvalidIdentifier(), lexbuf.LexemeRange))
+       Keywords.IdentifierToken args lexbuf (lexemeTrimLeft lexbuf 2) }
+
+ | "````" | "```" | "``" | "`"
+     { errorR(Error(FSComp.SR.lexInvalidIdentifier(), lexbuf.LexemeRange))
+       Keywords.IdentifierToken args lexbuf "" }
+
  | ('#' anywhite* | "#line" anywhite+ ) digit+ anywhite* ('@'? "\"" [^'\n''\r''"']+ '"')? anywhite* newline
      {  let pos = lexbuf.EndPos
         if skip then
@@ -898,8 +910,6 @@ rule token args skip = parse
 
  | "~" { RESERVED }
 
- | "`" { RESERVED }
-
  | ignored_op_char* '*' '*'                    op_char* { checkExprOp lexbuf; INFIX_STAR_STAR_OP(lexeme lexbuf) }
 
  | ignored_op_char* ('*' | '/'|'%')            op_char* { checkExprOp lexbuf; INFIX_STAR_DIV_MOD_OP(lexeme lexbuf) }

src/Compiler/pars.fsy

@@ -4343,6 +4343,9 @@ atomicExpr:
         let lhsm = rhs2 parseState 1 2
         SynExpr.Typar(typar, lhsm), false }
 
+  | RESERVED
+      { arbExpr ("unfinished identifier", rhs parseState 1), false }
+
   | atomicExpr DOT atomicExprQualification 
       { let arg1, hpa1 = $1 
         $3 arg1 (lhs parseState) (rhs parseState 2), hpa1 }
@@ -5792,7 +5795,10 @@ nameop:
 
 identExpr:
   | ident
-     { SynExpr.Ident($1) }
+     { if $1.idText = "" then
+           SynExpr.FromParseError(SynExpr.Ident($1), $1.idRange)
+       else
+           SynExpr.Ident($1) }
 
   | opName
      { let m = lhs parseState

src/Compiler/xlf/FSComp.txt.cs.xlf

@@ -527,6 +527,11 @@
         <target state="translated">Oblasti IF-FSHARP/IF-CAML už nejsou podporovány.</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">Znak }} musí být v interpolovaném řetězci uvozený (zdvojeným znakem).</target>

src/Compiler/xlf/FSComp.txt.de.xlf

@@ -527,6 +527,11 @@
         <target state="translated">IF-FSHARP-/IF-CAML-Regionen werden nicht mehr unterstützt</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">Ein }}-Zeichen muss in einer interpolierten Zeichenfolge (durch Verdoppeln) mit Escapezeichen versehen werden.</target>

src/Compiler/xlf/FSComp.txt.es.xlf

@@ -527,6 +527,11 @@
         <target state="translated">Ya no se admiten las regiones IF-FSHARP/IF-CAML</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">El carácter "}}" se debe escapar (duplicándose) en las cadenas interpoladas.</target>

src/Compiler/xlf/FSComp.txt.fr.xlf

@@ -527,6 +527,11 @@
         <target state="translated">Les régions IF-FSHARP/IF-CAML ne sont plus prises en charge.</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">Un caractère '}}' doit faire l'objet d'une séquence d'échappement (par doublement) dans une chaîne interpolée.</target>

src/Compiler/xlf/FSComp.txt.it.xlf

@@ -527,6 +527,11 @@
         <target state="translated">Le aree IF-FSHARP/IF-CAML non sono più supportate</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">In una stringa interpolata è necessario specificare il carattere di escape di un carattere '}}' raddoppiandolo.</target>

src/Compiler/xlf/FSComp.txt.ja.xlf

@@ -527,6 +527,11 @@
         <target state="translated">IF-FSHARP/IF-CAML リージョンは現在サポートされていません</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">文字 '}}' は、補間された文字列内で (二重にすることで) エスケープする必要があります。</target>

src/Compiler/xlf/FSComp.txt.ko.xlf

@@ -527,6 +527,11 @@
         <target state="translated">IF-FSHARP/IF-CAML 영역은 더 이상 지원되지 않습니다.</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">'}}' 문자는 보간된 문자열에서 이중으로 사용하여 이스케이프해야 합니다.</target>

src/Compiler/xlf/FSComp.txt.pl.xlf

@@ -527,6 +527,11 @@
         <target state="translated">Regiony IF-FSHARP/IF-CAML nie są już obsługiwane</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">W przypadku znaku „}}” należy zastosować ucieczkę (przez wpisanie dwóch takich znaków) w ciągu interpolowanym.</target>

src/Compiler/xlf/FSComp.txt.pt-BR.xlf

@@ -527,6 +527,11 @@
         <target state="translated">As regiões IF-FSHARP/IF-CAML não são mais suportadas</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">Um caractere ''}}' precisa ser de escape (ao duplicar) em uma cadeia de caracteres interpolada.</target>

src/Compiler/xlf/FSComp.txt.ru.xlf

@@ -527,6 +527,11 @@
         <target state="translated">Регионы IF-FSHARP/IF-CAML больше не поддерживаются</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">Символ "}}" необходимо экранировать (путем дублирования) в интерполированной строке.</target>

src/Compiler/xlf/FSComp.txt.tr.xlf

@@ -527,6 +527,11 @@
         <target state="translated">IF-FSHARP/IF-CAML bölgeleri artık desteklenmiyor</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">Bir '}}' karakteri, düz metin arasına kod eklenmiş bir dizede kaçış dizisi ile (yineleme yapılarak) belirtilir.</target>

src/Compiler/xlf/FSComp.txt.zh-Hans.xlf

@@ -527,6 +527,11 @@
         <target state="translated">不再支持 IF-FSHARP/IF-CAML 区域</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">在内插字符串中，必需对 "}}" 字符进行转义(通过加倍)。</target>

src/Compiler/xlf/FSComp.txt.zh-Hant.xlf

@@ -527,6 +527,11 @@
         <target state="translated">不再支援 IF-FSHARP/IF-CAML 區域</target>
         <note />
       </trans-unit>
+      <trans-unit id="lexInvalidIdentifier">
+        <source>This is not a valid identifier</source>
+        <target state="new">This is not a valid identifier</target>
+        <note />
+      </trans-unit>
       <trans-unit id="lexRBraceInInterpolatedString">
         <source>A '}}' character must be escaped (by doubling) in an interpolated string.</source>
         <target state="translated">在插補字串中，必須將 '}}' 字元逸出 (重複一次)。</target>

tests/FSharp.Compiler.ComponentTests/Conformance/BasicTypeAndModuleDefinitions/ExceptionDefinitions/ExceptionDefinitions.fs

@@ -182,7 +182,7 @@ module ExceptionDefinition =
         |> compile
         |> shouldFail
         |> withDiagnostics [
-            (Error 10, Line 7, Col 11, Line 7, Col 12, "Unexpected reserved keyword in exception definition. Expected identifier or other token.")
+            (Error 3563, Line 7, Col 11, Line 7, Col 17, "This is not a valid identifier")
         ]
 
     // SOURCE=E_DynamicInvocationNotSupported.fsx SCFLAGS=--test:ErrorRanges                   # E_DynamicInvocationNotSupported.fsx

tests/FSharp.Compiler.ComponentTests/Conformance/DeclarationElements/LetBindings/Basic/Basic.fs

@@ -89,10 +89,8 @@ module Basic =
         |> verifyCompile
         |> shouldFail
         |> withDiagnostics [
-            (Error 10, Line 7, Col 13, Line 7, Col 14, "Unexpected reserved keyword in pattern")
-            (Error 583, Line 7, Col 8, Line 7, Col 9, "Unmatched '('")
-            (Error 10, Line 7, Col 25, Line 7, Col 26, "Unexpected reserved keyword in binding")
-            (Error 583, Line 7, Col 19, Line 7, Col 20, "Unmatched '('")
+            (Error 3563, Line 7, Col 13, Line 7, Col 14, "This is not a valid identifier")
+            (Error 3563, Line 7, Col 25, Line 7, Col 26, "This is not a valid identifier")
         ]
 
     // SOURCE=E_InvalidInnerRecursiveBinding.fs SCFLAGS="--test:ErrorRanges"   # E_InvalidInnerRecursiveBinding.fs

tests/FSharp.Compiler.ComponentTests/Conformance/UnionTypes/UnionTypes.fs

@@ -91,7 +91,7 @@ module UnionTypes =
         |> verifyCompile
         |> shouldFail
         |> withDiagnostics [
-            (Error 10, Line 9, Col 12, Line 9, Col 13, "Unexpected reserved keyword in union case. Expected identifier, '(', '(*)' or other token.")
+            (Error 3563, Line 9, Col 12, Line 9, Col 18, "This is not a valid identifier")
         ]
 
     //SOURCE=E_BeginWithUppercaseNoPipe01.fsx SCFLAGS="--test:ErrorRanges"                        # E_BeginWithUppercaseNoPipe01.fsx

tests/service/data/SyntaxTree/Expression/Unfinished escaped ident 01.fs

@@ -0,0 +1,46 @@
+module Module
+
+do
+    ` 
+
+do
+    `` 
+
+do
+    ``` 
+
+do
+    ```` 
+
+do
+    `` `` 
+
+do
+    ` `` ` 
+
+do
+    ` `` `` 
+
+do
+    ```````` 
+
+do
+    ``````` 
+
+do
+    `````` 
+
+do
+    ````` 
+
+do
+    ``` `` 
+
+do
+    ``` ``` 
+
+do
+    ````.P
+
+do
+    1 + `

tests/service/data/SyntaxTree/Expression/Unfinished escaped ident 01.fs.bsl

@@ -0,0 +1,94 @@
+ImplFile
+  (ParsedImplFileInput
+     ("/root/Expression/Unfinished escaped ident 01.fs", false,
+      QualifiedNameOfFile Module, [], [],
+      [SynModuleOrNamespace
+         ([Module], false, NamedModule,
+          [Expr
+             (Do (FromParseError (Ident , (4,4--4,5)), (3,0--4,5)), (3,0--4,5));
+           Expr (Do (Ident  , (6,0--7,7)), (6,0--7,7));
+           Expr (Do (Ident ` , (9,0--10,8)), (9,0--10,8));
+           Expr
+             (Do (FromParseError (Ident , (13,4--13,8)), (12,0--13,8)),
+              (12,0--13,8)); Expr (Do (Ident  , (15,0--16,9)), (15,0--16,9));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false, FromParseError (Ident , (19,4--19,5)),
+                    Ident  ` , (19,4--19,11)), (18,0--19,11)), (18,0--19,11));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false, FromParseError (Ident , (22,4--22,5)),
+                    Ident  , (22,4--22,11)), (21,0--22,11)), (21,0--22,11));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false, FromParseError (Ident , (25,4--25,8)),
+                    FromParseError (Ident , (25,8--25,12)), (25,4--25,12)),
+                 (24,0--25,12)), (24,0--25,12));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false, FromParseError (Ident , (28,4--28,8)),
+                    Ident ` , (28,4--28,12)), (27,0--28,12)), (27,0--28,12));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false, FromParseError (Ident , (31,4--31,8)),
+                    Ident  , (31,4--31,11)), (30,0--31,11)), (30,0--31,11));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false, FromParseError (Ident , (34,4--34,8)),
+                    FromParseError (Ident , (34,8--34,9)), (34,4--34,9)),
+                 (33,0--34,9)), (33,0--34,9));
+           Expr (Do (Ident ` , (36,0--37,10)), (36,0--37,10));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false, Ident ` ,
+                    FromParseError (Ident , (40,10--40,11)), (40,4--40,11)),
+                 (39,0--40,11)), (39,0--40,11));
+           Expr
+             (Do
+                (DotGet
+                   (FromParseError (Ident , (43,4--43,8)), (43,8--43,9),
+                    SynLongIdent ([P], [], [None]), (43,4--43,10)),
+                 (42,0--43,10)), (42,0--43,10));
+           Expr
+             (Do
+                (App
+                   (NonAtomic, false,
+                    App
+                      (NonAtomic, true,
+                       LongIdent
+                         (false,
+                          SynLongIdent
+                            ([op_Addition], [], [Some (OriginalNotation "+")]),
+                          None, (46,6--46,7)), Const (Int32 1, (46,4--46,5)),
+                       (46,4--46,7)), FromParseError (Ident , (46,8--46,9)),
+                    (46,4--46,9)), (45,0--46,9)), (45,0--46,9))],
+          PreXmlDoc ((1,0), FSharp.Compiler.Xml.XmlDocCollector), [], None,
+          (1,0--46,9), { LeadingKeyword = Module (1,0--1,6) })], (true, true),
+      { ConditionalDirectives = []
+        CodeComments = [] }, set []))
+
+(4,4)-(4,5) parse error This is not a valid identifier
+(7,4)-(7,7) parse error This is not a valid identifier
+(10,4)-(10,8) parse error This is not a valid identifier
+(13,4)-(13,8) parse error This is not a valid identifier
+(19,4)-(19,5) parse error This is not a valid identifier
+(19,6)-(19,11) parse error This is not a valid identifier
+(22,4)-(22,5) parse error This is not a valid identifier
+(25,4)-(25,8) parse error This is not a valid identifier
+(25,8)-(25,12) parse error This is not a valid identifier
+(28,4)-(28,8) parse error This is not a valid identifier
+(28,8)-(28,12) parse error This is not a valid identifier
+(31,4)-(31,8) parse error This is not a valid identifier
+(31,8)-(31,11) parse error This is not a valid identifier
+(34,4)-(34,8) parse error This is not a valid identifier
+(34,8)-(34,9) parse error This is not a valid identifier
+(40,10)-(40,11) parse error This is not a valid identifier
+(43,4)-(43,8) parse error This is not a valid identifier
+(46,8)-(46,9) parse error This is not a valid identifier

tests/service/data/SyntaxTree/Expression/Unfinished escaped ident 02.fs

@@ -0,0 +1,6 @@
+module Module
+
+type T() =
+    member this.`() = ()
+
+    member this.````() = ()