Tokenizer: don't assert non-empty identifiers (#15462)

* Tokenizer: don't assert non-empty identifiers

* Add test

Author: auduchinok

src/Compiler/Service/ServiceLexing.fs

@@ -173,11 +173,7 @@ module internal TokenClassifications =
         match token with
         | HASH_IDENT s
         | IDENT s ->
-            if s.Length <= 0 then
-                System.Diagnostics.Debug.Assert(false, "BUG: Received zero length IDENT token.")
-                // This is related to 4783. Recover by treating as lower case identifier.
-                (FSharpTokenColorKind.Identifier, FSharpTokenCharKind.Identifier, FSharpTokenTriggerClass.None)
-            else if Char.ToUpperInvariant s[0] = s[0] then
+            if s.Length > 0 && Char.ToUpperInvariant s[0] = s[0] then
                 (FSharpTokenColorKind.UpperIdentifier, FSharpTokenCharKind.Identifier, FSharpTokenTriggerClass.None)
             else
                 (FSharpTokenColorKind.Identifier, FSharpTokenCharKind.Identifier, FSharpTokenTriggerClass.None)

tests/service/TokenizerTests.fs

@@ -9,7 +9,7 @@ module FSharp.Compiler.Service.Tests.TokenizerTests
 #endif
 
 open FSharp.Compiler.Tokenization
-
+open FsUnit
 open NUnit.Framework
 
 let sourceTok = FSharpSourceTokenizer([], Some "C:\\test.fsx", None)
@@ -214,3 +214,22 @@ let ``Tokenizer test - single-line nested string interpolation``() =
         printfn "expected = %A" expected
         Assert.Fail(sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)
 
+
+[<Test>]
+let ``Unfinished idents``() =
+    let tokenizedLines =
+      tokenizeLines
+        [| "`"; "``"; "``a"; "``a`"; "```" |]
+
+    let actual =
+        [ for lineTokens in tokenizedLines |> List.map snd do
+            [ for str, info in lineTokens -> info.TokenName, str ] ]
+
+    let expected =
+        [["IDENT", "`"]
+         ["IDENT", "``"]
+         ["IDENT", "``a"]
+         ["IDENT", "``a`"]
+         ["IDENT", "```"]]
+
+    actual |> shouldEqual expected