Parser: fix while, for, do expression ranges

src/Compiler/Service/ServiceLexing.fs

@@ -316,12 +316,12 @@ module internal TokenClassifications =
         | NAMESPACE
         | OASSERT
         | OLAZY
-        | ODECLEND
+        | ODECLEND _
         | OBLOCKSEP
         | OEND
         | OBLOCKBEGIN
-        | ORIGHT_BLOCK_END
-        | OBLOCKEND
+        | ORIGHT_BLOCK_END _
+        | OBLOCKEND _
         | OBLOCKEND_COMING_SOON
         | OBLOCKEND_IS_HERE
         | OTHEN
@@ -1506,9 +1506,9 @@ type FSharpToken =
         | STRING_TEXT _ -> FSharpTokenKind.StringText
         | FIXED -> FSharpTokenKind.Fixed
         | OINTERFACE_MEMBER -> FSharpTokenKind.OffsideInterfaceMember
-        | OBLOCKEND -> FSharpTokenKind.OffsideBlockEnd
-        | ORIGHT_BLOCK_END -> FSharpTokenKind.OffsideRightBlockEnd
-        | ODECLEND -> FSharpTokenKind.OffsideDeclEnd
+        | OBLOCKEND _ -> FSharpTokenKind.OffsideBlockEnd
+        | ORIGHT_BLOCK_END _ -> FSharpTokenKind.OffsideRightBlockEnd
+        | ODECLEND _ -> FSharpTokenKind.OffsideDeclEnd
         | OEND -> FSharpTokenKind.OffsideEnd
         | OBLOCKSEP -> FSharpTokenKind.OffsideBlockSep
         | OBLOCKBEGIN -> FSharpTokenKind.OffsideBlockBegin

src/Compiler/SyntaxTree/LexFilter.fs

@@ -210,7 +210,7 @@ let rec isIfBlockContinuator token =
     //    end else ...
     | END | RPAREN -> true
     // The following arise during reprocessing of the inserted tokens, e.g. when we hit a DONE
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
     | ODUMMY token -> isIfBlockContinuator token
     | _ -> false
 
@@ -224,7 +224,7 @@ let rec isMatchBlockContinuator token =
     //         with ...
     | WITH -> true
     // The following arise during reprocessing of the inserted tokens when we hit a DONE
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
     | ODUMMY token -> isMatchBlockContinuator token
     | _ -> false
 
@@ -237,14 +237,14 @@ let rec isTryBlockContinuator token =
     //         with ...
     | FINALLY | WITH -> true
     // The following arise during reprocessing of the inserted tokens when we hit a DONE
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
     | ODUMMY token -> isTryBlockContinuator token
     | _ -> false
 
 let rec isThenBlockContinuator token =
     match token with
     // The following arise during reprocessing of the inserted tokens when we hit a DONE
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
     | ODUMMY token -> isThenBlockContinuator token
     | _ -> false
 
@@ -256,7 +256,7 @@ let rec isDoContinuator token =
     //                             ...
     //                          done *)
     | DONE -> true
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
     | ODUMMY token -> isDoContinuator token
     | _ -> false
 
@@ -267,7 +267,7 @@ let rec isInterfaceContinuator token =
     //                         ...
     //                       end
     | END -> true
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
     | ODUMMY token -> isInterfaceContinuator token
     | _ -> false
 
@@ -304,7 +304,7 @@ let rec isTypeContinuator token =
     | RBRACE _ | WITH | BAR | AND | END -> true
 
     // The following arise during reprocessing of the inserted tokens when we hit a DONE
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
     | ODUMMY token -> isTypeContinuator token
     | _ -> false
 
@@ -315,7 +315,7 @@ let rec isForLoopContinuator token =
     //                          ...
     //                       done
     | DONE -> true
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true// The following arise during reprocessing of the inserted tokens when we hit a DONE
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true// The following arise during reprocessing of the inserted tokens when we hit a DONE
     | ODUMMY token -> isForLoopContinuator token
     | _ -> false
 
@@ -326,7 +326,7 @@ let rec isWhileBlockContinuator token =
     //                          ...
     //                       done
     | DONE -> true
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
     | ODUMMY token -> isWhileBlockContinuator token
     | _ -> false
 
@@ -336,7 +336,7 @@ let rec isLetContinuator token =
     //                       let ...
     //                       and ...
     | AND -> true
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
     | ODUMMY token -> isLetContinuator token
     | _ -> false
 
@@ -349,7 +349,7 @@ let rec isTypeSeqBlockElementContinuator token =
     //   member x.M1
     //   member x.M2
     | BAR -> true
-    | OBLOCKBEGIN | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
+    | OBLOCKBEGIN | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
     | ODUMMY token -> isTypeSeqBlockElementContinuator token
     | _ -> false
 
@@ -373,7 +373,7 @@ let rec isSeqBlockElementContinuator token =
     | END | AND | WITH | THEN | RPAREN | RBRACE _ | BAR_RBRACE | RBRACK | BAR_RBRACK | RQUOTE _ -> true
 
     // The following arise during reprocessing of the inserted tokens when we hit a DONE
-    | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
+    | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
     | ODUMMY token -> isSeqBlockElementContinuator token
     | _ -> false
 
@@ -1271,6 +1271,10 @@ type LexFilterImpl (
             delayToken tokenTup
             hwTokenFetch false
 
+        let getLastTokenEndRange () =
+            let lastTokenPos = tokenTup.LastTokenPos
+            mkSynRange lastTokenPos lastTokenPos
+
         let insertTokenFromPrevPosToCurrentPos tok =
             delayToken tokenTup
             if debug then dprintf "inserting %+A\n" tok
@@ -1344,18 +1348,17 @@ type LexFilterImpl (
             | CtxtWithAsLet _ ->
                 Some OEND
 
-            | CtxtWithAsAugment _ ->
-                Some ODECLEND
+            | CtxtWithAsAugment _
 
             | CtxtDo _
             | CtxtLetDecl (true, _) ->
-                Some ODECLEND
+                Some (ODECLEND(getLastTokenEndRange ()))
 
             | CtxtSeqBlock(_, _, AddBlockEnd) ->
-                Some OBLOCKEND
+                Some (OBLOCKEND(getLastTokenEndRange ()))
 
             | CtxtSeqBlock(_, _, AddOneSidedBlockEnd) ->
-                Some ORIGHT_BLOCK_END
+                Some (ORIGHT_BLOCK_END(getLastTokenEndRange ()))
 
             | _ ->
                 None
@@ -1426,15 +1429,15 @@ type LexFilterImpl (
                         popCtxt()
                         effectsToDo <- (fun() ->
                             if debug then dprintf "--> because %s is coming, inserting OBLOCKEND\n" keywordName
-                            delayTokenNoProcessing (pool.UseLocation(tokenTup, OBLOCKEND))) :: effectsToDo
+                            delayTokenNoProcessing (pool.UseLocation(tokenTup, OBLOCKEND(getLastTokenEndRange ())))) :: effectsToDo
                     | CtxtSeqBlock(_, _, NoAddBlockEnd) ->
                         if debug then dprintf "--> because %s is coming, popping CtxtSeqBlock\n" keywordName
                         popCtxt()
                     | CtxtSeqBlock(_, _, AddOneSidedBlockEnd) ->
                         popCtxt()
                         effectsToDo <- (fun() ->
                             if debug then dprintf "--> because %s is coming, inserting ORIGHT_BLOCK_END\n" keywordName
-                            delayTokenNoProcessing (pool.UseLocation(tokenTup, ORIGHT_BLOCK_END))) :: effectsToDo
+                            delayTokenNoProcessing (pool.UseLocation(tokenTup, ORIGHT_BLOCK_END(getLastTokenEndRange ())))) :: effectsToDo
                     | CtxtVanilla _ ->
                         if debug then dprintf "--> because %s is coming, popping CtxtVanilla\n" keywordName
                         popCtxt()
@@ -1497,15 +1500,15 @@ type LexFilterImpl (
             popCtxt()
             // Make sure we queue a dummy token at this position to check if any other pop rules apply
             delayToken(pool.UseLocation(tokenTup, ODUMMY token))
-            returnToken tokenLexbufState (if blockLet then ODECLEND else token)
+            returnToken tokenLexbufState (if blockLet then ODECLEND(getLastTokenEndRange ()) else token)
 
         // Balancing rule. Encountering a 'done' balances with a 'do'. i.e. even a non-offside 'done' closes a 'do'
         // The 'DONE' token is thrown away and becomes an ODECLEND
         | DONE, CtxtDo offsidePos :: _ ->
             if debug then dprintf "DONE at %a terminates CtxtDo(offsidePos=%a)\n" outputPos tokenStartPos outputPos offsidePos
             popCtxt()
             // reprocess as the DONE may close a DO context
-            delayToken(pool.UseLocation(tokenTup, ODECLEND))
+            delayToken(pool.UseLocation(tokenTup, ODECLEND(mkSynRange tokenTup.StartPos tokenTup.EndPos)))
             pool.Return tokenTup
             hwTokenFetch useBlockRule
 
@@ -1670,8 +1673,8 @@ type LexFilterImpl (
             popCtxt()
             if debug then (match addBlockEnd with AddBlockEnd -> dprintf "end of CtxtSeqBlock, insert OBLOCKEND \n" | _ -> ())
             match addBlockEnd with
-            | AddBlockEnd -> insertToken OBLOCKEND
-            | AddOneSidedBlockEnd -> insertToken ORIGHT_BLOCK_END
+            | AddBlockEnd -> insertToken (OBLOCKEND(getLastTokenEndRange ()))
+            | AddOneSidedBlockEnd -> insertToken (ORIGHT_BLOCK_END(getLastTokenEndRange ()))
             | NoAddBlockEnd -> reprocess()
 
         //  Offside rule for SeqBlock.
@@ -1753,7 +1756,7 @@ type LexFilterImpl (
                         isSemiSemi || (if relaxWhitespace2OffsideRule || isLetContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
             if debug then dprintf "token at column %d is offside from LET(offsidePos=%a)! delaying token, returning ODECLEND\n" tokenStartCol outputPos offsidePos
             popCtxt()
-            insertToken ODECLEND
+            insertToken (ODECLEND(getLastTokenEndRange ()))
 
         // do ignore (
         //     1
@@ -1763,7 +1766,7 @@ type LexFilterImpl (
                 when isSemiSemi || (if isDoContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
             if debug then dprintf "token at column %d is offside from DO(offsidePos=%a)! delaying token, returning ODECLEND\n" tokenStartCol outputPos offsidePos
             popCtxt()
-            insertToken ODECLEND
+            insertToken (ODECLEND(getLastTokenEndRange ()))
 
         // class
         //    interface AAA
@@ -1815,7 +1818,7 @@ type LexFilterImpl (
         | _, CtxtMemberBody offsidePos :: _ when isSemiSemi || (if false then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
             if debug then dprintf "token at column %d is offside from MEMBER/OVERRIDE head with offsidePos %a!\n" tokenStartCol outputPos offsidePos
             popCtxt()
-            insertToken ODECLEND
+            insertToken (ODECLEND(getLastTokenEndRange ()))
 
         // Pop CtxtMemberHead when offside
         | _, CtxtMemberHead offsidePos :: _ when isSemiSemi || (if relaxWhitespace2OffsideRule then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
@@ -1839,7 +1842,7 @@ type LexFilterImpl (
                     when isSemiSemi || (if relaxWhitespace2OffsideRule || isWithAugmentBlockContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
             if debug then dprintf "offside from CtxtWithAsAugment, isWithAugmentBlockContinuator = %b\n" (isWithAugmentBlockContinuator token)
             popCtxt()
-            insertToken ODECLEND
+            insertToken (ODECLEND(getLastTokenEndRange ()))
 
         | _, CtxtMatch offsidePos :: _
                     when isSemiSemi || (if relaxWhitespace2OffsideRule || relaxWhitespace2 && isMatchBlockContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
@@ -2593,7 +2596,7 @@ type LexFilter (indentationSyntaxStatus: IndentationAwareSyntaxStatus, compiling
         | RPAREN ->
             insertComingSoonTokens RPAREN_COMING_SOON RPAREN_IS_HERE
             lexer.GetToken()
-        | OBLOCKEND ->
+        | OBLOCKEND _ ->
             insertComingSoonTokens OBLOCKEND_COMING_SOON OBLOCKEND_IS_HERE
             lexer.GetToken()
         | _ -> token

src/Compiler/SyntaxTree/LexHelpers.fs

@@ -390,7 +390,7 @@ module Keywords =
             (*------- for prototyping and explaining offside rule *)
             FSHARP, "__token_OBLOCKSEP", OBLOCKSEP
             FSHARP, "__token_OWITH", OWITH
-            FSHARP, "__token_ODECLEND", ODECLEND
+            FSHARP, "__token_ODECLEND", ODECLEND range0
             FSHARP, "__token_OTHEN", OTHEN
             FSHARP, "__token_OELSE", OELSE
             FSHARP, "__token_OEND", OEND