Skip to content

Commit e333828

Browse files
auduchinokauduchinok
authored
Parser: fix while, for, do expression ranges (#15369)
* Parser: fix 'done' range in expressions
1 parent dbe12da commit e333828

File tree

16 files changed

+270
-141
lines changed

16 files changed

+270
-141
lines changed

‎src/Compiler/Service/ServiceLexing.fs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -316,12 +316,12 @@ module internal TokenClassifications =
316316
| NAMESPACE
317317
| OASSERT
318318
| OLAZY
319-
| ODECLEND
319+
| ODECLEND _
320320
| OBLOCKSEP
321321
| OEND
322322
| OBLOCKBEGIN
323-
| ORIGHT_BLOCK_END
324-
| OBLOCKEND
323+
| ORIGHT_BLOCK_END _
324+
| OBLOCKEND _
325325
| OBLOCKEND_COMING_SOON
326326
| OBLOCKEND_IS_HERE
327327
| OTHEN
@@ -1506,9 +1506,9 @@ type FSharpToken =
15061506
| STRING_TEXT _ -> FSharpTokenKind.StringText
15071507
| FIXED -> FSharpTokenKind.Fixed
15081508
| OINTERFACE_MEMBER -> FSharpTokenKind.OffsideInterfaceMember
1509-
| OBLOCKEND -> FSharpTokenKind.OffsideBlockEnd
1510-
| ORIGHT_BLOCK_END -> FSharpTokenKind.OffsideRightBlockEnd
1511-
| ODECLEND -> FSharpTokenKind.OffsideDeclEnd
1509+
| OBLOCKEND _ -> FSharpTokenKind.OffsideBlockEnd
1510+
| ORIGHT_BLOCK_END _ -> FSharpTokenKind.OffsideRightBlockEnd
1511+
| ODECLEND _ -> FSharpTokenKind.OffsideDeclEnd
15121512
| OEND -> FSharpTokenKind.OffsideEnd
15131513
| OBLOCKSEP -> FSharpTokenKind.OffsideBlockSep
15141514
| OBLOCKBEGIN -> FSharpTokenKind.OffsideBlockBegin

‎src/Compiler/SyntaxTree/LexFilter.fs‎

Lines changed: 31 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ let rec isIfBlockContinuator token =
210210
// end else ...
211211
| END | RPAREN -> true
212212
// The following arise during reprocessing of the inserted tokens, e.g. when we hit a DONE
213-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
213+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
214214
| ODUMMY token -> isIfBlockContinuator token
215215
| _ -> false
216216

@@ -224,7 +224,7 @@ let rec isMatchBlockContinuator token =
224224
// with ...
225225
| WITH -> true
226226
// The following arise during reprocessing of the inserted tokens when we hit a DONE
227-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
227+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
228228
| ODUMMY token -> isMatchBlockContinuator token
229229
| _ -> false
230230

@@ -237,14 +237,14 @@ let rec isTryBlockContinuator token =
237237
// with ...
238238
| FINALLY | WITH -> true
239239
// The following arise during reprocessing of the inserted tokens when we hit a DONE
240-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
240+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
241241
| ODUMMY token -> isTryBlockContinuator token
242242
| _ -> false
243243

244244
let rec isThenBlockContinuator token =
245245
match token with
246246
// The following arise during reprocessing of the inserted tokens when we hit a DONE
247-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
247+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
248248
| ODUMMY token -> isThenBlockContinuator token
249249
| _ -> false
250250

@@ -256,7 +256,7 @@ let rec isDoContinuator token =
256256
// ...
257257
// done *)
258258
| DONE -> true
259-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
259+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
260260
| ODUMMY token -> isDoContinuator token
261261
| _ -> false
262262

@@ -267,7 +267,7 @@ let rec isInterfaceContinuator token =
267267
// ...
268268
// end
269269
| END -> true
270-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
270+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
271271
| ODUMMY token -> isInterfaceContinuator token
272272
| _ -> false
273273

@@ -304,7 +304,7 @@ let rec isTypeContinuator token =
304304
| RBRACE _ | WITH | BAR | AND | END -> true
305305

306306
// The following arise during reprocessing of the inserted tokens when we hit a DONE
307-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
307+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
308308
| ODUMMY token -> isTypeContinuator token
309309
| _ -> false
310310

@@ -315,7 +315,7 @@ let rec isForLoopContinuator token =
315315
// ...
316316
// done
317317
| DONE -> true
318-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true// The following arise during reprocessing of the inserted tokens when we hit a DONE
318+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true// The following arise during reprocessing of the inserted tokens when we hit a DONE
319319
| ODUMMY token -> isForLoopContinuator token
320320
| _ -> false
321321

@@ -326,7 +326,7 @@ let rec isWhileBlockContinuator token =
326326
// ...
327327
// done
328328
| DONE -> true
329-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
329+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
330330
| ODUMMY token -> isWhileBlockContinuator token
331331
| _ -> false
332332

@@ -336,7 +336,7 @@ let rec isLetContinuator token =
336336
// let ...
337337
// and ...
338338
| AND -> true
339-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
339+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
340340
| ODUMMY token -> isLetContinuator token
341341
| _ -> false
342342

@@ -349,7 +349,7 @@ let rec isTypeSeqBlockElementContinuator token =
349349
// member x.M1
350350
// member x.M2
351351
| BAR -> true
352-
| OBLOCKBEGIN | ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
352+
| OBLOCKBEGIN | ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true // The following arise during reprocessing of the inserted tokens when we hit a DONE
353353
| ODUMMY token -> isTypeSeqBlockElementContinuator token
354354
| _ -> false
355355

@@ -373,7 +373,7 @@ let rec isSeqBlockElementContinuator token =
373373
| END | AND | WITH | THEN | RPAREN | RBRACE _ | BAR_RBRACE | RBRACK | BAR_RBRACK | RQUOTE _ -> true
374374

375375
// The following arise during reprocessing of the inserted tokens when we hit a DONE
376-
| ORIGHT_BLOCK_END | OBLOCKEND | ODECLEND -> true
376+
| ORIGHT_BLOCK_END _ | OBLOCKEND _ | ODECLEND _ -> true
377377
| ODUMMY token -> isSeqBlockElementContinuator token
378378
| _ -> false
379379

@@ -1271,6 +1271,10 @@ type LexFilterImpl (
12711271
delayToken tokenTup
12721272
hwTokenFetch false
12731273

1274+
let getLastTokenEndRange () =
1275+
let lastTokenPos = tokenTup.LastTokenPos
1276+
mkSynRange lastTokenPos lastTokenPos
1277+
12741278
let insertTokenFromPrevPosToCurrentPos tok =
12751279
delayToken tokenTup
12761280
if debug then dprintf "inserting %+A\n" tok
@@ -1344,18 +1348,17 @@ type LexFilterImpl (
13441348
| CtxtWithAsLet _ ->
13451349
Some OEND
13461350

1347-
| CtxtWithAsAugment _ ->
1348-
Some ODECLEND
1351+
| CtxtWithAsAugment _
13491352

13501353
| CtxtDo _
13511354
| CtxtLetDecl (true, _) ->
1352-
Some ODECLEND
1355+
Some (ODECLEND(getLastTokenEndRange ()))
13531356

13541357
| CtxtSeqBlock(_, _, AddBlockEnd) ->
1355-
Some OBLOCKEND
1358+
Some (OBLOCKEND(getLastTokenEndRange ()))
13561359

13571360
| CtxtSeqBlock(_, _, AddOneSidedBlockEnd) ->
1358-
Some ORIGHT_BLOCK_END
1361+
Some (ORIGHT_BLOCK_END(getLastTokenEndRange ()))
13591362

13601363
| _ ->
13611364
None
@@ -1426,15 +1429,15 @@ type LexFilterImpl (
14261429
popCtxt()
14271430
effectsToDo <- (fun() ->
14281431
if debug then dprintf "--> because %s is coming, inserting OBLOCKEND\n" keywordName
1429-
delayTokenNoProcessing (pool.UseLocation(tokenTup, OBLOCKEND))) :: effectsToDo
1432+
delayTokenNoProcessing (pool.UseLocation(tokenTup, OBLOCKEND(getLastTokenEndRange ())))) :: effectsToDo
14301433
| CtxtSeqBlock(_, _, NoAddBlockEnd) ->
14311434
if debug then dprintf "--> because %s is coming, popping CtxtSeqBlock\n" keywordName
14321435
popCtxt()
14331436
| CtxtSeqBlock(_, _, AddOneSidedBlockEnd) ->
14341437
popCtxt()
14351438
effectsToDo <- (fun() ->
14361439
if debug then dprintf "--> because %s is coming, inserting ORIGHT_BLOCK_END\n" keywordName
1437-
delayTokenNoProcessing (pool.UseLocation(tokenTup, ORIGHT_BLOCK_END))) :: effectsToDo
1440+
delayTokenNoProcessing (pool.UseLocation(tokenTup, ORIGHT_BLOCK_END(getLastTokenEndRange ())))) :: effectsToDo
14381441
| CtxtVanilla _ ->
14391442
if debug then dprintf "--> because %s is coming, popping CtxtVanilla\n" keywordName
14401443
popCtxt()
@@ -1497,15 +1500,15 @@ type LexFilterImpl (
14971500
popCtxt()
14981501
// Make sure we queue a dummy token at this position to check if any other pop rules apply
14991502
delayToken(pool.UseLocation(tokenTup, ODUMMY token))
1500-
returnToken tokenLexbufState (if blockLet then ODECLEND else token)
1503+
returnToken tokenLexbufState (if blockLet then ODECLEND(getLastTokenEndRange ()) else token)
15011504

15021505
// Balancing rule. Encountering a 'done' balances with a 'do'. i.e. even a non-offside 'done' closes a 'do'
15031506
// The 'DONE' token is thrown away and becomes an ODECLEND
15041507
| DONE, CtxtDo offsidePos :: _ ->
15051508
if debug then dprintf "DONE at %a terminates CtxtDo(offsidePos=%a)\n" outputPos tokenStartPos outputPos offsidePos
15061509
popCtxt()
15071510
// reprocess as the DONE may close a DO context
1508-
delayToken(pool.UseLocation(tokenTup, ODECLEND))
1511+
delayToken(pool.UseLocation(tokenTup, ODECLEND(mkSynRange tokenTup.StartPos tokenTup.EndPos)))
15091512
pool.Return tokenTup
15101513
hwTokenFetch useBlockRule
15111514

@@ -1670,8 +1673,8 @@ type LexFilterImpl (
16701673
popCtxt()
16711674
if debug then (match addBlockEnd with AddBlockEnd -> dprintf "end of CtxtSeqBlock, insert OBLOCKEND \n" | _ -> ())
16721675
match addBlockEnd with
1673-
| AddBlockEnd -> insertToken OBLOCKEND
1674-
| AddOneSidedBlockEnd -> insertToken ORIGHT_BLOCK_END
1676+
| AddBlockEnd -> insertToken (OBLOCKEND(getLastTokenEndRange ()))
1677+
| AddOneSidedBlockEnd -> insertToken (ORIGHT_BLOCK_END(getLastTokenEndRange ()))
16751678
| NoAddBlockEnd -> reprocess()
16761679

16771680
// Offside rule for SeqBlock.
@@ -1753,7 +1756,7 @@ type LexFilterImpl (
17531756
isSemiSemi || (if relaxWhitespace2OffsideRule || isLetContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
17541757
if debug then dprintf "token at column %d is offside from LET(offsidePos=%a)! delaying token, returning ODECLEND\n" tokenStartCol outputPos offsidePos
17551758
popCtxt()
1756-
insertToken ODECLEND
1759+
insertToken (ODECLEND(getLastTokenEndRange ()))
17571760

17581761
// do ignore (
17591762
// 1
@@ -1763,7 +1766,7 @@ type LexFilterImpl (
17631766
when isSemiSemi || (if isDoContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
17641767
if debug then dprintf "token at column %d is offside from DO(offsidePos=%a)! delaying token, returning ODECLEND\n" tokenStartCol outputPos offsidePos
17651768
popCtxt()
1766-
insertToken ODECLEND
1769+
insertToken (ODECLEND(getLastTokenEndRange ()))
17671770

17681771
// class
17691772
// interface AAA
@@ -1815,7 +1818,7 @@ type LexFilterImpl (
18151818
| _, CtxtMemberBody offsidePos :: _ when isSemiSemi || (if false then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
18161819
if debug then dprintf "token at column %d is offside from MEMBER/OVERRIDE head with offsidePos %a!\n" tokenStartCol outputPos offsidePos
18171820
popCtxt()
1818-
insertToken ODECLEND
1821+
insertToken (ODECLEND(getLastTokenEndRange ()))
18191822

18201823
// Pop CtxtMemberHead when offside
18211824
| _, CtxtMemberHead offsidePos :: _ when isSemiSemi || (if relaxWhitespace2OffsideRule then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
@@ -1839,7 +1842,7 @@ type LexFilterImpl (
18391842
when isSemiSemi || (if relaxWhitespace2OffsideRule || isWithAugmentBlockContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
18401843
if debug then dprintf "offside from CtxtWithAsAugment, isWithAugmentBlockContinuator = %b\n" (isWithAugmentBlockContinuator token)
18411844
popCtxt()
1842-
insertToken ODECLEND
1845+
insertToken (ODECLEND(getLastTokenEndRange ()))
18431846

18441847
| _, CtxtMatch offsidePos :: _
18451848
when isSemiSemi || (if relaxWhitespace2OffsideRule || relaxWhitespace2 && isMatchBlockContinuator token then tokenStartCol + 1 else tokenStartCol) <= offsidePos.Column ->
@@ -2593,7 +2596,7 @@ type LexFilter (indentationSyntaxStatus: IndentationAwareSyntaxStatus, compiling
25932596
| RPAREN ->
25942597
insertComingSoonTokens RPAREN_COMING_SOON RPAREN_IS_HERE
25952598
lexer.GetToken()
2596-
| OBLOCKEND ->
2599+
| OBLOCKEND _ ->
25972600
insertComingSoonTokens OBLOCKEND_COMING_SOON OBLOCKEND_IS_HERE
25982601
lexer.GetToken()
25992602
| _ -> token

‎src/Compiler/SyntaxTree/LexHelpers.fs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ module Keywords =
390390
(*------- for prototyping and explaining offside rule *)
391391
FSHARP, "__token_OBLOCKSEP", OBLOCKSEP
392392
FSHARP, "__token_OWITH", OWITH
393-
FSHARP, "__token_ODECLEND", ODECLEND
393+
FSHARP, "__token_ODECLEND", ODECLEND range0
394394
FSHARP, "__token_OTHEN", OTHEN
395395
FSHARP, "__token_OELSE", OELSE
396396
FSHARP, "__token_OEND", OEND

0 commit comments

Comments
 (0)