From 85fe02eb49309dc073afc2641f4cca7db91f8fa1 Mon Sep 17 00:00:00 2001 From: dawe Date: Tue, 27 Feb 2024 11:39:15 +0100 Subject: [PATCH 1/2] fix wrong range start of INTERP_STRING_END --- src/Compiler/Service/FSharpCheckerResults.fs | 2 +- src/Compiler/Service/ServiceLexing.fs | 12 ++-- src/Compiler/SyntaxTree/LexHelpers.fs | 2 +- src/Compiler/SyntaxTree/ParseHelpers.fs | 4 +- src/Compiler/SyntaxTree/ParseHelpers.fsi | 2 +- src/Compiler/lex.fsl | 62 ++++++++++--------- src/Compiler/pars.fsy | 11 +++- ...tringWithTripleQuoteMultipleDollars.fs.bsl | 2 +- ...edStringWithTripleQuoteMultipleDollars2.fs | 2 + ...ringWithTripleQuoteMultipleDollars2.fs.bsl | 18 ++++++ 10 files changed, 74 insertions(+), 43 deletions(-) create mode 100644 tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs create mode 100644 tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs.bsl diff --git a/src/Compiler/Service/FSharpCheckerResults.fs b/src/Compiler/Service/FSharpCheckerResults.fs index 045e0aae7a0..dda21fffd39 100644 --- a/src/Compiler/Service/FSharpCheckerResults.fs +++ b/src/Compiler/Service/FSharpCheckerResults.fs @@ -2685,7 +2685,7 @@ module internal ParseAndCheckFile = | INTERP_STRING_BEGIN_PART _ | INTERP_STRING_PART _ as tok, _ -> let braceOffset = match tok with - | INTERP_STRING_BEGIN_PART(_, SynStringKind.TripleQuote, (LexerContinuation.Token(_, (_, _, dl, _) :: _))) -> + | INTERP_STRING_BEGIN_PART(_, SynStringKind.TripleQuote, (LexerContinuation.Token(_, (_, _, dl, _, _) :: _))) -> dl - 1 | _ -> 0 diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index 66893ac950e..bc967f30dbe 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -36,7 +36,7 @@ module FSharpTokenTag = tagOfToken (INTERP_STRING_BEGIN_PART("a", SynStringKind.Regular, LexCont.Default)) let INTERP_STRING_PART = tagOfToken (INTERP_STRING_PART("a", LexCont.Default)) - let INTERP_STRING_END = tagOfToken (INTERP_STRING_END("a", LexCont.Default)) + let INTERP_STRING_END = tagOfToken (INTERP_STRING_END("a", None, LexCont.Default)) let LPAREN = tagOfToken LPAREN let RPAREN = tagOfToken RPAREN let LBRACK = tagOfToken LBRACK @@ -493,7 +493,7 @@ module internal LexerStateEncoding = | INTERP_STRING_BEGIN_PART(_, _, cont) | INTERP_STRING_PART(_, cont) | INTERP_STRING_BEGIN_END(_, _, cont) - | INTERP_STRING_END(_, cont) + | INTERP_STRING_END(_, _, cont) | LBRACE cont | RBRACE cont | BYTEARRAY(_, _, cont) @@ -621,12 +621,12 @@ module internal LexerStateEncoding = let tag1, i1, kind1, rest = match stringNest with | [] -> false, 0, 0, [] - | (i1, kind1, _, _) :: rest -> true, i1, encodeStringStyle kind1, rest + | (i1, kind1, _, _, _) :: rest -> true, i1, encodeStringStyle kind1, rest let tag2, i2, kind2 = match rest with | [] -> false, 0, 0 - | (i2, kind2, _, _) :: _ -> true, i2, encodeStringStyle kind2 + | (i2, kind2, _, _, _) :: _ -> true, i2, encodeStringStyle kind2 (if tag1 then 0b100000000000 else 0) ||| (if tag2 then 0b010000000000 else 0) @@ -696,9 +696,9 @@ module internal LexerStateEncoding = let nest = [ if tag1 then - i1, decodeStringStyle kind1, 0, range0 + i1, decodeStringStyle kind1, 0, None, range0 if tag2 then - i2, decodeStringStyle kind2, 0, range0 + i2, decodeStringStyle kind2, 0, None, range0 ] nest diff --git a/src/Compiler/SyntaxTree/LexHelpers.fs b/src/Compiler/SyntaxTree/LexHelpers.fs index 02d4da364d4..5ee9a16c90b 100644 --- a/src/Compiler/SyntaxTree/LexHelpers.fs +++ b/src/Compiler/SyntaxTree/LexHelpers.fs @@ -190,7 +190,7 @@ type LexerStringFinisher = else if isPart then INTERP_STRING_PART(s, cont) else - INTERP_STRING_END(s, cont) + INTERP_STRING_END(s, None, cont) elif kind.IsByteString then let synByteStringKind = if isVerbatim then diff --git a/src/Compiler/SyntaxTree/ParseHelpers.fs b/src/Compiler/SyntaxTree/ParseHelpers.fs index cd4b41787e1..d90e395c0c9 100644 --- a/src/Compiler/SyntaxTree/ParseHelpers.fs +++ b/src/Compiler/SyntaxTree/ParseHelpers.fs @@ -308,7 +308,7 @@ type LexerStringKind = /// Represents the degree of nesting of '{..}' and the style of the string to continue afterwards, in an interpolation fill. /// Nesting counters and styles of outer interpolating strings are pushed on this stack. -type LexerInterpolatedStringNesting = (int * LexerStringStyle * int * range) list +type LexerInterpolatedStringNesting = (int * LexerStringStyle * int * range option * range) list /// The parser defines a number of tokens for whitespace and /// comments eliminated by the lexer. These carry a specification of @@ -973,7 +973,7 @@ let checkEndOfFileError t = match nesting with | [] -> () - | (_, _, _, m) :: _ -> reportParseErrorAt m (FSComp.SR.parsEofInInterpolatedStringFill ()) + | (_, _, _, _, m) :: _ -> reportParseErrorAt m (FSComp.SR.parsEofInInterpolatedStringFill ()) type BindingSet = BindingSetPreAttrs of range * bool * bool * (SynAttributes -> SynAccess option -> SynAttributes * SynBinding list) * range diff --git a/src/Compiler/SyntaxTree/ParseHelpers.fsi b/src/Compiler/SyntaxTree/ParseHelpers.fsi index 9add16af683..a8d61f3cb7e 100644 --- a/src/Compiler/SyntaxTree/ParseHelpers.fsi +++ b/src/Compiler/SyntaxTree/ParseHelpers.fsi @@ -118,7 +118,7 @@ type LexerStringKind = static member String: LexerStringKind -type LexerInterpolatedStringNesting = (int * LexerStringStyle * int * range) list +type LexerInterpolatedStringNesting = (int * LexerStringStyle * int * range option * range) list [] type LexerContinuation = diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index 995f3fe09c0..958c2fa51ea 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -120,7 +120,7 @@ let checkExprGreaterColonOp (lexbuf:UnicodeLexing.Lexbuf) = let unexpectedChar lexbuf = LEX_FAILURE (FSComp.SR.lexUnexpectedChar(lexeme lexbuf)) -let startString args (lexbuf: UnicodeLexing.Lexbuf) = +let startString args (lexbuf: UnicodeLexing.Lexbuf) altStartForStringEnd = let buf = ByteBuffer.Create StringCapacity let m = lexbuf.LexemeRange let startp = lexbuf.StartPos @@ -160,7 +160,7 @@ let startString args (lexbuf: UnicodeLexing.Lexbuf) = if isPart then INTERP_STRING_PART (s, cont) else - INTERP_STRING_END (s, cont) + INTERP_STRING_END (s, altStartForStringEnd, cont) else let s = Lexhelp.stringBufferAsString buf let synStringKind = @@ -587,12 +587,12 @@ rule token (args: LexArgs) (skip: bool) = parse else mlOnly m args skip lexbuf } | '"' - { let buf, fin, m = startString args lexbuf + { let buf, fin, m = startString args lexbuf None // Single quote in triple quote ok, others disallowed match args.stringNest with - | (_, LexerStringStyle.ExtendedInterpolated, _, _) :: _ - | (_, LexerStringStyle.TripleQuote, _, _) :: _ -> () + | (_, LexerStringStyle.ExtendedInterpolated, _, _, _) :: _ + | (_, LexerStringStyle.TripleQuote, _, _, _) :: _ -> () | _ :: _ -> errorR(Error(FSComp.SR.lexSingleQuoteInSingleQuote(), m)) | [] -> () @@ -600,7 +600,7 @@ rule token (args: LexArgs) (skip: bool) = parse else singleQuoteString (buf, fin, m, LexerStringKind.String, args) skip lexbuf } | '$' '"' '"' '"' - { let buf, fin, m = startString args lexbuf + { let buf, fin, m = startString args lexbuf None // Single quote in triple quote ok, others disallowed match args.stringNest with @@ -612,7 +612,7 @@ rule token (args: LexArgs) (skip: bool) = parse else tripleQuoteString (buf, fin, m, LexerStringKind.InterpolatedStringFirst, args) skip lexbuf } | ('$'+) '"' '"' '"' - { let buf, fin, m = startString args lexbuf + { let buf, fin, m = startString args lexbuf None if lexbuf.SupportsFeature LanguageFeature.ExtendedStringInterpolation then // Single quote in triple quote ok, others disallowed @@ -635,11 +635,11 @@ rule token (args: LexArgs) (skip: bool) = parse } | '$' '"' - { let buf,fin,m = startString args lexbuf + { let buf,fin,m = startString args lexbuf None // Single quote in triple quote ok, others disallowed match args.stringNest with - | (_, style, _, _) :: _ when style = LexerStringStyle.ExtendedInterpolated || style = LexerStringStyle.TripleQuote -> () + | (_, style, _, _, _) :: _ when style = LexerStringStyle.ExtendedInterpolated || style = LexerStringStyle.TripleQuote -> () | _ :: _ -> errorR(Error(FSComp.SR.lexSingleQuoteInSingleQuote(), m)) | _ -> () @@ -649,7 +649,7 @@ rule token (args: LexArgs) (skip: bool) = parse singleQuoteString (buf, fin, m, LexerStringKind.InterpolatedStringFirst, args) skip lexbuf } | '"' '"' '"' - { let buf, fin, m = startString args lexbuf + { let buf, fin, m = startString args lexbuf None args.interpolationDelimiterLength <- 0 @@ -664,12 +664,12 @@ rule token (args: LexArgs) (skip: bool) = parse tripleQuoteString (buf, fin, m, LexerStringKind.String, args) skip lexbuf } | '@' '"' - { let buf, fin, m = startString args lexbuf + { let buf, fin, m = startString args lexbuf None // Single quote in triple quote ok, others disallowed match args.stringNest with - | (_, LexerStringStyle.ExtendedInterpolated, _, _) :: _ - | (_, LexerStringStyle.TripleQuote, _, _) :: _ -> () + | (_, LexerStringStyle.ExtendedInterpolated, _, _, _) :: _ + | (_, LexerStringStyle.TripleQuote, _, _, _) :: _ -> () | _ :: _ -> errorR(Error(FSComp.SR.lexSingleQuoteInSingleQuote(), m)) | _ -> () @@ -679,11 +679,11 @@ rule token (args: LexArgs) (skip: bool) = parse verbatimString (buf, fin, m, LexerStringKind.String, args) skip lexbuf } | ("$@" | "@$") '"' - { let buf, fin, m = startString args lexbuf + { let buf, fin, m = startString args lexbuf None // Single quote in triple quote ok, others disallowed match args.stringNest with - | (_, style, _, _) :: _ when style = LexerStringStyle.ExtendedInterpolated || style = LexerStringStyle.TripleQuote -> () + | (_, style, _, _, _) :: _ when style = LexerStringStyle.ExtendedInterpolated || style = LexerStringStyle.TripleQuote -> () | _ :: _ -> errorR(Error(FSComp.SR.lexSingleQuoteInSingleQuote(), m)) | _ -> () @@ -888,10 +888,10 @@ rule token (args: LexArgs) (skip: bool) = parse { match args.stringNest with | [] -> () - | (counter, style, d, m) :: rest -> + | (counter, style, d, _, m) :: rest -> // Note, we do not update the 'm', any incomplete-interpolation error // will be reported w.r.t. the first '{' - args.stringNest <- (counter + 1, style, d, m) :: rest + args.stringNest <- (counter + 1, style, d, None, m) :: rest // To continue token-by-token lexing may involve picking up the new args.stringNes let cont = LexCont.Token(args.ifdefStack, args.stringNest) LBRACE cont @@ -904,12 +904,17 @@ rule token (args: LexArgs) (skip: bool) = parse // We encounter a '}' in the expression token stream. First check if we're in an interpolated string expression // and continue the string if necessary match args.stringNest with - | (1, LexerStringStyle.ExtendedInterpolated, delimLength, r) :: rest when delimLength > 1 -> - args.stringNest <- (1, LexerStringStyle.ExtendedInterpolated, delimLength - 1, r) :: rest + | (1, LexerStringStyle.ExtendedInterpolated, delimLength, altR, r) :: rest when delimLength > 1 -> + // On the first "}" of multiple "}", keep the range of the starting "}" for later processing in startString + let altStart = + match altR with + | None -> Some lexbuf.LexemeRange + | _ -> altR + args.stringNest <- (1, LexerStringStyle.ExtendedInterpolated, delimLength - 1, altStart, r) :: rest token args skip lexbuf - | (1, style, _, _) :: rest -> + | (1, style, _, altR, _r) :: rest -> args.stringNest <- rest - let buf, fin, m = startString args lexbuf + let buf, fin, m = startString args lexbuf altR if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, style, LexerStringKind.InterpolatedStringPart, args.interpolationDelimiterLength, m)) else @@ -918,11 +923,10 @@ rule token (args: LexArgs) (skip: bool) = parse | LexerStringStyle.SingleQuote -> singleQuoteString (buf, fin, m, LexerStringKind.InterpolatedStringPart, args) skip lexbuf | LexerStringStyle.TripleQuote -> tripleQuoteString (buf, fin, m, LexerStringKind.InterpolatedStringPart, args) skip lexbuf | LexerStringStyle.ExtendedInterpolated -> extendedInterpolatedString (buf, fin, m, LexerStringKind.InterpolatedStringPart, args) skip lexbuf - - | (counter, style, d, m) :: rest -> + | (counter, style, d, altR, m) :: rest -> // Note, we do not update the 'm', any incomplete-interpolation error // will be reported w.r.t. the first '{' - args.stringNest <- (counter - 1, style, d, m) :: rest + args.stringNest <- (counter - 1, style, d, altR, m) :: rest let cont = LexCont.Token(args.ifdefStack, args.stringNest) RBRACE cont @@ -1260,7 +1264,7 @@ and singleQuoteString (sargs: LexerStringArgs) (skip: bool) = parse if kind.IsInterpolated then // get a new range for where the fill starts let m2 = lexbuf.LexemeRange - args.stringNest <- (1, LexerStringStyle.SingleQuote, args.interpolationDelimiterLength, m2) :: args.stringNest + args.stringNest <- (1, LexerStringStyle.SingleQuote, args.interpolationDelimiterLength, None, m2) :: args.stringNest let cont = LexCont.Token(args.ifdefStack, args.stringNest) fin.Finish buf kind LexerStringFinisherContext.InterpolatedPart cont else @@ -1376,7 +1380,7 @@ and verbatimString (sargs: LexerStringArgs) (skip: bool) = parse if kind.IsInterpolated then // get a new range for where the fill starts let m2 = lexbuf.LexemeRange - args.stringNest <- (1, LexerStringStyle.Verbatim, args.interpolationDelimiterLength, m2) :: args.stringNest + args.stringNest <- (1, LexerStringStyle.Verbatim, args.interpolationDelimiterLength, None, m2) :: args.stringNest let cont = LexCont.Token(args.ifdefStack, args.stringNest) fin.Finish buf kind (LexerStringFinisherContext.InterpolatedPart ||| LexerStringFinisherContext.Verbatim) cont else @@ -1495,7 +1499,7 @@ and tripleQuoteString (sargs: LexerStringArgs) (skip: bool) = parse if kind.IsInterpolated then // get a new range for where the fill starts let m2 = lexbuf.LexemeRange - args.stringNest <- (1, LexerStringStyle.TripleQuote, args.interpolationDelimiterLength, m2) :: args.stringNest + args.stringNest <- (1, LexerStringStyle.TripleQuote, args.interpolationDelimiterLength, None, m2) :: args.stringNest let cont = LexCont.Token(args.ifdefStack, args.stringNest) fin.Finish buf kind (LexerStringFinisherContext.InterpolatedPart ||| LexerStringFinisherContext.TripleQuote) cont else @@ -1600,7 +1604,7 @@ and extendedInterpolatedString (sargs: LexerStringArgs) (skip: bool) = parse let maxBraces = 2 * args.interpolationDelimiterLength - 1 if numBraces > maxBraces then let m2 = lexbuf.LexemeRange - args.stringNest <- (1, LexerStringStyle.ExtendedInterpolated, args.interpolationDelimiterLength, m2) :: args.stringNest + args.stringNest <- (1, LexerStringStyle.ExtendedInterpolated, args.interpolationDelimiterLength, None, m2) :: args.stringNest let cont = LexCont.Token(args.ifdefStack, args.stringNest) fail args lexbuf (FSComp.SR.lexTooManyLBracesInTripleQuote()) @@ -1621,7 +1625,7 @@ and extendedInterpolatedString (sargs: LexerStringArgs) (skip: bool) = parse String.replicate extraBraces "{" |> addUnicodeString buf // get a new range for where the fill starts let m2 = lexbuf.LexemeRange - args.stringNest <- (1, LexerStringStyle.ExtendedInterpolated, args.interpolationDelimiterLength, m2) :: args.stringNest + args.stringNest <- (1, LexerStringStyle.ExtendedInterpolated, args.interpolationDelimiterLength, None, m2) :: args.stringNest let cont = LexCont.Token(args.ifdefStack, args.stringNest) fin.Finish buf kind (LexerStringFinisherContext.InterpolatedPart ||| LexerStringFinisherContext.TripleQuote) cont } diff --git a/src/Compiler/pars.fsy b/src/Compiler/pars.fsy index 4080fc7ec8b..36e8d0838c8 100644 --- a/src/Compiler/pars.fsy +++ b/src/Compiler/pars.fsy @@ -38,7 +38,7 @@ let parse_error_rich = Some(fun (ctxt: ParseErrorContext<_>) -> %token INTERP_STRING_BEGIN_END %token INTERP_STRING_BEGIN_PART %token INTERP_STRING_PART -%token INTERP_STRING_END +%token INTERP_STRING_END %token LBRACE RBRACE %token KEYWORD_STRING // Like __SOURCE_DIRECTORY__ @@ -6774,7 +6774,14 @@ interpolatedStringFill: interpolatedStringParts: | INTERP_STRING_END - { [ SynInterpolatedStringPart.String(fst $1, rhs parseState 1) ] } + { + let (s, altStart, _) = $1 + let mOrig = rhs parseState 1 + let m = + match altStart with + | Some r -> unionRanges r mOrig + | None -> mOrig + [ SynInterpolatedStringPart.String(s, m) ] } | INTERP_STRING_PART interpolatedStringFill interpolatedStringParts { SynInterpolatedStringPart.String(fst $1, rhs parseState 1) :: SynInterpolatedStringPart.FillExpr $2 :: $3 } diff --git a/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars.fs.bsl b/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars.fs.bsl index 8fbc4d9d76c..e59b33d951a 100644 --- a/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars.fs.bsl +++ b/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars.fs.bsl @@ -20,7 +20,7 @@ ImplFile FillExpr (Const (Int32 41, (2,21--2,23)), None); String (" = ", (2,25--2,32)); FillExpr (Const (Int32 6, (2,32--2,33)), None); - String (" * 7", (2,35--2,43))], TripleQuote, (2,8--2,43)), + String (" * 7", (2,33--2,43))], TripleQuote, (2,8--2,43)), (2,4--2,5), Yes (2,0--2,43), { LeadingKeyword = Let (2,0--2,3) InlineKeyword = None EqualsRange = Some (2,6--2,7) })], diff --git a/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs b/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs new file mode 100644 index 00000000000..d3eb8c7b5a1 --- /dev/null +++ b/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs @@ -0,0 +1,2 @@ + +$$$"""{{{5}}}""" \ No newline at end of file diff --git a/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs.bsl b/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs.bsl new file mode 100644 index 00000000000..152cb27e9b3 --- /dev/null +++ b/tests/service/data/SyntaxTree/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs.bsl @@ -0,0 +1,18 @@ +ImplFile + (ParsedImplFileInput + ("/root/String/SynExprInterpolatedStringWithTripleQuoteMultipleDollars2.fs", + false, + QualifiedNameOfFile + SynExprInterpolatedStringWithTripleQuoteMultipleDollars2, [], [], + [SynModuleOrNamespace + ([SynExprInterpolatedStringWithTripleQuoteMultipleDollars2], false, + AnonModule, + [Expr + (InterpolatedString + ([String ("", (2,0--2,9)); + FillExpr (Const (Int32 5, (2,9--2,10)), None); + String ("", (2,10--2,16))], TripleQuote, (2,0--2,16)), + (2,0--2,16))], PreXmlDocEmpty, [], None, (2,0--2,16), + { LeadingKeyword = None })], (true, true), + { ConditionalDirectives = [] + CodeComments = [] }, set [])) From 359e756eceed79804f0ce3ea1f6fb925bad631c7 Mon Sep 17 00:00:00 2001 From: dawe Date: Tue, 27 Feb 2024 11:45:05 +0100 Subject: [PATCH 2/2] add release notes entry --- docs/release-notes/.FSharp.Compiler.Service/8.0.300.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/.FSharp.Compiler.Service/8.0.300.md b/docs/release-notes/.FSharp.Compiler.Service/8.0.300.md index 80d327e5d9e..4c6a7db3979 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/8.0.300.md +++ b/docs/release-notes/.FSharp.Compiler.Service/8.0.300.md @@ -1,5 +1,6 @@ ### Fixed +* Fix wrong range start of INTERP_STRING_END. ([PR #16774](https://github.com/dotnet/fsharp/pull/16774)) * Fix missing warning for recursive calls in list comprehensions. ([PR #16652](https://github.com/dotnet/fsharp/pull/16652)) * Code generated files with > 64K methods and generated symbols crash when loaded. Use infered sequence points for debugging. ([Issue #16399](https://github.com/dotnet/fsharp/issues/16399), [#PR 16514](https://github.com/dotnet/fsharp/pull/16514)) * `nameof Module` expressions and patterns are processed to link files in `--test:GraphBasedChecking`. ([PR #16550](https://github.com/dotnet/fsharp/pull/16550), [PR #16743](https://github.com/dotnet/fsharp/pull/16743))