LexFilter: rework wrong indentation recovery (#15334)

* LexFilter: better debug diagnostics

* LexFilter: strict indentation for existing warnings

* Update xlf

* Signature generation: fix 'end' indentation in empty modules

* Update baselines

* Update baselines

* Try to undo range change

Author: auduchinok

src/Compiler/Checking/NicePrint.fs

@@ -2511,7 +2511,7 @@ module InferredSigPrinting =
                     else
                         // OK, this is a nested module, with indentation
                         if isEmptyL basic then 
-                            ((modNameEqualsL ^^ wordL (tagKeyword"begin")) @@* basic) @@ WordL.keywordEnd
+                            modNameEqualsL ^^ wordL (tagKeyword "begin") @@* basic @@* WordL.keywordEnd
                         else
                             modNameEqualsL @@* basic
                 layoutXmlDoc denv true mspec.XmlDoc basicL

src/Compiler/FSComp.txt

@@ -1574,6 +1574,7 @@ featureNestedCopyAndUpdate,"Nested record field copy-and-update"
 featureExtendedStringInterpolation,"Extended string interpolation similar to C# raw string literals."
 featureWarningWhenMultipleRecdTypeChoice,"Raises warnings when multiple record type matches were found during name resolution because of overlapping field names."
 featureImprovedImpliedArgumentNames,"Improved implied argument names"
+featureStrictIndentation,"Raises errors on incorrect indentation, allows better recovery and analysis during editing"
 3353,fsiInvalidDirective,"Invalid directive '#%s %s'"
 3354,tcNotAFunctionButIndexerNamedIndexingNotYetEnabled,"This value supports indexing, e.g. '%s.[index]'. The syntax '%s[index]' requires /langversion:preview. See https://aka.ms/fsharp-index-notation."
 3354,tcNotAFunctionButIndexerIndexingNotYetEnabled,"This expression supports indexing, e.g. 'expr.[index]'. The syntax 'expr[index]' requires /langversion:preview. See https://aka.ms/fsharp-index-notation."
@@ -1663,7 +1664,7 @@ reprStateMachineInvalidForm,"The state machine has an unexpected form"
 3521,tcInvalidMemberDeclNameMissingOrHasParen,"Invalid member declaration. The name of the member is missing or has parentheses."
 3522,tcAnonRecdDuplicateFieldId,"The field '%s' appears multiple times in this record expression."
 3523,tcAnonRecdTypeDuplicateFieldId,"The field '%s' appears multiple times in this anonymous record type."
-3524,parsExpectingExpressionInTuple,"Expecting expression"
+3524,parsExpectingExpression,"Expecting expression"
 3530,tcTraitIsStatic,"Trait '%s' is static"
 3531,tcTraitIsNotStatic,"Trait '%s' is not static"
 3532,tcTraitMayNotUseComplexThings,"A trait may not specify optional, in, out, ParamArray, CallerInfo or Quote arguments"

src/Compiler/Facilities/LanguageFeatures.fs

@@ -20,6 +20,7 @@ type LanguageFeature =
     | WildCardInForLoop
     | RelaxWhitespace
     | RelaxWhitespace2
+    | StrictIndentation
     | NameOf
     | ImplicitYield
     | OpenTypeDeclaration
@@ -163,6 +164,7 @@ type LanguageVersion(versionText) =
                 LanguageFeature.WarningWhenMultipleRecdTypeChoice, previewVersion
                 LanguageFeature.ImprovedImpliedArgumentNames, previewVersion
                 LanguageFeature.DiagnosticForObjInference, previewVersion
+                LanguageFeature.StrictIndentation, previewVersion
 
             ]
 
@@ -288,6 +290,7 @@ type LanguageVersion(versionText) =
         | LanguageFeature.WarningWhenMultipleRecdTypeChoice -> FSComp.SR.featureWarningWhenMultipleRecdTypeChoice ()
         | LanguageFeature.ImprovedImpliedArgumentNames -> FSComp.SR.featureImprovedImpliedArgumentNames ()
         | LanguageFeature.DiagnosticForObjInference -> FSComp.SR.featureInformationalObjInferenceDiagnostic ()
+        | LanguageFeature.StrictIndentation -> FSComp.SR.featureStrictIndentation ()
 
     /// Get a version string associated with the given feature.
     static member GetFeatureVersionString feature =

src/Compiler/Facilities/LanguageFeatures.fsi

@@ -10,6 +10,7 @@ type LanguageFeature =
     | WildCardInForLoop
     | RelaxWhitespace
     | RelaxWhitespace2
+    | StrictIndentation
     | NameOf
     | ImplicitYield
     | OpenTypeDeclaration

src/Compiler/Facilities/prim-lexing.fs

@@ -154,6 +154,8 @@ type internal Position =
     member x.ApplyLineDirective(fileIdx, line) =
         Position(fileIdx, line, x.OriginalLine, x.AbsoluteOffset, x.AbsoluteOffset)
 
+    override p.ToString() = $"({p.Line},{p.Column})"
+
     static member Empty = Position()
 
     static member FirstLine fileIdx = Position(fileIdx, 1, 0, 0, 0)

src/Compiler/SyntaxTree/LexFilter.fs

@@ -1727,7 +1727,9 @@ classDefnMembersAtLeastOne:
   | classDefnMember opt_seps classDefnMembers
      { match $1, $3 with
        | [ SynMemberDefn.Interface(members=Some []; range=m) ], nextMember :: _ ->
-           warning(IndentationProblem(FSComp.SR.lexfltTokenIsOffsideOfContextStartedEarlier(warningStringOfPos m.Start), nextMember.Range))
+           let strictIndentation = parseState.LexBuffer.SupportsFeature LanguageFeature.StrictIndentation
+           let warnF = if strictIndentation then errorR else warning 
+           warnF(IndentationProblem(FSComp.SR.lexfltTokenIsOffsideOfContextStartedEarlier(warningStringOfPos m.Start), nextMember.Range))
        | _ -> ()
        $1 @ $3 }
 
@@ -2712,8 +2714,7 @@ hardwhiteDoBinding:
   | ODO typedSequentialExprBlock hardwhiteDefnBindingsTerminator
       { let mDo = rhs parseState 1
         let _, _, mDone = $3
-        let mEnd = match mDone with Some m -> m | _ -> $2.Range
-        let mAll = unionRanges mDo mEnd
+        let mAll = unionRanges mDo $2.Range
         let seqPt = DebugPointAtBinding.NoneAtDo
         // any attributes prior to the 'let' are left free, e.g. become top-level attributes
         // associated with the module, 'main' function or assembly depending on their target
@@ -3531,6 +3532,11 @@ typedSequentialExprBlock:
       { if not $3 then reportParseErrorAt (rhs parseState 3) (FSComp.SR.parsUnexpectedEndOfFileExpression())
         exprFromParseError $2 }
 
+  | OBLOCKBEGIN oblockend
+      { let m = rhs parseState 1
+        reportParseErrorAt (rhs parseState 2) (FSComp.SR.parsExpectingExpression ())
+        arbExpr ("typedSequentialExprBlock1", m.EndRange) }
+
   | typedSequentialExpr
       { $1 }
 
@@ -3539,6 +3545,11 @@ declExprBlock:
   | OBLOCKBEGIN typedSequentialExpr oblockend
       { $2 }
 
+  | OBLOCKBEGIN oblockend
+      { let m = rhs parseState 1
+        reportParseErrorAt (rhs parseState 2) (FSComp.SR.parsExpectingExpression ())
+        arbExpr ("declExprBlock1", m.EndRange) }
+
   | declExpr
       { $1 }
 
@@ -3552,7 +3563,7 @@ typedSequentialExprBlockR:
     { fun _ -> $1 }
 
   | recover
-    { fun mStart -> arbExpr ("typedSequentialExprBlockR", mStart) }
+    { fun (mStart: range) -> arbExpr ("typedSequentialExprBlockR1", mStart.EndRange) }
 
 typedSequentialExpr:
   | sequentialExpr COLON typeWithTypeConstraints
@@ -3656,7 +3667,7 @@ declExpr:
 
   | hardwhiteDoBinding %prec expr_let
      { let (BindingSetPreAttrs(_, _, _, _, m)), e = $1
-       SynExpr.Do(e, m) }
+       SynExpr.Do(e, unionRanges (rhs parseState 1).StartRange e.Range) }
 
   | anonMatchingExpr %prec expr_function
       { $1 }
@@ -3720,6 +3731,42 @@ declExpr:
               WithToEndRange = mWhole }
         SynExpr.TryWith(expr, [], mWhole, spTry, spWith, trivia) }
 
+  | TRY ORIGHT_BLOCK_END %prec expr_try
+      { let mTry = rhs parseState 1
+        let mExpr = mTry.EndRange
+        let expr = arbExpr ("try1", mExpr)
+        let spTry = DebugPointAtTry.Yes mTry
+        let mWith, (clauses, mLast) = mExpr, ([], mExpr)
+        reportParseErrorAt (rhs parseState 2) (FSComp.SR.parsExpectingExpression ())
+        let spWith = DebugPointAtWith.Yes mWith
+        let mTryToWith = unionRanges mTry mWith
+        let mWithToLast = unionRanges mWith mLast
+        let mTryToLast = unionRanges mTry mLast
+        let trivia: SynExprTryWithTrivia =
+            { TryKeyword = mTry
+              TryToWithRange = mTryToWith
+              WithKeyword = mWith
+              WithToEndRange = mWithToLast }
+        SynExpr.TryWith(expr, clauses, mTryToLast, spTry, spWith, trivia) }
+
+  | TRY ORIGHT_BLOCK_END withClauses %prec expr_try
+      { let mTry = rhs parseState 1
+        let mExpr = mTry.EndRange
+        let expr = arbExpr ("try2", mExpr)
+        let spTry = DebugPointAtTry.Yes mTry
+        let mWith, (clauses, mLast) = $3
+        reportParseErrorAt (rhs parseState 2) (FSComp.SR.parsExpectingExpression ())
+        let spWith = DebugPointAtWith.Yes mWith
+        let mTryToWith = unionRanges mTry mWith
+        let mWithToLast = unionRanges mWith mLast
+        let mTryToLast = unionRanges mTry mLast
+        let trivia: SynExprTryWithTrivia =
+            { TryKeyword = mTry
+              TryToWithRange = mTryToWith
+              WithKeyword = mWith
+              WithToEndRange = mWithToLast }
+        SynExpr.TryWith(expr, clauses, mTryToLast, spTry, spWith, trivia) }
+
   | TRY typedSequentialExprBlockR FINALLY typedSequentialExprBlock %prec expr_try
       { let mTry = rhs parseState 1
         let tryExpr = $2 mTry
@@ -4233,6 +4280,11 @@ withClauses:
       { if not $3 then reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnexpectedEndOfFileWith())
         rhs parseState 1, $2 }
 
+  | OWITH recover
+      { let mWith = rhs parseState 1
+        if not $2 then reportParseErrorAt mWith (FSComp.SR.parsUnexpectedEndOfFileWith ())
+        mWith, ([], mWith.EndRange) }
+
 withPatternClauses:
   | patternClauses
       { $1 None }
@@ -4338,21 +4390,13 @@ ifExprThen:
       { let mThen = rhs parseState 1
         arbExpr ("ifThen1", mThen.EndRange), mThen }
 
-  | OTHEN OBLOCKBEGIN typedSequentialExpr oblockend %prec prec_then_if
-      { $3, rhs parseState 1 }
-
-  | OTHEN OBLOCKBEGIN typedSequentialExpr recover %prec prec_then_if
-      { if not $4 then reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnexpectedEndOfFileThen())
-        exprFromParseError $3, rhs parseState 1 }
+  | OTHEN typedSequentialExprBlock %prec prec_then_if
+      { $2, rhs parseState 1 }
 
   | OTHEN recover %prec prec_then_if
       { let mThen = rhs parseState 1
         arbExpr ("ifThen2", mThen.EndRange), mThen }
 
-  | OTHEN OBLOCKBEGIN recover %prec prec_then_if
-      { let mThen = rhs parseState 1
-        arbExpr ("ifThen3", mThen.EndRange), mThen }
-
 ifExprElifs:
   | /* EMPTY */
       { None, None }
@@ -4361,14 +4405,9 @@ ifExprElifs:
       { let mElse = rhs parseState 1
         Some mElse, Some $2 }
 
-  | OELSE OBLOCKBEGIN typedSequentialExpr oblockend
-      { let mElse = rhs parseState 1
-        Some mElse, Some $3 }
-
-  | OELSE OBLOCKBEGIN typedSequentialExpr recover
+  | OELSE typedSequentialExprBlock
       { let mElse = rhs parseState 1
-        if not $4 then reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnexpectedEndOfFileElse())
-        Some mElse, Some(exprFromParseError $3) }
+        Some mElse, Some $2 }
 
   | ELIF declExpr ifExprCases
       { let mElif = rhs parseState 1
@@ -4400,15 +4439,15 @@ tupleExpr:
       { let exprs, commas = $1
         let mComma1 = rhs parseState 2
         let mComma2 = rhs parseState 3
-        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpression ())
         let expr = arbExpr ("tupleExpr2", mComma1.EndRange)
         $4 :: expr :: exprs, (mComma2 :: mComma1 :: commas) }
 
   | tupleExpr COMMA COMMA ends_coming_soon_or_recover
       { let exprs, commas = $1
         let mComma1 = rhs parseState 2
         let mComma2 = rhs parseState 3
-        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpression ())
         if not $4 then reportParseErrorAt mComma2 (FSComp.SR.parsExpectedExpressionAfterToken ())
         let expr1 = arbExpr ("tupleExpr3", mComma1.EndRange)
         let expr2 = arbExpr ("tupleExpr4", mComma2.EndRange)
@@ -4425,7 +4464,7 @@ tupleExpr:
   | declExpr COMMA COMMA ends_coming_soon_or_recover
       { let mComma1 = rhs parseState 2
         let mComma2 = rhs parseState 3
-        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpression ())
         if not $4 then reportParseErrorAt mComma2 (FSComp.SR.parsExpectedExpressionAfterToken ())
         let expr1 = arbExpr ("tupleExpr6", mComma1.EndRange)
         let expr2 = arbExpr ("tupleExpr7", mComma2.EndRange)
@@ -4434,7 +4473,7 @@ tupleExpr:
   | declExpr COMMA COMMA declExpr
       { let mComma1 = rhs parseState 2
         let mComma2 = rhs parseState 3
-        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpression ())
         let expr = arbExpr ("tupleExpr8", mComma1.EndRange)
         [$4; expr; $1], [mComma2; mComma1] }
 

src/Compiler/pars.fsy

@@ -407,6 +407,11 @@
         <target state="translated">Statické členy v rozhraních</target>
         <note />
       </trans-unit>
+      <trans-unit id="featureStrictIndentation">
+        <source>Raises errors on incorrect indentation, allows better recovery and analysis during editing</source>
+        <target state="new">Raises errors on incorrect indentation, allows better recovery and analysis during editing</target>
+        <note />
+      </trans-unit>
       <trans-unit id="featureStringInterpolation">
         <source>string interpolation</source>
         <target state="translated">interpolace řetězce</target>
@@ -792,9 +797,9 @@
         <target state="translated">Za tímto bodem byl očekáván vzor</target>
         <note />
       </trans-unit>
-      <trans-unit id="parsExpectingExpressionInTuple">
+      <trans-unit id="parsExpectingExpression">
         <source>Expecting expression</source>
-        <target state="translated">Byl očekáván výraz.</target>
+        <target state="new">Expecting expression</target>
         <note />
       </trans-unit>
       <trans-unit id="parsExpectingPatternInTuple">

src/Compiler/xlf/FSComp.txt.cs.xlf

@@ -407,6 +407,11 @@
         <target state="translated">Statische Member in Schnittstellen</target>
         <note />
       </trans-unit>
+      <trans-unit id="featureStrictIndentation">
+        <source>Raises errors on incorrect indentation, allows better recovery and analysis during editing</source>
+        <target state="new">Raises errors on incorrect indentation, allows better recovery and analysis during editing</target>
+        <note />
+      </trans-unit>
       <trans-unit id="featureStringInterpolation">
         <source>string interpolation</source>
         <target state="translated">Zeichenfolgeninterpolation</target>
@@ -792,9 +797,9 @@
         <target state="translated">Nach diesem Punkt wurde ein Muster erwartet.</target>
         <note />
       </trans-unit>
-      <trans-unit id="parsExpectingExpressionInTuple">
+      <trans-unit id="parsExpectingExpression">
         <source>Expecting expression</source>
-        <target state="translated">Ausdruck wird erwartet</target>
+        <target state="new">Expecting expression</target>
         <note />
       </trans-unit>
       <trans-unit id="parsExpectingPatternInTuple">

src/Compiler/xlf/FSComp.txt.de.xlf

@@ -407,6 +407,11 @@
         <target state="translated">Miembros estáticos en interfaces</target>
         <note />
       </trans-unit>
+      <trans-unit id="featureStrictIndentation">
+        <source>Raises errors on incorrect indentation, allows better recovery and analysis during editing</source>
+        <target state="new">Raises errors on incorrect indentation, allows better recovery and analysis during editing</target>
+        <note />
+      </trans-unit>
       <trans-unit id="featureStringInterpolation">
         <source>string interpolation</source>
         <target state="translated">interpolación de cadena</target>
@@ -792,9 +797,9 @@
         <target state="translated">Se esperaba un patrón después de este punto</target>
         <note />
       </trans-unit>
-      <trans-unit id="parsExpectingExpressionInTuple">
+      <trans-unit id="parsExpectingExpression">
         <source>Expecting expression</source>
-        <target state="translated">Se espera una expresión</target>
+        <target state="new">Expecting expression</target>
         <note />
       </trans-unit>
       <trans-unit id="parsExpectingPatternInTuple">