Fsc: add --tokenize-debug

src/Compiler/Driver/CompilerConfig.fs

@@ -323,6 +323,7 @@ type LStatus =
 type TokenizeOption =
     | AndCompile
     | Only
+    | Debug
     | Unfiltered
 
 type PackageManagerLine =

src/Compiler/Driver/CompilerConfig.fsi

@@ -158,6 +158,8 @@ type TokenizeOption =
 
     | Only
 
+    | Debug
+
     | Unfiltered
 
 type PackageManagerLine =

src/Compiler/Driver/CompilerOptions.fs

@@ -1561,6 +1561,14 @@ let internalFlags (tcConfigB: TcConfigBuilder) =
             None
         )
 
+        CompilerOption(
+            "tokenize-debug",
+            tagNone,
+            OptionUnit(fun () -> tcConfigB.tokenize <- TokenizeOption.Debug),
+            Some(InternalCommandLineOption("--tokenize-debug", rangeCmdArgs)),
+            None
+        )
+
         CompilerOption(
             "tokenize-unfiltered",
             tagNone,

src/Compiler/Driver/ParseAndCheckInputs.fs

@@ -636,7 +636,8 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, lexbuf, fileNam
                                 indentationSyntaxStatus,
                                 tcConfig.compilingFSharpCore,
                                 Lexer.token lexargs skipWhitespaceTokens,
-                                lexbuf
+                                lexbuf,
+                                tcConfig.tokenize = TokenizeOption.Debug
                             )
                             .GetToken,
                         true
@@ -646,7 +647,8 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, lexbuf, fileNam
                                 indentationSyntaxStatus,
                                 tcConfig.compilingFSharpCore,
                                 Lexer.token lexargs skipWhitespaceTokens,
-                                lexbuf
+                                lexbuf,
+                                tcConfig.tokenize = TokenizeOption.Debug
                             )
                             .GetToken,
                         false

src/Compiler/Interactive/fsi.fs

@@ -3578,7 +3578,13 @@ type FsiStdinLexerProvider
             )
 
         let tokenizer =
-            LexFilter.LexFilter(indentationSyntaxStatus, tcConfigB.compilingFSharpCore, Lexer.token lexargs skip, lexbuf)
+            LexFilter.LexFilter(
+                indentationSyntaxStatus,
+                tcConfigB.compilingFSharpCore,
+                Lexer.token lexargs skip,
+                lexbuf,
+                tcConfigB.tokenize = TokenizeOption.Debug
+            )
 
         tokenizer
 

src/Compiler/Service/FSharpCheckerResults.fs

@@ -2340,7 +2340,7 @@ module internal ParseAndCheckFile =
             )
 
         let tokenizer =
-            LexFilter.LexFilter(indentationSyntaxStatus, options.CompilingFSharpCore, Lexer.token lexargs true, lexbuf)
+            LexFilter.LexFilter(indentationSyntaxStatus, options.CompilingFSharpCore, Lexer.token lexargs true, lexbuf, false)
 
         (fun _ -> tokenizer.GetToken())
 

src/Compiler/Service/ServiceLexing.fs

@@ -1883,7 +1883,7 @@ module FSharpLexerImpl =
 
             if canUseLexFilter then
                 let lexFilter =
-                    LexFilter.LexFilter(lexargs.indentationSyntaxStatus, isCompilingFSharpCore, lexer, lexbuf)
+                    LexFilter.LexFilter(lexargs.indentationSyntaxStatus, isCompilingFSharpCore, lexer, lexbuf, false)
 
                 (fun _ -> lexFilter.GetToken())
             else

src/Compiler/SyntaxTree/LexFilter.fs

@@ -16,7 +16,7 @@ open FSharp.Compiler.ParseHelpers
 open FSharp.Compiler.Parser
 open FSharp.Compiler.UnicodeLexing
 
-let debug = false
+let forceDebug = false
 
 let stringOfPos (pos: Position) = sprintf "(%d:%d)" pos.OriginalLine pos.Column
 
@@ -565,7 +565,8 @@ type LexFilterImpl (
     indentationSyntaxStatus: IndentationAwareSyntaxStatus,
     compilingFSharpCore,
     lexer: (Lexbuf -> token),
-    lexbuf: Lexbuf
+    lexbuf: Lexbuf,
+    debug: bool
 ) =
 
     //----------------------------------------------------------------------------
@@ -2564,8 +2565,9 @@ type LexFilterImpl (
 // LexFilterImpl does the majority of the work for offsides rules and other magic.
 // LexFilter just wraps it with light post-processing that introduces a few more 'coming soon' symbols, to
 // make it easier for the parser to 'look ahead' and safely shift tokens in a number of recovery scenarios.
-type LexFilter (indentationSyntaxStatus: IndentationAwareSyntaxStatus, compilingFSharpCore, lexer, lexbuf: UnicodeLexing.Lexbuf) =
-    let inner = LexFilterImpl(indentationSyntaxStatus, compilingFSharpCore, lexer, lexbuf)
+type LexFilter (indentationSyntaxStatus: IndentationAwareSyntaxStatus, compilingFSharpCore, lexer, lexbuf: Lexbuf, debug) =
+    let debug = debug || forceDebug
+    let inner = LexFilterImpl(indentationSyntaxStatus, compilingFSharpCore, lexer, lexbuf, debug)
 
     // We don't interact with lexbuf state at all, any inserted tokens have same state/location as the real one read, so
     // we don't have to do any of the wrapped lexbuf magic that you see in LexFilterImpl.

src/Compiler/SyntaxTree/LexFilter.fsi

@@ -21,7 +21,8 @@ type LexFilter =
         indentationSyntaxStatus: IndentationAwareSyntaxStatus *
         compilingFSharpCore: bool *
         lexer: (LexBuffer<char> -> token) *
-        lexbuf: LexBuffer<char> ->
+        lexbuf: LexBuffer<char> *
+        debug: bool ->
             LexFilter
 
     /// The LexBuffer associated with the filter