From 3f0fc91f5816995031a7f16e619918ef55706b87 Mon Sep 17 00:00:00 2001 From: Eugene Auduchinok Date: Mon, 12 Jun 2023 19:21:27 +0200 Subject: [PATCH 1/2] Fsc: add '--tokenize-debug' --- src/Compiler/Driver/CompilerConfig.fs | 1 + src/Compiler/Driver/CompilerConfig.fsi | 2 ++ src/Compiler/Driver/CompilerOptions.fs | 8 ++++++++ src/Compiler/Driver/ParseAndCheckInputs.fs | 6 ++++-- src/Compiler/Interactive/fsi.fs | 2 +- src/Compiler/Service/FSharpCheckerResults.fs | 2 +- src/Compiler/Service/ServiceLexing.fs | 2 +- src/Compiler/SyntaxTree/LexFilter.fs | 10 ++++++---- src/Compiler/SyntaxTree/LexFilter.fsi | 3 ++- 9 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/Compiler/Driver/CompilerConfig.fs b/src/Compiler/Driver/CompilerConfig.fs index 117aa66391c..325b5531ea6 100644 --- a/src/Compiler/Driver/CompilerConfig.fs +++ b/src/Compiler/Driver/CompilerConfig.fs @@ -323,6 +323,7 @@ type LStatus = type TokenizeOption = | AndCompile | Only + | Debug | Unfiltered type PackageManagerLine = diff --git a/src/Compiler/Driver/CompilerConfig.fsi b/src/Compiler/Driver/CompilerConfig.fsi index 7f29c346eb8..234c2594197 100644 --- a/src/Compiler/Driver/CompilerConfig.fsi +++ b/src/Compiler/Driver/CompilerConfig.fsi @@ -158,6 +158,8 @@ type TokenizeOption = | Only + | Debug + | Unfiltered type PackageManagerLine = diff --git a/src/Compiler/Driver/CompilerOptions.fs b/src/Compiler/Driver/CompilerOptions.fs index 9996567a97a..ffaa016ba22 100644 --- a/src/Compiler/Driver/CompilerOptions.fs +++ b/src/Compiler/Driver/CompilerOptions.fs @@ -1561,6 +1561,14 @@ let internalFlags (tcConfigB: TcConfigBuilder) = None ) + CompilerOption( + "tokenize-debug", + tagNone, + OptionUnit(fun () -> tcConfigB.tokenize <- TokenizeOption.Debug), + Some(InternalCommandLineOption("--tokenize-debug", rangeCmdArgs)), + None + ) + CompilerOption( "tokenize-unfiltered", tagNone, diff --git a/src/Compiler/Driver/ParseAndCheckInputs.fs b/src/Compiler/Driver/ParseAndCheckInputs.fs index a3c02a7e9cc..dfdbdf682d4 100644 --- a/src/Compiler/Driver/ParseAndCheckInputs.fs +++ b/src/Compiler/Driver/ParseAndCheckInputs.fs @@ -636,7 +636,8 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, lexbuf, fileNam indentationSyntaxStatus, tcConfig.compilingFSharpCore, Lexer.token lexargs skipWhitespaceTokens, - lexbuf + lexbuf, + tcConfig.tokenize = TokenizeOption.Debug ) .GetToken, true @@ -646,7 +647,8 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, lexbuf, fileNam indentationSyntaxStatus, tcConfig.compilingFSharpCore, Lexer.token lexargs skipWhitespaceTokens, - lexbuf + lexbuf, + tcConfig.tokenize = TokenizeOption.Debug ) .GetToken, false diff --git a/src/Compiler/Interactive/fsi.fs b/src/Compiler/Interactive/fsi.fs index c7806622a65..a98d4c48a48 100644 --- a/src/Compiler/Interactive/fsi.fs +++ b/src/Compiler/Interactive/fsi.fs @@ -3578,7 +3578,7 @@ type FsiStdinLexerProvider ) let tokenizer = - LexFilter.LexFilter(indentationSyntaxStatus, tcConfigB.compilingFSharpCore, Lexer.token lexargs skip, lexbuf) + LexFilter.LexFilter(indentationSyntaxStatus, tcConfigB.compilingFSharpCore, Lexer.token lexargs skip, lexbuf, tcConfigB.tokenize = TokenizeOption.Debug) tokenizer diff --git a/src/Compiler/Service/FSharpCheckerResults.fs b/src/Compiler/Service/FSharpCheckerResults.fs index 1767eb06145..559a1fdbb15 100644 --- a/src/Compiler/Service/FSharpCheckerResults.fs +++ b/src/Compiler/Service/FSharpCheckerResults.fs @@ -2340,7 +2340,7 @@ module internal ParseAndCheckFile = ) let tokenizer = - LexFilter.LexFilter(indentationSyntaxStatus, options.CompilingFSharpCore, Lexer.token lexargs true, lexbuf) + LexFilter.LexFilter(indentationSyntaxStatus, options.CompilingFSharpCore, Lexer.token lexargs true, lexbuf, false) (fun _ -> tokenizer.GetToken()) diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index d1186d2557b..372aae908df 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -1883,7 +1883,7 @@ module FSharpLexerImpl = if canUseLexFilter then let lexFilter = - LexFilter.LexFilter(lexargs.indentationSyntaxStatus, isCompilingFSharpCore, lexer, lexbuf) + LexFilter.LexFilter(lexargs.indentationSyntaxStatus, isCompilingFSharpCore, lexer, lexbuf, false) (fun _ -> lexFilter.GetToken()) else diff --git a/src/Compiler/SyntaxTree/LexFilter.fs b/src/Compiler/SyntaxTree/LexFilter.fs index 270a9f3c96f..04b1bfc0c14 100644 --- a/src/Compiler/SyntaxTree/LexFilter.fs +++ b/src/Compiler/SyntaxTree/LexFilter.fs @@ -16,7 +16,7 @@ open FSharp.Compiler.ParseHelpers open FSharp.Compiler.Parser open FSharp.Compiler.UnicodeLexing -let debug = false +let forceDebug = false let stringOfPos (pos: Position) = sprintf "(%d:%d)" pos.OriginalLine pos.Column @@ -565,7 +565,8 @@ type LexFilterImpl ( indentationSyntaxStatus: IndentationAwareSyntaxStatus, compilingFSharpCore, lexer: (Lexbuf -> token), - lexbuf: Lexbuf + lexbuf: Lexbuf, + debug: bool ) = //---------------------------------------------------------------------------- @@ -2564,8 +2565,9 @@ type LexFilterImpl ( // LexFilterImpl does the majority of the work for offsides rules and other magic. // LexFilter just wraps it with light post-processing that introduces a few more 'coming soon' symbols, to // make it easier for the parser to 'look ahead' and safely shift tokens in a number of recovery scenarios. -type LexFilter (indentationSyntaxStatus: IndentationAwareSyntaxStatus, compilingFSharpCore, lexer, lexbuf: UnicodeLexing.Lexbuf) = - let inner = LexFilterImpl(indentationSyntaxStatus, compilingFSharpCore, lexer, lexbuf) +type LexFilter (indentationSyntaxStatus: IndentationAwareSyntaxStatus, compilingFSharpCore, lexer, lexbuf: Lexbuf, debug) = + let debug = debug || forceDebug + let inner = LexFilterImpl(indentationSyntaxStatus, compilingFSharpCore, lexer, lexbuf, debug) // We don't interact with lexbuf state at all, any inserted tokens have same state/location as the real one read, so // we don't have to do any of the wrapped lexbuf magic that you see in LexFilterImpl. diff --git a/src/Compiler/SyntaxTree/LexFilter.fsi b/src/Compiler/SyntaxTree/LexFilter.fsi index de98566ffe0..319fd5ecd90 100644 --- a/src/Compiler/SyntaxTree/LexFilter.fsi +++ b/src/Compiler/SyntaxTree/LexFilter.fsi @@ -21,7 +21,8 @@ type LexFilter = indentationSyntaxStatus: IndentationAwareSyntaxStatus * compilingFSharpCore: bool * lexer: (LexBuffer -> token) * - lexbuf: LexBuffer -> + lexbuf: LexBuffer * + debug: bool -> LexFilter /// The LexBuffer associated with the filter From 5f56331294cd738080c53f27e13d18626c5063f2 Mon Sep 17 00:00:00 2001 From: Eugene Auduchinok Date: Mon, 12 Jun 2023 19:53:34 +0200 Subject: [PATCH 2/2] Fantomas --- src/Compiler/Interactive/fsi.fs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Compiler/Interactive/fsi.fs b/src/Compiler/Interactive/fsi.fs index a98d4c48a48..4457ac3142d 100644 --- a/src/Compiler/Interactive/fsi.fs +++ b/src/Compiler/Interactive/fsi.fs @@ -3578,7 +3578,13 @@ type FsiStdinLexerProvider ) let tokenizer = - LexFilter.LexFilter(indentationSyntaxStatus, tcConfigB.compilingFSharpCore, Lexer.token lexargs skip, lexbuf, tcConfigB.tokenize = TokenizeOption.Debug) + LexFilter.LexFilter( + indentationSyntaxStatus, + tcConfigB.compilingFSharpCore, + Lexer.token lexargs skip, + lexbuf, + tcConfigB.tokenize = TokenizeOption.Debug + ) tokenizer