From 6136b8b0f312f71a1eddb36f21065e6209eaf3cc Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 10 Dec 2018 19:59:45 -0800 Subject: [PATCH 01/33] Initial ISourceText implementation (does not work yet) --- src/fsharp/UnicodeLexing.fs | 9 ++ src/fsharp/UnicodeLexing.fsi | 1 + src/fsharp/service/ServiceXmlDocParser.fs | 10 ++ src/fsharp/service/ServiceXmlDocParser.fsi | 3 + src/fsharp/service/service.fs | 102 ++++++++++++++---- src/fsharp/service/service.fsi | 76 +++++++++++++ src/utils/prim-lexing.fs | 23 ++++ src/utils/prim-lexing.fsi | 24 +++++ .../Commands/HelpContextService.fs | 7 +- .../Commands/XmlDocCommandService.fs | 4 +- .../src/FSharp.Editor/Common/Extensions.fs | 40 +++++++ .../Completion/CompletionProvider.fs | 2 +- .../FSharp.Editor/Completion/SignatureHelp.fs | 2 +- .../Debugging/BreakpointResolutionService.fs | 2 +- .../Diagnostics/DocumentDiagnosticAnalyzer.fs | 4 +- .../DocumentHighlightsService.fs | 2 +- .../Formatting/BraceMatchingService.fs | 5 +- .../FSharpCheckerExtensions.fs | 21 ++-- .../LanguageService/SymbolHelpers.fs | 2 +- .../Navigation/NavigateToSearchService.fs | 2 +- .../QuickInfo/QuickInfoProvider.fs | 2 +- 21 files changed, 291 insertions(+), 52 deletions(-) diff --git a/src/fsharp/UnicodeLexing.fs b/src/fsharp/UnicodeLexing.fs index b6013c0342d..669403ccb15 100644 --- a/src/fsharp/UnicodeLexing.fs +++ b/src/fsharp/UnicodeLexing.fs @@ -19,6 +19,15 @@ let StringAsLexbuf (s:string) : Lexbuf = let FunctionAsLexbuf (bufferFiller: char[] * int * int -> int) : Lexbuf = LexBuffer<_>.FromFunction bufferFiller + +let SourceTextAsLexbuf (sourceText: ISourceText) = + LexBuffer.FromFunction(fun (chars, start, length) -> + let mutable count = 0 + for i = start to length - 1 do + chars.[count] <- sourceText.[i] + count <- count + 1 + count + ) // The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure // uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold diff --git a/src/fsharp/UnicodeLexing.fsi b/src/fsharp/UnicodeLexing.fsi index 58f6828c95e..dfc25a71515 100644 --- a/src/fsharp/UnicodeLexing.fsi +++ b/src/fsharp/UnicodeLexing.fsi @@ -9,3 +9,4 @@ type Lexbuf = LexBuffer val internal StringAsLexbuf : string -> Lexbuf val public FunctionAsLexbuf : (char [] * int * int -> int) -> Lexbuf val public UnicodeFileAsLexbuf :string * int option * (*retryLocked*) bool -> Lexbuf +val public SourceTextAsLexbuf : ISourceText -> Lexbuf diff --git a/src/fsharp/service/ServiceXmlDocParser.fs b/src/fsharp/service/ServiceXmlDocParser.fs index 0cfd5ec6b7b..853ee7b3b62 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fs +++ b/src/fsharp/service/ServiceXmlDocParser.fs @@ -2,6 +2,7 @@ namespace Microsoft.FSharp.Compiler.SourceCodeServices +open Microsoft.FSharp.Compiler open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library /// Represent an Xml documentation block in source code @@ -187,4 +188,13 @@ module XmlDocParser = /// Get the list of Xml documentation from current source code let getXmlDocables (sourceCodeOfTheFile, input) = let sourceCodeLinesOfTheFile = String.getLines sourceCodeOfTheFile + XmlDocParsing.getXmlDocablesImpl (sourceCodeLinesOfTheFile, input) + + /// Get the list of Xml documentation from current source code + let getXmlDocablesFromSourceText (sourceText: ISourceText, input) = + let sourceCodeLinesOfTheFile = + [| + for i = 0 to sourceText.Lines.Count - 1 do + yield sourceText.Lines.[i].ToString() + |] XmlDocParsing.getXmlDocablesImpl (sourceCodeLinesOfTheFile, input) \ No newline at end of file diff --git a/src/fsharp/service/ServiceXmlDocParser.fsi b/src/fsharp/service/ServiceXmlDocParser.fsi index 7ba0ae6e7ba..c754b7dcafe 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fsi +++ b/src/fsharp/service/ServiceXmlDocParser.fsi @@ -18,4 +18,7 @@ module public XmlDocComment = module public XmlDocParser = /// Get the list of Xml documentation from current source code val getXmlDocables : sourceCodeOfTheFile : string * input : Ast.ParsedInput option -> XmlDocable list + + /// Get the list of Xml documentation from current source code + val getXmlDocablesFromSourceText : ISourceText * input: Ast.ParsedInput option -> XmlDocable list \ No newline at end of file diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index 0546dc9da90..642f5a74e79 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -1467,6 +1467,25 @@ type FSharpParsingOptions = IsExe = tcConfigB.target.IsExe } +[] +type Source = + | SourceText of ISourceText + | String of string + + override this.Equals(source) = + match source with + | :? Source as source -> + match this, source with + | Source.SourceText(sourceText1), Source.SourceText(sourceText2) -> sourceText1.ContentEquals(sourceText2) + | Source.String(str1), Source.String(str2) -> str1 = str2 + | _ -> false + | _ -> false + + override this.GetHashCode() = + match this with + | Source.SourceText(sourceText1) -> sourceText1.GetHashCode() + | Source.String(str) -> str.GetHashCode() + module internal Parser = // We'll need number of lines for adjusting error messages at EOF @@ -1479,13 +1498,16 @@ module internal Parser = /// Error handler for parsing & type checking while processing a single file - type ErrorHandler(reportErrors, mainInputFileName, errorSeverityOptions: FSharpErrorSeverityOptions, source) = + type ErrorHandler(reportErrors, mainInputFileName, errorSeverityOptions: FSharpErrorSeverityOptions, source: Source) = let mutable options = errorSeverityOptions let errorsAndWarningsCollector = new ResizeArray<_>() let mutable errorCount = 0 // We'll need number of lines for adjusting error messages at EOF - let fileInfo = GetFileInfoForLastLineErrors source + let fileInfo = + match source with + | Source.SourceText(sourceText) -> (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Length) + | Source.String(str) -> GetFileInfoForLastLineErrors str // This function gets called whenever an error happens during parsing or checking let diagnosticSink sev (exn: PhasedDiagnostic) = @@ -1550,6 +1572,11 @@ module internal Parser = let addNewLine (source: string) = if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + let createLexbuf (source: Source) = + match source with + | Source.SourceText(sourceText) -> UnicodeLexing.SourceTextAsLexbuf(sourceText) + | Source.String(str) -> UnicodeLexing.StringAsLexbuf(addNewLine str) + let matchBraces(source, fileName, options: FSharpParsingOptions, userOpName: string) = let delayedLogger = CapturingErrorLogger("matchBraces") use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _ -> delayedLogger) @@ -1563,7 +1590,7 @@ module internal Parser = use _unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse let matchingBraces = new ResizeArray<_>() - Lexhelp.usingLexbufForParsing(UnicodeLexing.StringAsLexbuf(addNewLine source), fileName) (fun lexbuf -> + Lexhelp.usingLexbufForParsing(createLexbuf source, fileName) (fun lexbuf -> let errHandler = ErrorHandler(false, fileName, options.ErrorSeverityOptions, source) let lexfun = createLexerFunction fileName options lexbuf errHandler let parenTokensBalance t1 t2 = @@ -1599,7 +1626,7 @@ module internal Parser = use unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse let parseResult = - Lexhelp.usingLexbufForParsing(UnicodeLexing.StringAsLexbuf(addNewLine source), fileName) (fun lexbuf -> + Lexhelp.usingLexbufForParsing(createLexbuf source, fileName) (fun lexbuf -> let lexfun = createLexerFunction fileName options lexbuf errHandler let isLastCompiland = fileName.Equals(options.LastFileName, StringComparison.CurrentCultureIgnoreCase) || @@ -1617,7 +1644,7 @@ module internal Parser = // Type check a single file against an initial context, gleaning both errors and intellisense information. let CheckOneFile (parseResults: FSharpParseFileResults, - source: string, + source: Source, mainInputFileName: string, projectFileName: string, tcConfig: TcConfig, @@ -1720,7 +1747,11 @@ module internal Parser = tcState.NiceNameGenerator.Reset() // Typecheck the real input. - let sink = TcResultsSinkImpl(tcGlobals, source = source) + let sink = + match source with + | Source.SourceText(_) -> TcResultsSinkImpl(tcGlobals) + | Source.String(str) -> TcResultsSinkImpl(tcGlobals, source = str) + let! ct = Async.CancellationToken let! resOpt = @@ -2197,14 +2228,14 @@ module Helpers = && FSharpProjectOptions.UseSameProject(o1,o2) /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. parsing - let AreSameForParsing((fileName1: string, source1: string, options1), (fileName2, source2, options2)) = + let AreSameForParsing((fileName1: string, source1: Source, options1), (fileName2, source2, options2)) = fileName1 = fileName2 && options1 = options2 && source1 = source2 let AreSimilarForParsing((fileName1, _, _), (fileName2, _, _)) = fileName1 = fileName2 /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. checking - let AreSameForChecking3((fileName1: string, source1: string, options1: FSharpProjectOptions), (fileName2, source2, options2)) = + let AreSameForChecking3((fileName1: string, source1: Source, options1: FSharpProjectOptions), (fileName2, source2, options2)) = (fileName1 = fileName2) && FSharpProjectOptions.AreSameForChecking(options1,options2) && (source1 = source2) @@ -2324,8 +2355,7 @@ module CompileHelpers = | None -> () -type FileName = string -type Source = string +type FileName = string type FilePath = string type ProjectPath = string type FileVersion = int @@ -2517,9 +2547,9 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC member bc.ImplicitlyStartCheckProjectInBackground(options, userOpName) = if implicitlyStartBackgroundWork then - bc.CheckProjectInBackground(options, userOpName + ".ImplicitlyStartCheckProjectInBackground") + bc.CheckProjectInBackground(options, userOpName + ".ImplicitlyStartCheckProjectInBackground") - member bc.ParseFile(filename: string, source: string, options: FSharpParsingOptions, userOpName: string) = + member bc.ParseFile(filename: string, source: Source, options: FSharpParsingOptions, userOpName: string) = async { match parseCacheLock.AcquireLock(fun ltok -> parseFileCache.TryGet(ltok, (filename, source, options))) with | Some res -> return res @@ -2577,7 +2607,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC /// 7. Releases the file "lock". member private bc.CheckOneFileImpl (parseResults: FSharpParseFileResults, - source: string, + source: Source, fileName: string, options: FSharpProjectOptions, textSnapshotInfo: obj option, @@ -2778,11 +2808,11 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC /// Try to get recent approximate type check results for a file. - member bc.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, source, _userOpName: string) = + member bc.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, source: string option, _userOpName: string) = match source with | Some sourceText -> parseCacheLock.AcquireLock (fun ltok -> - match checkFileInProjectCache.TryGet(ltok,(filename,sourceText,options)) with + match checkFileInProjectCache.TryGet(ltok,(filename,Source.String(sourceText),options)) with | Some (a,b,c,_) -> Some (a,b,c) | None -> parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCachePossiblyStale.TryGet(ltok,(filename,options)))) | None -> parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCachePossiblyStale.TryGet(ltok,(filename,options))) @@ -3018,6 +3048,10 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten member ic.ReferenceResolver = legacyReferenceResolver + member ic.MatchBraces(filename, source: string, parsingOptions: FSharpParsingOptions, ?userOpName: string) = + let userOpName = defaultArg userOpName "Unknown" + ic.MatchBraces(filename, Source.String(source), parsingOptions, userOpName) + member ic.MatchBraces(filename, source, options: FSharpParsingOptions, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" async { @@ -3034,10 +3068,14 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten let argv = List.ofArray options.OtherOptions ic.GetParsingOptionsFromCommandLineArgs(sourceFiles, argv, options.UseScriptResolutionRules) - member ic.MatchBraces(filename, source, options: FSharpProjectOptions, ?userOpName: string) = + member ic.MatchBraces(filename, source: string, options: FSharpProjectOptions, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" let parsingOptions, _ = ic.GetParsingOptionsFromProjectOptions(options) - ic.MatchBraces(filename, source, parsingOptions, userOpName) + ic.MatchBraces(filename, Source.String(source), parsingOptions, userOpName) + + member ic.ParseFile(filename, source, options, ?userOpName: string) = + let userOpName = defaultArg userOpName "Unknown" + ic.ParseFile(filename, Source.String(source), options, userOpName = userOpName) member ic.ParseFile(filename, source, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" @@ -3045,11 +3083,15 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten backgroundCompiler.ParseFile(filename, source, options, userOpName) - member ic.ParseFileInProject(filename, source, options, ?userOpName: string) = + member ic.ParseFileInProject(filename, source: string, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" let parsingOptions, _ = ic.GetParsingOptionsFromProjectOptions(options) ic.ParseFile(filename, source, parsingOptions, userOpName) + member ic.ParseFileInProject(filename, source: Source, options, ?userOpName: string) = + let userOpName = defaultArg userOpName "Unknown" + ic.ParseFile(filename, source, options, userOpName) + member ic.GetBackgroundParseResultsForFileInProject (filename,options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" backgroundCompiler.GetBackgroundParseResultsForFileInProject(filename, options, userOpName) @@ -3191,18 +3233,32 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten /// parse including the reconstructed types in the file. member ic.CheckFileInProjectAllowingStaleCachedResults(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" - backgroundCompiler.CheckFileInProjectAllowingStaleCachedResults(parseResults,filename,fileVersion,source,options,textSnapshotInfo, userOpName) + backgroundCompiler.CheckFileInProjectAllowingStaleCachedResults(parseResults,filename,fileVersion,Source.String(source),options,textSnapshotInfo, userOpName) /// Typecheck a source code file, returning a handle to the results of the /// parse including the reconstructed types in the file. member ic.CheckFileInProject(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = + let textSnapshotInfo = defaultArg textSnapshotInfo null + let userOpName = defaultArg userOpName "Unknown" + ic.CheckFileInProject(parseResults, filename, fileVersion, Source.String(source), options, textSnapshotInfo, userOpName) + + /// Typecheck a source code file, returning a handle to the results of the + /// parse including the reconstructed types in the file. + member ic.CheckFileInProject(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:Source, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() backgroundCompiler.CheckFileInProject(parseResults,filename,fileVersion,source,options,textSnapshotInfo, userOpName) /// Typecheck a source code file, returning a handle to the results of the /// parse including the reconstructed types in the file. - member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = + member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = + let textSnapshotInfo = defaultArg textSnapshotInfo null + let userOpName = defaultArg userOpName "Unknown" + ic.ParseAndCheckFileInProject(filename, fileVersion, Source.String(source), options, textSnapshotInfo, userOpName) + + /// Typecheck a source code file, returning a handle to the results of the + /// parse including the reconstructed types in the file. + member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, source:Source, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() backgroundCompiler.ParseAndCheckFileInProject(filename, fileVersion, source, options, textSnapshotInfo, userOpName) @@ -3312,13 +3368,13 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten type FsiInteractiveChecker(legacyReferenceResolver, reactorOps: IReactorOperations, tcConfig: TcConfig, tcGlobals, tcImports, tcState) = let keepAssemblyContents = false - member __.ParseAndCheckInteraction (ctok, source, ?userOpName: string) = + member __.ParseAndCheckInteraction (ctok, source: string, ?userOpName: string) = async { let userOpName = defaultArg userOpName "Unknown" let filename = Path.Combine(tcConfig.implicitIncludeDir, "stdin.fsx") // Note: projectSourceFiles is only used to compute isLastCompiland, and is ignored if Build.IsScript(mainInputFileName) is true (which it is in this case). let parsingOptions = FSharpParsingOptions.FromTcConfig(tcConfig, [| filename |], true) - let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (source, filename, parsingOptions, userOpName) + let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (Source.String(source), filename, parsingOptions, userOpName) let dependencyFiles = [| |] // interactions have no dependencies let parseResults = FSharpParseFileResults(parseErrors, parseTreeOpt, parseHadErrors = anyErrors, dependencyFiles = dependencyFiles) @@ -3331,7 +3387,7 @@ type FsiInteractiveChecker(legacyReferenceResolver, reactorOps: IReactorOperatio CompileOptions.ParseCompilerOptions (ignore, fsiCompilerOptions, [ ]) let loadClosure = LoadClosure.ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, CodeContext.Editing, tcConfig.useSimpleResolution, tcConfig.useFsiAuxLib, new Lexhelp.LexResourceManager(), applyCompilerOptions, assumeDotNetFramework, tryGetMetadataSnapshot=(fun _ -> None), reduceMemoryUsage=reduceMemoryUsage) - let! tcErrors, tcFileResult = Parser.CheckOneFile(parseResults, source, filename, "project", tcConfig, tcGlobals, tcImports, tcState, Some loadClosure, backgroundDiagnostics, reactorOps, (fun () -> true), None, userOpName) + let! tcErrors, tcFileResult = Parser.CheckOneFile(parseResults, Source.String(source), filename, "project", tcConfig, tcGlobals, tcImports, tcState, Some loadClosure, backgroundDiagnostics, reactorOps, (fun () -> true), None, userOpName) return match tcFileResult with diff --git a/src/fsharp/service/service.fsi b/src/fsharp/service/service.fsi index 2d07c235614..a9edf5b7c87 100755 --- a/src/fsharp/service/service.fsi +++ b/src/fsharp/service/service.fsi @@ -306,6 +306,11 @@ type public FSharpParsingOptions = } static member Default: FSharpParsingOptions +[] +type Source = + | SourceText of ISourceText + | String of string + /// A set of information describing a project or script build configuration. type public FSharpProjectOptions = { @@ -384,6 +389,17 @@ type public FSharpChecker = /// An optional string used for tracing compiler operations associated with this request. member MatchBraces: filename: string * source: string * options: FSharpParsingOptions * ?userOpName: string -> Async<(range * range)[]> + /// + /// Parse a source code file, returning information about brace matching in the file. + /// Return an enumeration of the matching parenthetical tokens in the file. + /// + /// + /// The filename for the file, used to help caching of results. + /// The full source for the file. + /// Parsing options for the project or script. + /// An optional string used for tracing compiler operations associated with this request. + member MatchBraces: filename: string * source: Source * options: FSharpParsingOptions * ?userOpName: string -> Async<(range * range)[]> + /// /// Parse a source code file, returning information about brace matching in the file. /// Return an enumeration of the matching parenthetical tokens in the file. @@ -407,6 +423,17 @@ type public FSharpChecker = /// An optional string used for tracing compiler operations associated with this request. member ParseFile: filename: string * source: string * options: FSharpParsingOptions * ?userOpName: string -> Async + /// + /// Parse a source code file, returning a handle that can be used for obtaining navigation bar information + /// To get the full information, call 'CheckFileInProject' method on the result + /// + /// + /// The filename for the file. + /// The full source for the file. + /// Parsing options for the project or script. + /// An optional string used for tracing compiler operations associated with this request. + member ParseFile: filename: string * source: Source * options: FSharpParsingOptions * ?userOpName: string -> Async + /// /// Parse a source code file, returning a handle that can be used for obtaining navigation bar information /// To get the full information, call 'CheckFileInProject' method on the result @@ -470,6 +497,31 @@ type public FSharpChecker = /// An optional string used for tracing compiler operations associated with this request. member CheckFileInProject : parsed: FSharpParseFileResults * filename: string * fileversion: int * source: string * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + /// + /// + /// Check a source code file, returning a handle to the results + /// + /// + /// Note: all files except the one being checked are read from the FileSystem API + /// + /// + /// Return FSharpCheckFileAnswer.Aborted if a parse tree was not available. + /// + /// + /// + /// The results of ParseFile for this file. + /// The name of the file in the project whose source is being checked. + /// An integer that can be used to indicate the version of the file. This will be returned by TryGetRecentCheckResultsForFile when looking up the file. + /// The full source for the file. + /// The options for the project or script. + /// + /// An item passed back to 'hasTextChangedSinceLastTypecheck' (from some calls made on 'FSharpCheckFileResults') to help determine if + /// an approximate intellisense resolution is inaccurate because a range of text has changed. This + /// can be used to marginally increase accuracy of intellisense results in some situations. + /// + /// An optional string used for tracing compiler operations associated with this request. + member CheckFileInProject : parsed: FSharpParseFileResults * filename: string * fileversion: int * source: Source * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + /// /// /// Parse and check a source code file, returning a handle to the results @@ -494,6 +546,30 @@ type public FSharpChecker = /// An optional string used for tracing compiler operations associated with this request. member ParseAndCheckFileInProject : filename: string * fileversion: int * source: string * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + /// + /// + /// Parse and check a source code file, returning a handle to the results + /// + /// + /// Note: all files except the one being checked are read from the FileSystem API + /// + /// + /// Return FSharpCheckFileAnswer.Aborted if a parse tree was not available. + /// + /// + /// + /// The name of the file in the project whose source is being checked. + /// An integer that can be used to indicate the version of the file. This will be returned by TryGetRecentCheckResultsForFile when looking up the file. + /// The full source for the file. + /// The options for the project or script. + /// + /// An item passed back to 'hasTextChangedSinceLastTypecheck' (from some calls made on 'FSharpCheckFileResults') to help determine if + /// an approximate intellisense resolution is inaccurate because a range of text has changed. This + /// can be used to marginally increase accuracy of intellisense results in some situations. + /// + /// An optional string used for tracing compiler operations associated with this request. + member ParseAndCheckFileInProject : filename: string * fileversion: int * source: Source * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + /// /// Parse and typecheck all files in a project. /// All files are read from the FileSystem API diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 531e3e5f183..2c005ff0cda 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -2,6 +2,29 @@ #nowarn "47" // recursive initialization of LexBuffer +namespace Microsoft.FSharp.Compiler + +type ITextLine = + + abstract Length : int + + abstract TextString : string + +type ITextLineCollection = + + abstract Item : int -> ITextLine with get + + abstract Count : int + +type ISourceText = + + abstract Item : int -> char with get + + abstract Lines : ITextLineCollection + + abstract Length : int + + abstract ContentEquals : ISourceText -> bool namespace Internal.Utilities.Text.Lexing diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index 49eb2e6bc40..a6361330fa2 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -3,6 +3,30 @@ // LexBuffers are for use with automatically generated lexical analyzers, // in particular those produced by 'fslex'. +namespace Microsoft.FSharp.Compiler + +type ITextLine = + + abstract Length : int + + abstract TextString : string + +type ITextLineCollection = + + abstract Item : int -> ITextLine with get + + abstract Count : int + +type ISourceText = + + abstract Item : int -> char with get + + abstract Lines : ITextLineCollection + + abstract Length : int + + abstract ContentEquals : ISourceText -> bool + namespace Internal.Utilities.Text.Lexing open System.Collections.Generic diff --git a/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs b/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs index e7cd7306190..8f6d7109a2e 100644 --- a/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs +++ b/vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs @@ -25,7 +25,7 @@ type internal FSharpHelpContextService static let userOpName = "ImplementInterfaceCodeFix" static member GetHelpTerm(checker: FSharpChecker, sourceText : SourceText, fileName, options, span: TextSpan, tokens: List, textVersion, perfOptions) : Async = asyncMaybe { - let! _, _, check = checker.ParseAndCheckDocument(fileName, textVersion, sourceText.ToString(), options, perfOptions, userOpName = userOpName) + let! _, _, check = checker.ParseAndCheckDocument(fileName, textVersion, sourceText, options, perfOptions, userOpName = userOpName) let textLines = sourceText.Lines let lineInfo = textLines.GetLineFromPosition(span.Start) let line = lineInfo.LineNumber @@ -34,12 +34,11 @@ type internal FSharpHelpContextService let caretColumn = textLines.GetLinePosition(span.Start).Character let shouldTryToFindSurroundingIdent (token : ClassifiedSpan) = - let span = token.TextSpan - let content = sourceText.ToString().Substring(span.Start, span.End - span.Start) + let content = sourceText.GetSubText(token.TextSpan) match token.ClassificationType with | ClassificationTypeNames.Text | ClassificationTypeNames.WhiteSpace -> true - | (ClassificationTypeNames.Operator|ClassificationTypeNames.Punctuation)when content = "." -> true + | (ClassificationTypeNames.Operator|ClassificationTypeNames.Punctuation)when content.Length > 0 && content.[0] = '.' -> true | _ -> false let tokenInformation, col = diff --git a/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs b/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs index b7850948a86..53c15a06d33 100644 --- a/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs +++ b/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs @@ -68,9 +68,9 @@ type internal XmlDocCommandFilter let curLineNum = wpfTextView.Caret.Position.BufferPosition.GetContainingLine().LineNumber + 1 let! document = document.Value let! parsingOptions, _options = projectInfoManager.TryGetOptionsForEditingDocumentOrProject(document, CancellationToken.None) - let sourceText = wpfTextView.TextBuffer.CurrentSnapshot.GetText() + let! sourceText = document.GetTextAsync(CancellationToken.None) let! parsedInput = checker.ParseDocument(document, parsingOptions, sourceText, userOpName) - let xmlDocables = XmlDocParser.getXmlDocables (sourceText, Some parsedInput) + let xmlDocables = XmlDocParser.getXmlDocablesFromSourceText (sourceText.ToFSharpSourceText(), Some parsedInput) let xmlDocablesBelowThisLine = // +1 because looking below current line for e.g. a 'member' xmlDocables |> List.filter (fun (XmlDocable(line,_indent,_paramNames)) -> line = curLineNum+1) diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index d1de7e83508..487a2cfdccc 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -6,7 +6,9 @@ module internal Microsoft.VisualStudio.FSharp.Editor.Extensions open System open System.IO open Microsoft.CodeAnalysis +open Microsoft.CodeAnalysis.Text open Microsoft.CodeAnalysis.Host +open Microsoft.FSharp.Compiler open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.SourceCodeServices @@ -40,6 +42,44 @@ type Document with languageServices.GetService<'T>() |> Some +type TextLine with + + member this.ToFSharpTextLine() = + { new ITextLine with + + member __.Length = this.Span.Length + + member __.TextString = this.ToString() + } + +type TextLineCollection with + + member this.ToFSharpTextLineCollection() = + { new ITextLineCollection with + + member __.Item with get index = this.[index].ToFSharpTextLine() + + member __.Count = this.Count + } + + +type SourceText with + + member this.ToFSharpSourceText() = + { new ISourceText with + + member __.Item with get index = this.[index] + + member __.Lines = this.Lines.ToFSharpTextLineCollection() + + member __.ContentEquals(sourceText) = + match sourceText with + | :? SourceText as sourceText -> this.ContentEquals(sourceText) + | _ -> false + + member __.Length = this.Length + } + type FSharpNavigationDeclarationItem with member x.RoslynGlyph : Glyph = match x.Glyph with diff --git a/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs b/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs index 297f7c20954..0332b6f1848 100644 --- a/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs +++ b/vsintegration/src/FSharp.Editor/Completion/CompletionProvider.fs @@ -105,7 +105,7 @@ type internal FSharpCompletionProvider static member ProvideCompletionsAsyncAux(checker: FSharpChecker, sourceText: SourceText, caretPosition: int, options: FSharpProjectOptions, filePath: string, textVersionHash: int, getAllSymbols: FSharpCheckFileResults -> AssemblySymbol list, languageServicePerformanceOptions: LanguageServicePerformanceOptions, intellisenseOptions: IntelliSenseOptions) = asyncMaybe { - let! parseResults, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions, userOpName = userOpName) + let! parseResults, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions, userOpName = userOpName) let textLines = sourceText.Lines let caretLinePos = textLines.GetLinePosition(caretPosition) diff --git a/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs b/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs index 2d158e50ed2..a6ed17f6696 100644 --- a/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs +++ b/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs @@ -36,7 +36,7 @@ type internal FSharpSignatureHelpProvider // Unit-testable core routine static member internal ProvideMethodsAsyncAux(checker: FSharpChecker, documentationBuilder: IDocumentationBuilder, sourceText: SourceText, caretPosition: int, options: FSharpProjectOptions, triggerIsTypedChar: char option, filePath: string, textVersionHash: int) = async { - let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToString(), options, userOpName = userOpName) + let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, Source.SourceText(sourceText.ToFSharpSourceText()), options, userOpName = userOpName) match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> return None | FSharpCheckFileAnswer.Succeeded(checkFileResults) -> diff --git a/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs b/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs index c4fc0c4d4b0..7bb8cb93172 100644 --- a/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs +++ b/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs @@ -37,7 +37,7 @@ type internal FSharpBreakpointResolutionService else let textLineColumn = textLinePos.Character let fcsTextLineNumber = Line.fromZ textLinePos.Line // Roslyn line numbers are zero-based, FSharp.Compiler.Service line numbers are 1-based - let! parseResults = checker.ParseFile(fileName, sourceText.ToString(), parsingOptions, userOpName = userOpName) + let! parseResults = checker.ParseFile(fileName, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName = userOpName) return parseResults.ValidateBreakpointLocation(mkPos fcsTextLineNumber textLineColumn) } diff --git a/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs b/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs index bda8f5ca64f..f132717e8d9 100644 --- a/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs +++ b/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs @@ -59,12 +59,12 @@ type internal FSharpDocumentDiagnosticAnalyzer() = static member GetDiagnostics(checker: FSharpChecker, filePath: string, sourceText: SourceText, textVersionHash: int, parsingOptions: FSharpParsingOptions, options: FSharpProjectOptions, diagnosticType: DiagnosticsType) = async { - let! parseResults = checker.ParseFile(filePath, sourceText.ToString(), parsingOptions, userOpName=userOpName) + let! parseResults = checker.ParseFile(filePath, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName=userOpName) let! errors = async { match diagnosticType with | DiagnosticsType.Semantic -> - let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, sourceText.ToString(), options, userOpName=userOpName) + let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, Source.SourceText(sourceText.ToFSharpSourceText()), options, userOpName=userOpName) match checkResultsAnswer with | FSharpCheckFileAnswer.Aborted -> return [||] | FSharpCheckFileAnswer.Succeeded results -> diff --git a/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs b/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs index e9ea87fea5d..bc55ce999fa 100644 --- a/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs +++ b/vsintegration/src/FSharp.Editor/DocumentHighlights/DocumentHighlightsService.fs @@ -59,7 +59,7 @@ type internal FSharpDocumentHighlightsService [] (checkerP let textLinePos = sourceText.Lines.GetLinePosition(position) let fcsTextLineNumber = Line.fromZ textLinePos.Line let! symbol = Tokenizer.getSymbolAtPosition(documentKey, sourceText, position, filePath, defines, SymbolLookupKind.Greedy, false) - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions, userOpName = userOpName) + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions, userOpName = userOpName) let! symbolUse = checkFileResults.GetSymbolUseAtLocation(fcsTextLineNumber, symbol.Ident.idRange.EndColumn, textLine.ToString(), symbol.FullIsland, userOpName=userOpName) let! symbolUses = checkFileResults.GetUsesOfSymbolInFile(symbolUse.Symbol) |> liftAsync return diff --git a/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs b/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs index 77dc1ca35e1..bb48af701b0 100644 --- a/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs @@ -3,6 +3,7 @@ namespace Microsoft.VisualStudio.FSharp.Editor open System.ComponentModel.Composition +open Microsoft.CodeAnalysis.Text open Microsoft.CodeAnalysis.Editor open Microsoft.FSharp.Compiler.SourceCodeServices open System.Runtime.InteropServices @@ -17,9 +18,9 @@ type internal FSharpBraceMatchingService static let defaultUserOpName = "BraceMatching" - static member GetBraceMatchingResult(checker: FSharpChecker, sourceText, fileName, parsingOptions: FSharpParsingOptions, position: int, userOpName: string, [] forFormatting: bool) = + static member GetBraceMatchingResult(checker: FSharpChecker, sourceText: SourceText, fileName, parsingOptions: FSharpParsingOptions, position: int, userOpName: string, [] forFormatting: bool) = async { - let! matchedBraces = checker.MatchBraces(fileName, sourceText.ToString(), parsingOptions, userOpName) + let! matchedBraces = checker.MatchBraces(fileName, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName) let isPositionInRange range = match RoslynHelpers.TryFSharpRangeToTextSpan(sourceText, range) with | None -> false diff --git a/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs b/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs index c191dee6fb9..2412b833671 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs @@ -10,20 +10,17 @@ open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.SourceCodeServices type FSharpChecker with - member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: string, userOpName: string) = + member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: SourceText, userOpName: string) = asyncMaybe { - let! fileParseResults = checker.ParseFile(document.FilePath, sourceText, parsingOptions, userOpName=userOpName) |> liftAsync + let! fileParseResults = checker.ParseFile(document.FilePath, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName=userOpName) |> liftAsync return! fileParseResults.ParseTree } - member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: SourceText, userOpName: string) = - checker.ParseDocument(document, parsingOptions, sourceText=sourceText.ToString(), userOpName=userOpName) - - member checker.ParseAndCheckDocument(filePath: string, textVersionHash: int, sourceText: string, options: FSharpProjectOptions, languageServicePerformanceOptions: LanguageServicePerformanceOptions, userOpName: string) = + member checker.ParseAndCheckDocument(filePath: string, textVersionHash: int, sourceText: SourceText, options: FSharpProjectOptions, languageServicePerformanceOptions: LanguageServicePerformanceOptions, userOpName: string) = async { let parseAndCheckFile = async { - let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText, options, userOpName=userOpName) + let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, Source.SourceText(sourceText.ToFSharpSourceText()), options, userOpName=userOpName) return match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> @@ -82,21 +79,21 @@ type FSharpChecker with match allowStaleResults with | Some b -> { document.FSharpOptions.LanguageServicePerformance with AllowStaleCompletionResults = b } | _ -> document.FSharpOptions.LanguageServicePerformance - return! checker.ParseAndCheckDocument(document.FilePath, textVersion.GetHashCode(), sourceText.ToString(), options, perfOpts, userOpName=userOpName) + return! checker.ParseAndCheckDocument(document.FilePath, textVersion.GetHashCode(), sourceText, options, perfOpts, userOpName=userOpName) } - member checker.TryParseAndCheckFileInProject (projectOptions, fileName, source, userOpName) = async { - let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0, source,projectOptions, userOpName=userOpName) + member checker.TryParseAndCheckFileInProject (projectOptions, fileName, sourceText: SourceText, userOpName) = async { + let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0, Source.SourceText(sourceText.ToFSharpSourceText()),projectOptions, userOpName=userOpName) match checkAnswer with | FSharpCheckFileAnswer.Aborted -> return None | FSharpCheckFileAnswer.Succeeded checkResults -> return Some (parseResults,checkResults) } - member checker.GetAllUsesOfAllSymbolsInSourceString (projectOptions, fileName, source: string, checkForUnusedOpens, userOpName) = async { + member checker.GetAllUsesOfAllSymbolsInSourceString (projectOptions, fileName, sourceText: SourceText, checkForUnusedOpens, userOpName) = async { - let! parseAndCheckResults = checker.TryParseAndCheckFileInProject (projectOptions, fileName, source, userOpName=userOpName) + let! parseAndCheckResults = checker.TryParseAndCheckFileInProject (projectOptions, fileName, sourceText, userOpName=userOpName) match parseAndCheckResults with | None -> return [||] | Some(_parseResults,checkResults) -> diff --git a/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs b/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs index 00abd4c309c..31366d930f9 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/SymbolHelpers.fs @@ -34,7 +34,7 @@ module internal SymbolHelpers = let defines = CompilerEnvironment.GetCompilationDefinesForEditing parsingOptions let! symbol = Tokenizer.getSymbolAtPosition(document.Id, sourceText, position, document.FilePath, defines, SymbolLookupKind.Greedy, false) let settings = document.FSharpOptions - let! _, _, checkFileResults = checker.ParseAndCheckDocument(document.FilePath, textVersionHash, sourceText.ToString(), projectOptions, settings.LanguageServicePerformance, userOpName = userOpName) + let! _, _, checkFileResults = checker.ParseAndCheckDocument(document.FilePath, textVersionHash, sourceText, projectOptions, settings.LanguageServicePerformance, userOpName = userOpName) let! symbolUse = checkFileResults.GetSymbolUseAtLocation(fcsTextLineNumber, symbol.Ident.idRange.EndColumn, textLine.ToString(), symbol.FullIsland, userOpName=userOpName) let! symbolUses = checkFileResults.GetUsesOfSymbolInFile(symbolUse.Symbol) |> liftAsync return symbolUses diff --git a/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs b/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs index eedcf57b007..fe256e3a8ef 100644 --- a/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs +++ b/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs @@ -199,7 +199,7 @@ type internal FSharpNavigateToSearchService async { let! cancellationToken = Async.CancellationToken let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask - let! parseResults = checkerProvider.Checker.ParseFile(document.FilePath, sourceText.ToString(), parsingOptions) + let! parseResults = checkerProvider.Checker.ParseFile(document.FilePath, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions) let navItems parsedInput = NavigateTo.getNavigableItems parsedInput diff --git a/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs b/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs index 5547aa177aa..6758584d51a 100644 --- a/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs +++ b/vsintegration/src/FSharp.Editor/QuickInfo/QuickInfoProvider.fs @@ -169,7 +169,7 @@ type internal FSharpAsyncQuickInfoSource // test helper static member ProvideQuickInfo(checker:FSharpChecker, documentId:DocumentId, sourceText:SourceText, filePath:string, position:int, parsingOptions:FSharpParsingOptions, options:FSharpProjectOptions, textVersionHash:int, languageServicePerformanceOptions: LanguageServicePerformanceOptions) = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions, userOpName=FSharpQuickInfo.userOpName) + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions, userOpName=FSharpQuickInfo.userOpName) let textLine = sourceText.Lines.GetLineFromPosition position let textLineNumber = textLine.LineNumber + 1 // Roslyn line numbers are zero-based let defines = CompilerEnvironment.GetCompilationDefinesForEditing parsingOptions From a62601d9885ccf1afaa58e28472b33d728b2b8ef Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 10 Dec 2018 20:25:56 -0800 Subject: [PATCH 02/33] Lexbuffer works --- src/fsharp/UnicodeLexing.fs | 17 ++++++++++++----- src/utils/prim-lexing.fs | 2 ++ src/utils/prim-lexing.fsi | 2 ++ .../src/FSharp.Editor/Common/Extensions.fs | 3 +++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/fsharp/UnicodeLexing.fs b/src/fsharp/UnicodeLexing.fs index 669403ccb15..3d8f9ad79bb 100644 --- a/src/fsharp/UnicodeLexing.fs +++ b/src/fsharp/UnicodeLexing.fs @@ -21,12 +21,19 @@ let FunctionAsLexbuf (bufferFiller: char[] * int * int -> int) : Lexbuf = LexBuffer<_>.FromFunction bufferFiller let SourceTextAsLexbuf (sourceText: ISourceText) = + let mutable currentSourceIndex = 0 LexBuffer.FromFunction(fun (chars, start, length) -> - let mutable count = 0 - for i = start to length - 1 do - chars.[count] <- sourceText.[i] - count <- count + 1 - count + let lengthToCopy = + if currentSourceIndex + length <= sourceText.Length then + length + else + sourceText.Length - currentSourceIndex + + if lengthToCopy = 0 then 0 + else + sourceText.CopyTo(currentSourceIndex, chars, start, lengthToCopy) + currentSourceIndex <- currentSourceIndex + lengthToCopy + lengthToCopy ) // The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 2c005ff0cda..efa0be44f5c 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -26,6 +26,8 @@ type ISourceText = abstract ContentEquals : ISourceText -> bool + abstract CopyTo : sourceIndex: int * destinationChars: char [] * destinationIndex: int * count: int -> unit + namespace Internal.Utilities.Text.Lexing open Microsoft.FSharp.Core diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index a6361330fa2..55b51c57e0a 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -27,6 +27,8 @@ type ISourceText = abstract ContentEquals : ISourceText -> bool + abstract CopyTo : sourceIndex: int * destinationChars: char [] * destinationIndex: int * count: int -> unit + namespace Internal.Utilities.Text.Lexing open System.Collections.Generic diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index 487a2cfdccc..cbe1a9a8f29 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -78,6 +78,9 @@ type SourceText with | _ -> false member __.Length = this.Length + + member __.CopyTo(sourceIndex, destinationChars, destinationIndex, count) = + this.CopyTo(sourceIndex, destinationChars, destinationIndex, count) } type FSharpNavigationDeclarationItem with From 59fd89a53519bc8dc7296637a881d7176744cf75 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 14:08:23 -0800 Subject: [PATCH 03/33] Removing Source. Now using only ISourceText. Added SourceText.ofString. --- src/fsharp/CompileOps.fs | 23 +-- src/fsharp/CompileOps.fsi | 3 +- src/fsharp/UnicodeLexing.fs | 18 +- src/fsharp/UnicodeLexing.fsi | 1 + src/fsharp/fsi/fsi.fs | 3 +- src/fsharp/service/ServiceXmlDocParser.fs | 9 +- src/fsharp/service/ServiceXmlDocParser.fsi | 5 +- src/fsharp/service/service.fs | 176 ++++++------------ src/fsharp/service/service.fsi | 101 ++-------- src/utils/prim-lexing.fs | 105 ++++++++++- src/utils/prim-lexing.fsi | 28 ++- .../Utils/LanguageServiceProfiling/Program.fs | 7 +- .../Commands/XmlDocCommandService.fs | 2 +- .../src/FSharp.Editor/Common/Extensions.fs | 20 +- .../FSharp.Editor/Completion/SignatureHelp.fs | 2 +- .../Debugging/BreakpointResolutionService.fs | 2 +- .../Diagnostics/DocumentDiagnosticAnalyzer.fs | 5 +- .../Formatting/BraceMatchingService.fs | 2 +- .../FSharpCheckerExtensions.fs | 6 +- .../FSharpProjectOptionsManager.fs | 4 +- .../Navigation/GoToDefinition.fs | 8 +- .../Navigation/NavigateToSearchService.fs | 2 +- .../BackgroundRequests.fs | 3 +- .../FSharp.LanguageService/FSharpSource.fs | 2 +- .../UnitTests/GoToDefinitionServiceTests.fs | 2 +- .../SemanticColorizationServiceTests.fs | 2 +- 26 files changed, 249 insertions(+), 292 deletions(-) diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index 200d4f5a69e..0d9cb81ff8f 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -22,7 +22,8 @@ open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Microsoft.FSharp.Compiler.AbstractIL.Extensions.ILX open Microsoft.FSharp.Compiler.AbstractIL.Diagnostics -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.AttributeChecking open Microsoft.FSharp.Compiler.ConstraintSolver @@ -5030,7 +5031,7 @@ module private ScriptPreprocessClosure = open Internal.Utilities.Text.Lexing /// Represents an input to the closure finding process - type ClosureSource = ClosureSource of filename: string * referenceRange: range * sourceText: string * parseRequired: bool + type ClosureSource = ClosureSource of filename: string * referenceRange: range * sourceText: ISourceText * parseRequired: bool /// Represents an output of the closure finding process type ClosureFile = ClosureFile of string * range * ParsedInput option * (PhasedDiagnostic * bool) list * (PhasedDiagnostic * bool) list * (string * range) list // filename, range, errors, warnings, nowarns @@ -5045,7 +5046,7 @@ module private ScriptPreprocessClosure = seen.ContainsKey(check) /// Parse a script from source. - let ParseScriptText(filename:string, source:string, tcConfig:TcConfig, codeContext, lexResourceManager:Lexhelp.LexResourceManager, errorLogger:ErrorLogger) = + let ParseScriptText(filename:string, sourceText:ISourceText, tcConfig:TcConfig, codeContext, lexResourceManager:Lexhelp.LexResourceManager, errorLogger:ErrorLogger) = // fsc.exe -- COMPILED\!INTERACTIVE // fsi.exe -- !COMPILED\INTERACTIVE @@ -5057,7 +5058,7 @@ module private ScriptPreprocessClosure = | CodeContext.CompilationAndEvaluation -> ["INTERACTIVE"] | CodeContext.Compilation -> ["COMPILED"] | CodeContext.Editing -> "EDITING" :: (if IsScript filename then ["INTERACTIVE"] else ["COMPILED"]) - let lexbuf = UnicodeLexing.StringAsLexbuf source + let lexbuf = UnicodeLexing.SourceTextAsLexbuf(sourceText) let isLastCompiland = (IsScript filename), tcConfig.target.IsExe // The root compiland is last in the list of compilands. ParseOneInputLexbuf (tcConfig, lexResourceManager, defines, lexbuf, filename, isLastCompiland, errorLogger) @@ -5094,7 +5095,7 @@ module private ScriptPreprocessClosure = | None -> new StreamReader(stream, true) | Some (n: int) -> new StreamReader(stream, Encoding.GetEncoding(n)) let source = reader.ReadToEnd() - [ClosureSource(filename, m, source, parseRequired)] + [ClosureSource(filename, m, SourceText.ofString source, parseRequired)] with e -> errorRecovery e m [] @@ -5122,7 +5123,7 @@ module private ScriptPreprocessClosure = let tcConfig = ref tcConfig let observedSources = Observed() - let rec loop (ClosureSource(filename, m, source, parseRequired)) = + let rec loop (ClosureSource(filename, m, sourceText, parseRequired)) = [ if not (observedSources.HaveSeen(filename)) then observedSources.SetSeen(filename) //printfn "visiting %s" filename @@ -5130,7 +5131,7 @@ module private ScriptPreprocessClosure = let parseResult, parseDiagnostics = let errorLogger = CapturingErrorLogger("FindClosureParse") use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _ -> errorLogger) - let result = ParseScriptText (filename, source, !tcConfig, codeContext, lexResourceManager, errorLogger) + let result = ParseScriptText (filename, sourceText, !tcConfig, codeContext, lexResourceManager, errorLogger) result, errorLogger.Diagnostics match parseResult with @@ -5230,7 +5231,7 @@ module private ScriptPreprocessClosure = result /// Given source text, find the full load closure. Used from service.fs, when editing a script file - let GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) = + let GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) = // Resolve the basic references such as FSharp.Core.dll first, before processing any #I directives in the script // // This is tries to mimic the action of running the script in F# Interactive - the initial context for scripting is created @@ -5243,7 +5244,7 @@ module private ScriptPreprocessClosure = let tcConfig = CreateScriptTextTcConfig(legacyReferenceResolver, defaultFSharpBinariesDir, filename, codeContext, useSimpleResolution, useFsiAuxLib, Some references0, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) - let closureSources = [ClosureSource(filename, range0, source, true)] + let closureSources = [ClosureSource(filename, range0, sourceText, true)] let closureFiles, tcConfig = FindClosureFiles(closureSources, tcConfig, codeContext, lexResourceManager) GetLoadClosure(ctok, filename, closureFiles, tcConfig, codeContext) @@ -5261,9 +5262,9 @@ type LoadClosure with // /// A temporary TcConfig is created along the way, is why this routine takes so many arguments. We want to be sure to use exactly the /// same arguments as the rest of the application. - static member ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename:string, source:string, codeContext, useSimpleResolution:bool, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) : LoadClosure = + static member ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename:string, sourceText:ISourceText, codeContext, useSimpleResolution:bool, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) : LoadClosure = use unwindBuildPhase = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse - ScriptPreprocessClosure.GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) + ScriptPreprocessClosure.GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) /// Analyze a set of script files and find the closure of their references. The resulting references are then added to the given TcConfig. /// Used from fsi.fs and fsc.fs, for #load and command line. diff --git a/src/fsharp/CompileOps.fsi b/src/fsharp/CompileOps.fsi index eb371fa5927..0e9f53c843e 100755 --- a/src/fsharp/CompileOps.fsi +++ b/src/fsharp/CompileOps.fsi @@ -11,6 +11,7 @@ open Microsoft.FSharp.Compiler.AbstractIL.IL open Microsoft.FSharp.Compiler.AbstractIL.ILBinaryReader open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.TypeChecker open Microsoft.FSharp.Compiler.Range open Microsoft.FSharp.Compiler.Ast @@ -796,7 +797,7 @@ type LoadClosure = // /// A temporary TcConfig is created along the way, is why this routine takes so many arguments. We want to be sure to use exactly the /// same arguments as the rest of the application. - static member ComputeClosureOfScriptText: CompilationThreadToken * legacyReferenceResolver: ReferenceResolver.Resolver * defaultFSharpBinariesDir: string * filename: string * source: string * implicitDefines:CodeContext * useSimpleResolution: bool * useFsiAuxLib: bool * lexResourceManager: Lexhelp.LexResourceManager * applyCompilerOptions: (TcConfigBuilder -> unit) * assumeDotNetFramework: bool * tryGetMetadataSnapshot: ILReaderTryGetMetadataSnapshot * reduceMemoryUsage: ReduceMemoryFlag -> LoadClosure + static member ComputeClosureOfScriptText: CompilationThreadToken * legacyReferenceResolver: ReferenceResolver.Resolver * defaultFSharpBinariesDir: string * filename: string * sourceText: ISourceText * implicitDefines:CodeContext * useSimpleResolution: bool * useFsiAuxLib: bool * lexResourceManager: Lexhelp.LexResourceManager * applyCompilerOptions: (TcConfigBuilder -> unit) * assumeDotNetFramework: bool * tryGetMetadataSnapshot: ILReaderTryGetMetadataSnapshot * reduceMemoryUsage: ReduceMemoryFlag -> LoadClosure /// Analyze a set of script files and find the closure of their references. The resulting references are then added to the given TcConfig. /// Used from fsi.fs and fsc.fs, for #load and command line. diff --git a/src/fsharp/UnicodeLexing.fs b/src/fsharp/UnicodeLexing.fs index 3d8f9ad79bb..aad63513d00 100644 --- a/src/fsharp/UnicodeLexing.fs +++ b/src/fsharp/UnicodeLexing.fs @@ -6,6 +6,7 @@ module internal Microsoft.FSharp.Compiler.UnicodeLexing // Functions for Unicode char-based lexing (new code). // +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Internal.Utilities open System.IO @@ -20,21 +21,8 @@ let StringAsLexbuf (s:string) : Lexbuf = let FunctionAsLexbuf (bufferFiller: char[] * int * int -> int) : Lexbuf = LexBuffer<_>.FromFunction bufferFiller -let SourceTextAsLexbuf (sourceText: ISourceText) = - let mutable currentSourceIndex = 0 - LexBuffer.FromFunction(fun (chars, start, length) -> - let lengthToCopy = - if currentSourceIndex + length <= sourceText.Length then - length - else - sourceText.Length - currentSourceIndex - - if lengthToCopy = 0 then 0 - else - sourceText.CopyTo(currentSourceIndex, chars, start, lengthToCopy) - currentSourceIndex <- currentSourceIndex + lengthToCopy - lengthToCopy - ) +let SourceTextAsLexbuf (sourceText) = + LexBuffer.FromSourceText(sourceText) // The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure // uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold diff --git a/src/fsharp/UnicodeLexing.fsi b/src/fsharp/UnicodeLexing.fsi index dfc25a71515..a2c2c18fed6 100644 --- a/src/fsharp/UnicodeLexing.fsi +++ b/src/fsharp/UnicodeLexing.fsi @@ -2,6 +2,7 @@ module internal Microsoft.FSharp.Compiler.UnicodeLexing +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Text open Internal.Utilities.Text.Lexing diff --git a/src/fsharp/fsi/fsi.fs b/src/fsharp/fsi/fsi.fs index b2df5102032..07ddd0e395a 100644 --- a/src/fsharp/fsi/fsi.fs +++ b/src/fsharp/fsi/fsi.fs @@ -20,6 +20,7 @@ open System.Threading open System.Reflection open System.Runtime.CompilerServices open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL open Microsoft.FSharp.Compiler.AbstractIL.Diagnostics open Microsoft.FSharp.Compiler.AbstractIL.IL @@ -2350,7 +2351,7 @@ type internal FsiInteractionProcessor let tcConfig = TcConfig.Create(tcConfigB,validate=false) let fsiInteractiveChecker = FsiInteractiveChecker(legacyReferenceResolver, checker, tcConfig, istate.tcGlobals, istate.tcImports, istate.tcState) - fsiInteractiveChecker.ParseAndCheckInteraction(ctok, text) + fsiInteractiveChecker.ParseAndCheckInteraction(ctok, SourceText.ofString text) //---------------------------------------------------------------------------- diff --git a/src/fsharp/service/ServiceXmlDocParser.fs b/src/fsharp/service/ServiceXmlDocParser.fs index 853ee7b3b62..4554db327e2 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fs +++ b/src/fsharp/service/ServiceXmlDocParser.fs @@ -3,6 +3,7 @@ namespace Microsoft.FSharp.Compiler.SourceCodeServices open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library /// Represent an Xml documentation block in source code @@ -185,16 +186,12 @@ module XmlDocComment = res module XmlDocParser = - /// Get the list of Xml documentation from current source code - let getXmlDocables (sourceCodeOfTheFile, input) = - let sourceCodeLinesOfTheFile = String.getLines sourceCodeOfTheFile - XmlDocParsing.getXmlDocablesImpl (sourceCodeLinesOfTheFile, input) /// Get the list of Xml documentation from current source code - let getXmlDocablesFromSourceText (sourceText: ISourceText, input) = + let getXmlDocables (sourceText: ISourceText, input) = let sourceCodeLinesOfTheFile = [| for i = 0 to sourceText.Lines.Count - 1 do - yield sourceText.Lines.[i].ToString() + yield sourceText.GetTextString(sourceText.Lines.[i]) |] XmlDocParsing.getXmlDocablesImpl (sourceCodeLinesOfTheFile, input) \ No newline at end of file diff --git a/src/fsharp/service/ServiceXmlDocParser.fsi b/src/fsharp/service/ServiceXmlDocParser.fsi index c754b7dcafe..80c509ce8bf 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fsi +++ b/src/fsharp/service/ServiceXmlDocParser.fsi @@ -3,6 +3,7 @@ namespace Microsoft.FSharp.Compiler.SourceCodeServices open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Range open Microsoft.FSharp.Compiler.Ast @@ -16,9 +17,7 @@ module public XmlDocComment = val isBlank : string -> int option module public XmlDocParser = - /// Get the list of Xml documentation from current source code - val getXmlDocables : sourceCodeOfTheFile : string * input : Ast.ParsedInput option -> XmlDocable list /// Get the list of Xml documentation from current source code - val getXmlDocablesFromSourceText : ISourceText * input: Ast.ParsedInput option -> XmlDocable list + val getXmlDocables : ISourceText * input: Ast.ParsedInput option -> XmlDocable list \ No newline at end of file diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index 642f5a74e79..064999d75ad 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -14,7 +14,8 @@ open System.Reflection open System.Text open Microsoft.FSharp.Core.Printf -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AbstractIL open Microsoft.FSharp.Compiler.AbstractIL.IL open Microsoft.FSharp.Compiler.AbstractIL.ILBinaryReader @@ -1467,47 +1468,16 @@ type FSharpParsingOptions = IsExe = tcConfigB.target.IsExe } -[] -type Source = - | SourceText of ISourceText - | String of string - - override this.Equals(source) = - match source with - | :? Source as source -> - match this, source with - | Source.SourceText(sourceText1), Source.SourceText(sourceText2) -> sourceText1.ContentEquals(sourceText2) - | Source.String(str1), Source.String(str2) -> str1 = str2 - | _ -> false - | _ -> false - - override this.GetHashCode() = - match this with - | Source.SourceText(sourceText1) -> sourceText1.GetHashCode() - | Source.String(str) -> str.GetHashCode() - module internal Parser = - // We'll need number of lines for adjusting error messages at EOF - let GetFileInfoForLastLineErrors (source: string) = - // number of lines in the source file - let lastLine = (source |> Seq.sumBy (fun c -> if c = '\n' then 1 else 0)) + 1 - // length of the last line - let lastLineLength = source.Length - source.LastIndexOf("\n",StringComparison.Ordinal) - 1 - lastLine, lastLineLength - - /// Error handler for parsing & type checking while processing a single file - type ErrorHandler(reportErrors, mainInputFileName, errorSeverityOptions: FSharpErrorSeverityOptions, source: Source) = + type ErrorHandler(reportErrors, mainInputFileName, errorSeverityOptions: FSharpErrorSeverityOptions, sourceText: ISourceText) = let mutable options = errorSeverityOptions let errorsAndWarningsCollector = new ResizeArray<_>() let mutable errorCount = 0 // We'll need number of lines for adjusting error messages at EOF - let fileInfo = - match source with - | Source.SourceText(sourceText) -> (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Length) - | Source.String(str) -> GetFileInfoForLastLineErrors str + let fileInfo = (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Length) // This function gets called whenever an error happens during parsing or checking let diagnosticSink sev (exn: PhasedDiagnostic) = @@ -1572,12 +1542,10 @@ module internal Parser = let addNewLine (source: string) = if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - let createLexbuf (source: Source) = - match source with - | Source.SourceText(sourceText) -> UnicodeLexing.SourceTextAsLexbuf(sourceText) - | Source.String(str) -> UnicodeLexing.StringAsLexbuf(addNewLine str) + let createLexbuf (sourceText: ISourceText) = + UnicodeLexing.SourceTextAsLexbuf(sourceText) - let matchBraces(source, fileName, options: FSharpParsingOptions, userOpName: string) = + let matchBraces(sourceText, fileName, options: FSharpParsingOptions, userOpName: string) = let delayedLogger = CapturingErrorLogger("matchBraces") use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _ -> delayedLogger) use _unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse @@ -1590,8 +1558,8 @@ module internal Parser = use _unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse let matchingBraces = new ResizeArray<_>() - Lexhelp.usingLexbufForParsing(createLexbuf source, fileName) (fun lexbuf -> - let errHandler = ErrorHandler(false, fileName, options.ErrorSeverityOptions, source) + Lexhelp.usingLexbufForParsing(createLexbuf sourceText, fileName) (fun lexbuf -> + let errHandler = ErrorHandler(false, fileName, options.ErrorSeverityOptions, sourceText) let lexfun = createLexerFunction fileName options lexbuf errHandler let parenTokensBalance t1 t2 = match t1, t2 with @@ -1619,14 +1587,14 @@ module internal Parser = matchBraces []) matchingBraces.ToArray() - let parseFile(source, fileName, options: FSharpParsingOptions, userOpName: string) = + let parseFile(sourceText: ISourceText, fileName, options: FSharpParsingOptions, userOpName: string) = Trace.TraceInformation("FCS: {0}.{1} ({2})", userOpName, "parseFile", fileName) - let errHandler = new ErrorHandler(true, fileName, options.ErrorSeverityOptions, source) + let errHandler = new ErrorHandler(true, fileName, options.ErrorSeverityOptions, sourceText) use unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _oldLogger -> errHandler.ErrorLogger) use unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse let parseResult = - Lexhelp.usingLexbufForParsing(createLexbuf source, fileName) (fun lexbuf -> + Lexhelp.usingLexbufForParsing(createLexbuf sourceText, fileName) (fun lexbuf -> let lexfun = createLexerFunction fileName options lexbuf errHandler let isLastCompiland = fileName.Equals(options.LastFileName, StringComparison.CurrentCultureIgnoreCase) || @@ -1644,7 +1612,7 @@ module internal Parser = // Type check a single file against an initial context, gleaning both errors and intellisense information. let CheckOneFile (parseResults: FSharpParseFileResults, - source: Source, + sourceText: ISourceText, mainInputFileName: string, projectFileName: string, tcConfig: TcConfig, @@ -1670,7 +1638,7 @@ module internal Parser = // Run the type checker... | Some parsedMainInput -> // Initialize the error handler - let errHandler = new ErrorHandler(true, mainInputFileName, tcConfig.errorSeverityOptions, source) + let errHandler = new ErrorHandler(true, mainInputFileName, tcConfig.errorSeverityOptions, sourceText) use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _oldLogger -> errHandler.ErrorLogger) use _unwindBP = PushThreadBuildPhaseUntilUnwind BuildPhase.TypeCheck @@ -1747,10 +1715,7 @@ module internal Parser = tcState.NiceNameGenerator.Reset() // Typecheck the real input. - let sink = - match source with - | Source.SourceText(_) -> TcResultsSinkImpl(tcGlobals) - | Source.String(str) -> TcResultsSinkImpl(tcGlobals, source = str) + let sink = TcResultsSinkImpl(tcGlobals) let! ct = Async.CancellationToken @@ -2228,17 +2193,17 @@ module Helpers = && FSharpProjectOptions.UseSameProject(o1,o2) /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. parsing - let AreSameForParsing((fileName1: string, source1: Source, options1), (fileName2, source2, options2)) = - fileName1 = fileName2 && options1 = options2 && source1 = source2 + let AreSameForParsing((fileName1: string, source1: ISourceText, options1), (fileName2, source2, options2)) = + fileName1 = fileName2 && options1 = options2 && source1.ContentEquals(source2) let AreSimilarForParsing((fileName1, _, _), (fileName2, _, _)) = fileName1 = fileName2 /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. checking - let AreSameForChecking3((fileName1: string, source1: Source, options1: FSharpProjectOptions), (fileName2, source2, options2)) = + let AreSameForChecking3((fileName1: string, source1: ISourceText, options1: FSharpProjectOptions), (fileName2, source2, options2)) = (fileName1 = fileName2) && FSharpProjectOptions.AreSameForChecking(options1,options2) - && (source1 = source2) + && source1.ContentEquals(source2) /// Determine whether two (fileName,sourceText,options) keys should be identical w.r.t. resource usage let AreSubsumable3((fileName1:string,_,o1:FSharpProjectOptions),(fileName2:string,_,o2:FSharpProjectOptions)) = @@ -2500,7 +2465,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC // Also keyed on source. This can only be out of date if the antecedent is out of date let checkFileInProjectCache = - MruCache + MruCache (keepStrongly=checkFileInProjectCacheSize, areSame=AreSameForChecking3, areSimilar=AreSubsumable3) @@ -2534,7 +2499,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC | Parser.TypeCheckAborted.Yes -> FSharpCheckFileAnswer.Aborted | Parser.TypeCheckAborted.No scope -> FSharpCheckFileAnswer.Succeeded(MakeCheckFileResults(filename, options, builder, scope, dependencyFiles, creationErrors, parseErrors, tcErrors)) - member bc.RecordTypeCheckFileInProjectResults(filename,options,parsingOptions,parseResults,fileVersion,priorTimeStamp,checkAnswer,source) = + member bc.RecordTypeCheckFileInProjectResults(filename,options,parsingOptions,parseResults,fileVersion,priorTimeStamp,checkAnswer,sourceText) = match checkAnswer with | None | Some FSharpCheckFileAnswer.Aborted -> () @@ -2542,22 +2507,22 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC foregroundTypeCheckCount <- foregroundTypeCheckCount + 1 parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCachePossiblyStale.Set(ltok, (filename,options),(parseResults,typedResults,fileVersion)) - checkFileInProjectCache.Set(ltok, (filename,source,options),(parseResults,typedResults,fileVersion,priorTimeStamp)) - parseFileCache.Set(ltok, (filename, source, parsingOptions), parseResults)) + checkFileInProjectCache.Set(ltok, (filename,sourceText,options),(parseResults,typedResults,fileVersion,priorTimeStamp)) + parseFileCache.Set(ltok, (filename, sourceText, parsingOptions), parseResults)) member bc.ImplicitlyStartCheckProjectInBackground(options, userOpName) = if implicitlyStartBackgroundWork then bc.CheckProjectInBackground(options, userOpName + ".ImplicitlyStartCheckProjectInBackground") - member bc.ParseFile(filename: string, source: Source, options: FSharpParsingOptions, userOpName: string) = + member bc.ParseFile(filename: string, sourceText: ISourceText, options: FSharpParsingOptions, userOpName: string) = async { - match parseCacheLock.AcquireLock(fun ltok -> parseFileCache.TryGet(ltok, (filename, source, options))) with + match parseCacheLock.AcquireLock(fun ltok -> parseFileCache.TryGet(ltok, (filename, sourceText, options))) with | Some res -> return res | None -> foregroundParseCount <- foregroundParseCount + 1 - let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile(source, filename, options, userOpName) + let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile(sourceText, filename, options, userOpName) let res = FSharpParseFileResults(parseErrors, parseTreeOpt, anyErrors, options.SourceFiles) - parseCacheLock.AcquireLock(fun ltok -> parseFileCache.Set(ltok, (filename, source, options), res)) + parseCacheLock.AcquireLock(fun ltok -> parseFileCache.Set(ltok, (filename, sourceText, options), res)) return res } @@ -2576,9 +2541,9 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC } ) - member bc.GetCachedCheckFileResult(builder: IncrementalBuilder,filename,source,options) = + member bc.GetCachedCheckFileResult(builder: IncrementalBuilder,filename,sourceText,options) = // Check the cache. We can only use cached results when there is no work to do to bring the background builder up-to-date - let cachedResults = parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCache.TryGet(ltok, (filename,source,options))) + let cachedResults = parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCache.TryGet(ltok, (filename,sourceText,options))) match cachedResults with // | Some (parseResults, checkResults, _, _) when builder.AreCheckResultsBeforeFileInProjectReady(filename) -> @@ -2607,7 +2572,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC /// 7. Releases the file "lock". member private bc.CheckOneFileImpl (parseResults: FSharpParseFileResults, - source: Source, + sourceText: ISourceText, fileName: string, options: FSharpProjectOptions, textSnapshotInfo: obj option, @@ -2623,7 +2588,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC let rec loop() = async { // results may appear while we were waiting for the lock, let's recheck if it's the case - let cachedResults = bc.GetCachedCheckFileResult(builder, fileName, source, options) + let cachedResults = bc.GetCachedCheckFileResult(builder, fileName, sourceText, options) match cachedResults with | Some (_, checkResults) -> return FSharpCheckFileAnswer.Succeeded checkResults @@ -2634,11 +2599,11 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC // For scripts, this will have been recorded by GetProjectOptionsFromScript. let loadClosure = scriptClosureCacheLock.AcquireLock (fun ltok -> scriptClosureCache.TryGet (ltok, options)) let! tcErrors, tcFileResult = - Parser.CheckOneFile(parseResults, source, fileName, options.ProjectFileName, tcPrior.TcConfig, tcPrior.TcGlobals, tcPrior.TcImports, + Parser.CheckOneFile(parseResults, sourceText, fileName, options.ProjectFileName, tcPrior.TcConfig, tcPrior.TcGlobals, tcPrior.TcImports, tcPrior.TcState, loadClosure, tcPrior.TcErrors, reactorOps, (fun () -> builder.IsAlive), textSnapshotInfo, userOpName) let parsingOptions = FSharpParsingOptions.FromTcConfig(tcPrior.TcConfig, Array.ofList builder.SourceFiles, options.UseScriptResolutionRules) let checkAnswer = MakeCheckFileAnswer(fileName, tcFileResult, options, builder, Array.ofList tcPrior.TcDependencyFiles, creationErrors, parseResults.Errors, tcErrors) - bc.RecordTypeCheckFileInProjectResults(fileName, options, parsingOptions, parseResults, fileVersion, tcPrior.TimeStamp, Some checkAnswer, source) + bc.RecordTypeCheckFileInProjectResults(fileName, options, parsingOptions, parseResults, fileVersion, tcPrior.TimeStamp, Some checkAnswer, sourceText) return checkAnswer finally let dummy = ref () @@ -2808,11 +2773,11 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC /// Try to get recent approximate type check results for a file. - member bc.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, source: string option, _userOpName: string) = - match source with + member bc.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, sourceText: ISourceText option, _userOpName: string) = + match sourceText with | Some sourceText -> parseCacheLock.AcquireLock (fun ltok -> - match checkFileInProjectCache.TryGet(ltok,(filename,Source.String(sourceText),options)) with + match checkFileInProjectCache.TryGet(ltok,(filename,sourceText,options)) with | Some (a,b,c,_) -> Some (a,b,c) | None -> parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCachePossiblyStale.TryGet(ltok,(filename,options)))) | None -> parseCacheLock.AcquireLock (fun ltok -> checkFileInProjectCachePossiblyStale.TryGet(ltok,(filename,options))) @@ -2859,7 +2824,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC member bc.ParseAndCheckProject(options, userOpName) = reactor.EnqueueAndAwaitOpAsync(userOpName, "ParseAndCheckProject", options.ProjectFileName, fun ctok -> bc.ParseAndCheckProjectImpl(options, ctok, userOpName)) - member bc.GetProjectOptionsFromScript(filename, source, loadedTimeStamp, otherFlags, useFsiAuxLib: bool option, assumeDotNetFramework: bool option, extraProjectInfo: obj option, optionsStamp: int64 option, userOpName) = + member bc.GetProjectOptionsFromScript(filename, sourceText, loadedTimeStamp, otherFlags, useFsiAuxLib: bool option, assumeDotNetFramework: bool option, extraProjectInfo: obj option, optionsStamp: int64 option, userOpName) = reactor.EnqueueAndAwaitOpAsync (userOpName, "GetProjectOptionsFromScript", filename, fun ctok -> cancellable { use errors = new ErrorScope() @@ -2885,7 +2850,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC let loadClosure = LoadClosure.ComputeClosureOfScriptText(ctok, legacyReferenceResolver, - defaultFSharpBinariesDir, filename, source, + defaultFSharpBinariesDir, filename, sourceText, CodeContext.Editing, useSimpleResolution, useFsiAuxLib, new Lexhelp.LexResourceManager(), applyCompilerOptions, assumeDotNetFramework, tryGetMetadataSnapshot=tryGetMetadataSnapshot, @@ -3048,10 +3013,6 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten member ic.ReferenceResolver = legacyReferenceResolver - member ic.MatchBraces(filename, source: string, parsingOptions: FSharpParsingOptions, ?userOpName: string) = - let userOpName = defaultArg userOpName "Unknown" - ic.MatchBraces(filename, Source.String(source), parsingOptions, userOpName) - member ic.MatchBraces(filename, source, options: FSharpParsingOptions, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" async { @@ -3063,34 +3024,25 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten return res } - member ic.GetParsingOptionsFromProjectOptions(options): FSharpParsingOptions * _ = - let sourceFiles = List.ofArray options.SourceFiles - let argv = List.ofArray options.OtherOptions - ic.GetParsingOptionsFromCommandLineArgs(sourceFiles, argv, options.UseScriptResolutionRules) - member ic.MatchBraces(filename, source: string, options: FSharpProjectOptions, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" let parsingOptions, _ = ic.GetParsingOptionsFromProjectOptions(options) - ic.MatchBraces(filename, Source.String(source), parsingOptions, userOpName) + ic.MatchBraces(filename, SourceText.ofString source, parsingOptions, userOpName) - member ic.ParseFile(filename, source, options, ?userOpName: string) = - let userOpName = defaultArg userOpName "Unknown" - ic.ParseFile(filename, Source.String(source), options, userOpName = userOpName) + member ic.GetParsingOptionsFromProjectOptions(options): FSharpParsingOptions * _ = + let sourceFiles = List.ofArray options.SourceFiles + let argv = List.ofArray options.OtherOptions + ic.GetParsingOptionsFromCommandLineArgs(sourceFiles, argv, options.UseScriptResolutionRules) member ic.ParseFile(filename, source, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() backgroundCompiler.ParseFile(filename, source, options, userOpName) - member ic.ParseFileInProject(filename, source: string, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" let parsingOptions, _ = ic.GetParsingOptionsFromProjectOptions(options) - ic.ParseFile(filename, source, parsingOptions, userOpName) - - member ic.ParseFileInProject(filename, source: Source, options, ?userOpName: string) = - let userOpName = defaultArg userOpName "Unknown" - ic.ParseFile(filename, source, options, userOpName) + ic.ParseFile(filename, SourceText.ofString source, parsingOptions, userOpName) member ic.GetBackgroundParseResultsForFileInProject (filename,options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" @@ -3101,9 +3053,9 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten backgroundCompiler.GetBackgroundCheckResultsForFileInProject(filename,options, userOpName) /// Try to get recent approximate type check results for a file. - member ic.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, ?source, ?userOpName: string) = + member ic.TryGetRecentCheckResultsForFile(filename: string, options:FSharpProjectOptions, ?sourceText, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" - backgroundCompiler.TryGetRecentCheckResultsForFile(filename,options,source, userOpName) + backgroundCompiler.TryGetRecentCheckResultsForFile(filename,options,sourceText, userOpName) member ic.Compile(argv: string[], ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" @@ -3233,35 +3185,21 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten /// parse including the reconstructed types in the file. member ic.CheckFileInProjectAllowingStaleCachedResults(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" - backgroundCompiler.CheckFileInProjectAllowingStaleCachedResults(parseResults,filename,fileVersion,Source.String(source),options,textSnapshotInfo, userOpName) - - /// Typecheck a source code file, returning a handle to the results of the - /// parse including the reconstructed types in the file. - member ic.CheckFileInProject(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = - let textSnapshotInfo = defaultArg textSnapshotInfo null - let userOpName = defaultArg userOpName "Unknown" - ic.CheckFileInProject(parseResults, filename, fileVersion, Source.String(source), options, textSnapshotInfo, userOpName) + backgroundCompiler.CheckFileInProjectAllowingStaleCachedResults(parseResults,filename,fileVersion,SourceText.ofString source,options,textSnapshotInfo, userOpName) /// Typecheck a source code file, returning a handle to the results of the /// parse including the reconstructed types in the file. - member ic.CheckFileInProject(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, source:Source, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = + member ic.CheckFileInProject(parseResults:FSharpParseFileResults, filename:string, fileVersion:int, sourceText:ISourceText, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() - backgroundCompiler.CheckFileInProject(parseResults,filename,fileVersion,source,options,textSnapshotInfo, userOpName) - - /// Typecheck a source code file, returning a handle to the results of the - /// parse including the reconstructed types in the file. - member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, source:string, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = - let textSnapshotInfo = defaultArg textSnapshotInfo null - let userOpName = defaultArg userOpName "Unknown" - ic.ParseAndCheckFileInProject(filename, fileVersion, Source.String(source), options, textSnapshotInfo, userOpName) + backgroundCompiler.CheckFileInProject(parseResults,filename,fileVersion,sourceText,options,textSnapshotInfo, userOpName) /// Typecheck a source code file, returning a handle to the results of the /// parse including the reconstructed types in the file. - member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, source:Source, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = + member ic.ParseAndCheckFileInProject(filename:string, fileVersion:int, sourceText:ISourceText, options:FSharpProjectOptions, ?textSnapshotInfo:obj, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() - backgroundCompiler.ParseAndCheckFileInProject(filename, fileVersion, source, options, textSnapshotInfo, userOpName) + backgroundCompiler.ParseAndCheckFileInProject(filename, fileVersion, sourceText, options, textSnapshotInfo, userOpName) member ic.ParseAndCheckProject(options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" @@ -3273,9 +3211,9 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten backgroundCompiler.KeepProjectAlive(options, userOpName) /// For a given script file, get the ProjectOptions implied by the #load closure - member ic.GetProjectOptionsFromScript(filename, source, ?loadedTimeStamp, ?otherFlags, ?useFsiAuxLib, ?assumeDotNetFramework, ?extraProjectInfo: obj, ?optionsStamp: int64, ?userOpName: string) = + member ic.GetProjectOptionsFromScript(filename, sourceText, ?loadedTimeStamp, ?otherFlags, ?useFsiAuxLib, ?assumeDotNetFramework, ?extraProjectInfo: obj, ?optionsStamp: int64, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" - backgroundCompiler.GetProjectOptionsFromScript(filename, source, loadedTimeStamp, otherFlags, useFsiAuxLib, assumeDotNetFramework, extraProjectInfo, optionsStamp, userOpName) + backgroundCompiler.GetProjectOptionsFromScript(filename, sourceText, loadedTimeStamp, otherFlags, useFsiAuxLib, assumeDotNetFramework, extraProjectInfo, optionsStamp, userOpName) member ic.GetProjectOptionsFromCommandLineArgs(projectFileName, argv, ?loadedTimeStamp, ?extraProjectInfo: obj) = let loadedTimeStamp = defaultArg loadedTimeStamp DateTime.MaxValue // Not 'now', we don't want to force reloading @@ -3368,13 +3306,13 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten type FsiInteractiveChecker(legacyReferenceResolver, reactorOps: IReactorOperations, tcConfig: TcConfig, tcGlobals, tcImports, tcState) = let keepAssemblyContents = false - member __.ParseAndCheckInteraction (ctok, source: string, ?userOpName: string) = + member __.ParseAndCheckInteraction (ctok, sourceText: ISourceText, ?userOpName: string) = async { let userOpName = defaultArg userOpName "Unknown" let filename = Path.Combine(tcConfig.implicitIncludeDir, "stdin.fsx") // Note: projectSourceFiles is only used to compute isLastCompiland, and is ignored if Build.IsScript(mainInputFileName) is true (which it is in this case). let parsingOptions = FSharpParsingOptions.FromTcConfig(tcConfig, [| filename |], true) - let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (Source.String(source), filename, parsingOptions, userOpName) + let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (sourceText, filename, parsingOptions, userOpName) let dependencyFiles = [| |] // interactions have no dependencies let parseResults = FSharpParseFileResults(parseErrors, parseTreeOpt, parseHadErrors = anyErrors, dependencyFiles = dependencyFiles) @@ -3386,8 +3324,8 @@ type FsiInteractiveChecker(legacyReferenceResolver, reactorOps: IReactorOperatio let fsiCompilerOptions = CompileOptions.GetCoreFsiCompilerOptions tcConfigB CompileOptions.ParseCompilerOptions (ignore, fsiCompilerOptions, [ ]) - let loadClosure = LoadClosure.ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, CodeContext.Editing, tcConfig.useSimpleResolution, tcConfig.useFsiAuxLib, new Lexhelp.LexResourceManager(), applyCompilerOptions, assumeDotNetFramework, tryGetMetadataSnapshot=(fun _ -> None), reduceMemoryUsage=reduceMemoryUsage) - let! tcErrors, tcFileResult = Parser.CheckOneFile(parseResults, Source.String(source), filename, "project", tcConfig, tcGlobals, tcImports, tcState, Some loadClosure, backgroundDiagnostics, reactorOps, (fun () -> true), None, userOpName) + let loadClosure = LoadClosure.ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, CodeContext.Editing, tcConfig.useSimpleResolution, tcConfig.useFsiAuxLib, new Lexhelp.LexResourceManager(), applyCompilerOptions, assumeDotNetFramework, tryGetMetadataSnapshot=(fun _ -> None), reduceMemoryUsage=reduceMemoryUsage) + let! tcErrors, tcFileResult = Parser.CheckOneFile(parseResults, sourceText, filename, "project", tcConfig, tcGlobals, tcImports, tcState, Some loadClosure, backgroundDiagnostics, reactorOps, (fun () -> true), None, userOpName) return match tcFileResult with diff --git a/src/fsharp/service/service.fsi b/src/fsharp/service/service.fsi index a9edf5b7c87..82efb201c53 100755 --- a/src/fsharp/service/service.fsi +++ b/src/fsharp/service/service.fsi @@ -14,7 +14,8 @@ open Microsoft.FSharp.Compiler.AbstractIL open Microsoft.FSharp.Compiler.AbstractIL.IL open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library open Microsoft.FSharp.Compiler.AbstractIL.ILBinaryReader -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.Driver open Microsoft.FSharp.Compiler.ErrorLogger @@ -306,11 +307,6 @@ type public FSharpParsingOptions = } static member Default: FSharpParsingOptions -[] -type Source = - | SourceText of ISourceText - | String of string - /// A set of information describing a project or script build configuration. type public FSharpProjectOptions = { @@ -384,21 +380,10 @@ type public FSharpChecker = /// /// /// The filename for the file, used to help caching of results. - /// The full source for the file. - /// Parsing options for the project or script. - /// An optional string used for tracing compiler operations associated with this request. - member MatchBraces: filename: string * source: string * options: FSharpParsingOptions * ?userOpName: string -> Async<(range * range)[]> - - /// - /// Parse a source code file, returning information about brace matching in the file. - /// Return an enumeration of the matching parenthetical tokens in the file. - /// - /// - /// The filename for the file, used to help caching of results. - /// The full source for the file. + /// The full source for the file. /// Parsing options for the project or script. /// An optional string used for tracing compiler operations associated with this request. - member MatchBraces: filename: string * source: Source * options: FSharpParsingOptions * ?userOpName: string -> Async<(range * range)[]> + member MatchBraces: filename: string * sourceText: ISourceText * options: FSharpParsingOptions * ?userOpName: string -> Async<(range * range)[]> /// /// Parse a source code file, returning information about brace matching in the file. @@ -418,21 +403,10 @@ type public FSharpChecker = /// /// /// The filename for the file. - /// The full source for the file. - /// Parsing options for the project or script. - /// An optional string used for tracing compiler operations associated with this request. - member ParseFile: filename: string * source: string * options: FSharpParsingOptions * ?userOpName: string -> Async - - /// - /// Parse a source code file, returning a handle that can be used for obtaining navigation bar information - /// To get the full information, call 'CheckFileInProject' method on the result - /// - /// - /// The filename for the file. - /// The full source for the file. + /// The full source for the file. /// Parsing options for the project or script. /// An optional string used for tracing compiler operations associated with this request. - member ParseFile: filename: string * source: Source * options: FSharpParsingOptions * ?userOpName: string -> Async + member ParseFile: filename: string * sourceText: ISourceText * options: FSharpParsingOptions * ?userOpName: string -> Async /// /// Parse a source code file, returning a handle that can be used for obtaining navigation bar information @@ -487,56 +461,7 @@ type public FSharpChecker = /// The results of ParseFile for this file. /// The name of the file in the project whose source is being checked. /// An integer that can be used to indicate the version of the file. This will be returned by TryGetRecentCheckResultsForFile when looking up the file. - /// The full source for the file. - /// The options for the project or script. - /// - /// An item passed back to 'hasTextChangedSinceLastTypecheck' (from some calls made on 'FSharpCheckFileResults') to help determine if - /// an approximate intellisense resolution is inaccurate because a range of text has changed. This - /// can be used to marginally increase accuracy of intellisense results in some situations. - /// - /// An optional string used for tracing compiler operations associated with this request. - member CheckFileInProject : parsed: FSharpParseFileResults * filename: string * fileversion: int * source: string * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async - - /// - /// - /// Check a source code file, returning a handle to the results - /// - /// - /// Note: all files except the one being checked are read from the FileSystem API - /// - /// - /// Return FSharpCheckFileAnswer.Aborted if a parse tree was not available. - /// - /// - /// - /// The results of ParseFile for this file. - /// The name of the file in the project whose source is being checked. - /// An integer that can be used to indicate the version of the file. This will be returned by TryGetRecentCheckResultsForFile when looking up the file. - /// The full source for the file. - /// The options for the project or script. - /// - /// An item passed back to 'hasTextChangedSinceLastTypecheck' (from some calls made on 'FSharpCheckFileResults') to help determine if - /// an approximate intellisense resolution is inaccurate because a range of text has changed. This - /// can be used to marginally increase accuracy of intellisense results in some situations. - /// - /// An optional string used for tracing compiler operations associated with this request. - member CheckFileInProject : parsed: FSharpParseFileResults * filename: string * fileversion: int * source: Source * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async - - /// - /// - /// Parse and check a source code file, returning a handle to the results - /// - /// - /// Note: all files except the one being checked are read from the FileSystem API - /// - /// - /// Return FSharpCheckFileAnswer.Aborted if a parse tree was not available. - /// - /// - /// - /// The name of the file in the project whose source is being checked. - /// An integer that can be used to indicate the version of the file. This will be returned by TryGetRecentCheckResultsForFile when looking up the file. - /// The full source for the file. + /// The full source for the file. /// The options for the project or script. /// /// An item passed back to 'hasTextChangedSinceLastTypecheck' (from some calls made on 'FSharpCheckFileResults') to help determine if @@ -544,7 +469,7 @@ type public FSharpChecker = /// can be used to marginally increase accuracy of intellisense results in some situations. /// /// An optional string used for tracing compiler operations associated with this request. - member ParseAndCheckFileInProject : filename: string * fileversion: int * source: string * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + member CheckFileInProject : parsed: FSharpParseFileResults * filename: string * fileversion: int * sourceText: ISourceText * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async /// /// @@ -568,7 +493,7 @@ type public FSharpChecker = /// can be used to marginally increase accuracy of intellisense results in some situations. /// /// An optional string used for tracing compiler operations associated with this request. - member ParseAndCheckFileInProject : filename: string * fileversion: int * source: Source * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async + member ParseAndCheckFileInProject : filename: string * fileversion: int * sourceText: ISourceText * options: FSharpProjectOptions * ?textSnapshotInfo: obj * ?userOpName: string -> Async /// /// Parse and typecheck all files in a project. @@ -599,7 +524,7 @@ type public FSharpChecker = /// so that an 'unload' and 'reload' action will cause the script to be considered as a new project, /// so that references are re-resolved. /// An optional string used for tracing compiler operations associated with this request. - member GetProjectOptionsFromScript : filename: string * source: string * ?loadedTimeStamp: DateTime * ?otherFlags: string[] * ?useFsiAuxLib: bool * ?assumeDotNetFramework: bool * ?extraProjectInfo: obj * ?optionsStamp: int64 * ?userOpName: string -> Async + member GetProjectOptionsFromScript : filename: string * sourceText: ISourceText * ?loadedTimeStamp: DateTime * ?otherFlags: string[] * ?useFsiAuxLib: bool * ?assumeDotNetFramework: bool * ?extraProjectInfo: obj * ?optionsStamp: int64 * ?userOpName: string -> Async /// /// Get the FSharpProjectOptions implied by a set of command line arguments. @@ -697,9 +622,9 @@ type public FSharpChecker = /// /// The filename for the file. /// The options for the project or script, used to determine active --define conditionals and other options relevant to parsing. - /// Optionally, specify source that must match the previous parse precisely. + /// Optionally, specify source that must match the previous parse precisely. /// An optional string used for tracing compiler operations associated with this request. - member TryGetRecentCheckResultsForFile : filename: string * options:FSharpProjectOptions * ?source: string * ?userOpName: string -> (FSharpParseFileResults * FSharpCheckFileResults * (*version*)int) option + member TryGetRecentCheckResultsForFile : filename: string * options:FSharpProjectOptions * ?sourceText: ISourceText * ?userOpName: string -> (FSharpParseFileResults * FSharpCheckFileResults * (*version*)int) option /// This function is called when the entire environment is known to have changed for reasons not encoded in the ProjectOptions of any project/compilation. member InvalidateAll : unit -> unit @@ -799,7 +724,7 @@ type internal FsiInteractiveChecker = internal new : ReferenceResolver.Resolver * ops: IReactorOperations * tcConfig: TcConfig * tcGlobals: TcGlobals * tcImports: TcImports * tcState: TcState -> FsiInteractiveChecker /// An optional string used for tracing compiler operations associated with this request. - member internal ParseAndCheckInteraction : CompilationThreadToken * source:string * ?userOpName: string -> Async + member internal ParseAndCheckInteraction : CompilationThreadToken * sourceText:ISourceText * ?userOpName: string -> Async /// Information about the compilation environment [] diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index efa0be44f5c..669c14b2672 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -2,17 +2,33 @@ #nowarn "47" // recursive initialization of LexBuffer -namespace Microsoft.FSharp.Compiler +namespace Microsoft.FSharp.Compiler.Text -type ITextLine = +open System +open System.IO - abstract Length : int +// This could just be a normal Span someday. +[] +type TextSpan = + + val Start : int + + val Length : int + + member this.End = this.Start + this.Length - abstract TextString : string + new (start, length) = + if start < 0 then + raise (ArgumentOutOfRangeException("start")) + + if start + length < start then + raise (ArgumentOutOfRangeException("length")) + + { Start = start; Length = length } type ITextLineCollection = - abstract Item : int -> ITextLine with get + abstract Item : int -> TextSpan with get abstract Count : int @@ -26,11 +42,70 @@ type ISourceText = abstract ContentEquals : ISourceText -> bool - abstract CopyTo : sourceIndex: int * destinationChars: char [] * destinationIndex: int * count: int -> unit + abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit + + abstract GetTextString : unit -> string + + abstract GetTextString : TextSpan -> string + +[] +type StringTextLineCollection(str: string) = + + let getLines (str: string) = + lazy + use reader = new StringReader(str) + [| + let line = ref (reader.ReadLine()) + while not (isNull !line) do + yield !line + line := reader.ReadLine() + if str.EndsWith("\n", StringComparison.Ordinal) then + // last trailing space not returned + // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak + yield String.Empty + |] + + interface ITextLineCollection with + + member __.Item with get index = TextSpan(0, (getLines str).Value.[index].Length) + + member __.Count = (getLines str).Value.Length + +[] +type StringSourceText(str: string) = + + let lines = StringTextLineCollection(str) :> ITextLineCollection + + member __.String = str + + interface ISourceText with + + member __.Item with get index = str.[index] + + member __.Lines = lines + + member __.Length = str.Length + + member this.ContentEquals(sourceText) = + match sourceText with + | :? StringSourceText as sourceText when sourceText = this || sourceText.String = str -> true + | _ -> false + + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + str.CopyTo(sourceIndex, destination, destinationIndex, count) + + member __.GetTextString() = str + + member __.GetTextString(textSpan) = str.Substring(textSpan.Start, textSpan.Length) + +module SourceText = + + let ofString str = StringSourceText(str) :> ISourceText namespace Internal.Utilities.Text.Lexing open Microsoft.FSharp.Core + open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Collections open System.Collections.Generic @@ -192,7 +267,23 @@ namespace Internal.Utilities.Text.Lexing LexBuffer<'Char>.FromArrayNoCopy buffer // Important: This method takes ownership of the array - static member FromChars (arr:char[]) = LexBuffer.FromArrayNoCopy arr + static member FromChars (arr:char[]) = LexBuffer.FromArrayNoCopy arr + + static member FromSourceText (sourceText: ISourceText) = + let mutable currentSourceIndex = 0 + LexBuffer.FromFunction(fun (chars, start, length) -> + let lengthToCopy = + if currentSourceIndex + length <= sourceText.Length then + length + else + sourceText.Length - currentSourceIndex + + if lengthToCopy <= 0 then 0 + else + sourceText.CopyTo(currentSourceIndex, chars, start, lengthToCopy) + currentSourceIndex <- currentSourceIndex + lengthToCopy + lengthToCopy + ) module GenericImplFragments = let startInterpret(lexBuffer:LexBuffer)= diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index 55b51c57e0a..60134cf42b2 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -3,17 +3,22 @@ // LexBuffers are for use with automatically generated lexical analyzers, // in particular those produced by 'fslex'. -namespace Microsoft.FSharp.Compiler +namespace Microsoft.FSharp.Compiler.Text -type ITextLine = +[] +type TextSpan = - abstract Length : int + val Start : int + + val Length : int + + member End : int - abstract TextString : string + new : start: int * length: int -> TextSpan type ITextLineCollection = - abstract Item : int -> ITextLine with get + abstract Item : int -> TextSpan with get abstract Count : int @@ -27,11 +32,20 @@ type ISourceText = abstract ContentEquals : ISourceText -> bool - abstract CopyTo : sourceIndex: int * destinationChars: char [] * destinationIndex: int * count: int -> unit + abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit + + abstract GetTextString : unit -> string + + abstract GetTextString : TextSpan -> string + +module SourceText = + + val ofString : string -> ISourceText namespace Internal.Utilities.Text.Lexing open System.Collections.Generic +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Core open Microsoft.FSharp.Control @@ -95,6 +109,8 @@ type internal LexBuffer<'Char> = static member FromChars: char[] -> LexBuffer /// Create a lex buffer that reads character or byte inputs by using the given function. static member FromFunction: ('Char[] * int * int -> int) -> LexBuffer<'Char> + /// Create a lex buffer backed by source text. + static member FromSourceText : ISourceText -> LexBuffer /// The type of tables for an unicode lexer generated by fslex.exe. [] diff --git a/vsintegration/Utils/LanguageServiceProfiling/Program.fs b/vsintegration/Utils/LanguageServiceProfiling/Program.fs index 5413e8b9094..a131a5f65ab 100644 --- a/vsintegration/Utils/LanguageServiceProfiling/Program.fs +++ b/vsintegration/Utils/LanguageServiceProfiling/Program.fs @@ -42,6 +42,7 @@ Results look like this: *) open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.SourceCodeServices open System open System.IO @@ -89,7 +90,7 @@ let main argv = async { eprintfn "ParseAndCheckFileInProject(%s)..." options.FileToCheck let sw = Stopwatch.StartNew() - let! _, answer = checker.ParseAndCheckFileInProject(options.FileToCheck, fileVersion, File.ReadAllText options.FileToCheck, options.Options) + let! _, answer = checker.ParseAndCheckFileInProject(options.FileToCheck, fileVersion, SourceText.ofString (File.ReadAllText options.FileToCheck), options.Options) match answer with | FSharpCheckFileAnswer.Aborted -> eprintfn "Abortedin %O!" sw.Elapsed @@ -110,7 +111,7 @@ let main argv = let answers = options.FilesToCheck |> List.map (fun file -> eprintfn "doing %s" file - checker.ParseAndCheckFileInProject(file, fileVersion, File.ReadAllText file, options.Options) |> Async.RunSynchronously) + checker.ParseAndCheckFileInProject(file, fileVersion, SourceText.ofString (File.ReadAllText file), options.Options) |> Async.RunSynchronously) for _,answer in answers do match answer with | FSharpCheckFileAnswer.Aborted -> @@ -157,7 +158,7 @@ let main argv = match fileResults with | Some fileResults -> let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions(options.Options) - let! parseResult = checker.ParseFile(options.FileToCheck, getFileText(), parsingOptions) + let! parseResult = checker.ParseFile(options.FileToCheck, SourceText.ofString (getFileText()), parsingOptions) for completion in options.CompletionPositions do eprintfn "querying %A %s" completion.QualifyingNames completion.PartialName let! listInfo = diff --git a/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs b/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs index 53c15a06d33..46024320ca4 100644 --- a/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs +++ b/vsintegration/src/FSharp.Editor/Commands/XmlDocCommandService.fs @@ -70,7 +70,7 @@ type internal XmlDocCommandFilter let! parsingOptions, _options = projectInfoManager.TryGetOptionsForEditingDocumentOrProject(document, CancellationToken.None) let! sourceText = document.GetTextAsync(CancellationToken.None) let! parsedInput = checker.ParseDocument(document, parsingOptions, sourceText, userOpName) - let xmlDocables = XmlDocParser.getXmlDocablesFromSourceText (sourceText.ToFSharpSourceText(), Some parsedInput) + let xmlDocables = XmlDocParser.getXmlDocables (sourceText.ToFSharpSourceText(), Some parsedInput) let xmlDocablesBelowThisLine = // +1 because looking below current line for e.g. a 'member' xmlDocables |> List.filter (fun (XmlDocable(line,_indent,_paramNames)) -> line = curLineNum+1) diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index cbe1a9a8f29..6ed75ff8dbe 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -8,7 +8,7 @@ open System.IO open Microsoft.CodeAnalysis open Microsoft.CodeAnalysis.Text open Microsoft.CodeAnalysis.Host -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.SourceCodeServices @@ -44,20 +44,14 @@ type Document with type TextLine with - member this.ToFSharpTextLine() = - { new ITextLine with - - member __.Length = this.Span.Length - - member __.TextString = this.ToString() - } + member this.ToFSharpTextSpan() = TextSpan(this.Span.Start, this.Span.End) type TextLineCollection with member this.ToFSharpTextLineCollection() = { new ITextLineCollection with - member __.Item with get index = this.[index].ToFSharpTextLine() + member __.Item with get index = this.[index].ToFSharpTextSpan() member __.Count = this.Count } @@ -79,8 +73,12 @@ type SourceText with member __.Length = this.Length - member __.CopyTo(sourceIndex, destinationChars, destinationIndex, count) = - this.CopyTo(sourceIndex, destinationChars, destinationIndex, count) + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + this.CopyTo(sourceIndex, destination, destinationIndex, count) + + member __.GetTextString() = this.ToString() + + member __.GetTextString(textSpan) = this.ToString(Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(textSpan.Start, textSpan.End)) } type FSharpNavigationDeclarationItem with diff --git a/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs b/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs index a6ed17f6696..7293b43e380 100644 --- a/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs +++ b/vsintegration/src/FSharp.Editor/Completion/SignatureHelp.fs @@ -36,7 +36,7 @@ type internal FSharpSignatureHelpProvider // Unit-testable core routine static member internal ProvideMethodsAsyncAux(checker: FSharpChecker, documentationBuilder: IDocumentationBuilder, sourceText: SourceText, caretPosition: int, options: FSharpProjectOptions, triggerIsTypedChar: char option, filePath: string, textVersionHash: int) = async { - let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, Source.SourceText(sourceText.ToFSharpSourceText()), options, userOpName = userOpName) + let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToFSharpSourceText(), options, userOpName = userOpName) match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> return None | FSharpCheckFileAnswer.Succeeded(checkFileResults) -> diff --git a/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs b/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs index 7bb8cb93172..08a72c54473 100644 --- a/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs +++ b/vsintegration/src/FSharp.Editor/Debugging/BreakpointResolutionService.fs @@ -37,7 +37,7 @@ type internal FSharpBreakpointResolutionService else let textLineColumn = textLinePos.Character let fcsTextLineNumber = Line.fromZ textLinePos.Line // Roslyn line numbers are zero-based, FSharp.Compiler.Service line numbers are 1-based - let! parseResults = checker.ParseFile(fileName, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName = userOpName) + let! parseResults = checker.ParseFile(fileName, sourceText.ToFSharpSourceText(), parsingOptions, userOpName = userOpName) return parseResults.ValidateBreakpointLocation(mkPos fcsTextLineNumber textLineColumn) } diff --git a/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs b/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs index f132717e8d9..9ef06d1081e 100644 --- a/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs +++ b/vsintegration/src/FSharp.Editor/Diagnostics/DocumentDiagnosticAnalyzer.fs @@ -59,12 +59,13 @@ type internal FSharpDocumentDiagnosticAnalyzer() = static member GetDiagnostics(checker: FSharpChecker, filePath: string, sourceText: SourceText, textVersionHash: int, parsingOptions: FSharpParsingOptions, options: FSharpProjectOptions, diagnosticType: DiagnosticsType) = async { - let! parseResults = checker.ParseFile(filePath, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName=userOpName) + let fsSourceText = sourceText.ToFSharpSourceText() + let! parseResults = checker.ParseFile(filePath, fsSourceText, parsingOptions, userOpName=userOpName) let! errors = async { match diagnosticType with | DiagnosticsType.Semantic -> - let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, Source.SourceText(sourceText.ToFSharpSourceText()), options, userOpName=userOpName) + let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, fsSourceText, options, userOpName=userOpName) match checkResultsAnswer with | FSharpCheckFileAnswer.Aborted -> return [||] | FSharpCheckFileAnswer.Succeeded results -> diff --git a/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs b/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs index bb48af701b0..5a8380d9caf 100644 --- a/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/Formatting/BraceMatchingService.fs @@ -20,7 +20,7 @@ type internal FSharpBraceMatchingService static member GetBraceMatchingResult(checker: FSharpChecker, sourceText: SourceText, fileName, parsingOptions: FSharpParsingOptions, position: int, userOpName: string, [] forFormatting: bool) = async { - let! matchedBraces = checker.MatchBraces(fileName, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName) + let! matchedBraces = checker.MatchBraces(fileName, sourceText.ToFSharpSourceText(), parsingOptions, userOpName) let isPositionInRange range = match RoslynHelpers.TryFSharpRangeToTextSpan(sourceText, range) with | None -> false diff --git a/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs b/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs index 2412b833671..1f2538ee9ee 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/FSharpCheckerExtensions.fs @@ -12,7 +12,7 @@ open Microsoft.FSharp.Compiler.SourceCodeServices type FSharpChecker with member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: SourceText, userOpName: string) = asyncMaybe { - let! fileParseResults = checker.ParseFile(document.FilePath, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions, userOpName=userOpName) |> liftAsync + let! fileParseResults = checker.ParseFile(document.FilePath, sourceText.ToFSharpSourceText(), parsingOptions, userOpName=userOpName) |> liftAsync return! fileParseResults.ParseTree } @@ -20,7 +20,7 @@ type FSharpChecker with async { let parseAndCheckFile = async { - let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, Source.SourceText(sourceText.ToFSharpSourceText()), options, userOpName=userOpName) + let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToFSharpSourceText(), options, userOpName=userOpName) return match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> @@ -84,7 +84,7 @@ type FSharpChecker with member checker.TryParseAndCheckFileInProject (projectOptions, fileName, sourceText: SourceText, userOpName) = async { - let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0, Source.SourceText(sourceText.ToFSharpSourceText()),projectOptions, userOpName=userOpName) + let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0,sourceText.ToFSharpSourceText(),projectOptions, userOpName=userOpName) match checkAnswer with | FSharpCheckFileAnswer.Aborted -> return None | FSharpCheckFileAnswer.Succeeded checkResults -> return Some (parseResults,checkResults) diff --git a/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs b/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs index 2d9280c5474..eae4f7d1a77 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs @@ -119,9 +119,9 @@ type private FSharpProjectOptionsReactor (workspace: VisualStudioWorkspaceImpl, let rec tryComputeOptionsByFile (document: Document) cancellationToken = async { - let! text = document.GetTextAsync(cancellationToken) |> Async.AwaitTask + let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask let! fileStamp = document.GetTextVersionAsync(cancellationToken) |> Async.AwaitTask - let! scriptProjectOptions, _ = checkerProvider.Checker.GetProjectOptionsFromScript(document.FilePath, text.ToString(), DateTime.Now) + let! scriptProjectOptions, _ = checkerProvider.Checker.GetProjectOptionsFromScript(document.FilePath, sourceText.ToFSharpSourceText(), DateTime.Now) match singleFileCache.TryGetValue(document.Id) with | false, _ -> let projectOptions = diff --git a/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs b/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs index de57acbb3a0..bd3eda66114 100644 --- a/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs +++ b/vsintegration/src/FSharp.Editor/Navigation/GoToDefinition.fs @@ -207,12 +207,12 @@ type internal GoToDefinition(checker: FSharpChecker, projectInfoManager: FSharpP /// if the symbol is defined in the given file, return its declaration location, otherwise use the targetSymbol to find the first /// instance of its presence in the provided source file. The first case is needed to return proper declaration location for /// recursive type definitions, where the first its usage may not be the declaration. - member __.FindSymbolDeclarationInFile(targetSymbolUse: FSharpSymbolUse, filePath: string, source: string, options: FSharpProjectOptions, fileVersion:int) = + member __.FindSymbolDeclarationInFile(targetSymbolUse: FSharpSymbolUse, filePath: string, sourceText: SourceText, options: FSharpProjectOptions, fileVersion:int) = asyncMaybe { match targetSymbolUse.Symbol.DeclarationLocation with | Some decl when decl.FileName = filePath -> return decl | _ -> - let! _, checkFileAnswer = checker.ParseAndCheckFileInProject (filePath, fileVersion, source, options, userOpName = userOpName) |> liftAsync + let! _, checkFileAnswer = checker.ParseAndCheckFileInProject (filePath, fileVersion, sourceText.ToFSharpSourceText(), options, userOpName = userOpName) |> liftAsync match checkFileAnswer with | FSharpCheckFileAnswer.Aborted -> return! None | FSharpCheckFileAnswer.Succeeded checkFileResults -> @@ -273,7 +273,7 @@ type internal GoToDefinition(checker: FSharpChecker, projectInfoManager: FSharpP let! implSourceText = implDocument.GetTextAsync () |> liftTaskAsync let! implVersion = implDocument.GetTextVersionAsync () |> liftTaskAsync - let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText.ToString(), projectOptions, implVersion.GetHashCode()) + let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText, projectOptions, implVersion.GetHashCode()) let! implTextSpan = RoslynHelpers.TryFSharpRangeToTextSpan (implSourceText, targetRange) let navItem = FSharpNavigableItem (implDocument, implTextSpan) @@ -312,7 +312,7 @@ type internal GoToDefinition(checker: FSharpChecker, projectInfoManager: FSharpP let! implSourceText = implDocument.GetTextAsync () |> liftTaskAsync let! _parsingOptions, projectOptions = projectInfoManager.TryGetOptionsForEditingDocumentOrProject(implDocument, CancellationToken.None) - let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText.ToString(), projectOptions, implVersion.GetHashCode()) + let! targetRange = this.FindSymbolDeclarationInFile(targetSymbolUse, implFilePath, implSourceText, projectOptions, implVersion.GetHashCode()) let! implTextSpan = RoslynHelpers.TryFSharpRangeToTextSpan (implSourceText, targetRange) let navItem = FSharpNavigableItem (implDocument, implTextSpan) diff --git a/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs b/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs index fe256e3a8ef..bb65cb6452e 100644 --- a/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs +++ b/vsintegration/src/FSharp.Editor/Navigation/NavigateToSearchService.fs @@ -199,7 +199,7 @@ type internal FSharpNavigateToSearchService async { let! cancellationToken = Async.CancellationToken let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask - let! parseResults = checkerProvider.Checker.ParseFile(document.FilePath, Source.SourceText(sourceText.ToFSharpSourceText()), parsingOptions) + let! parseResults = checkerProvider.Checker.ParseFile(document.FilePath, sourceText.ToFSharpSourceText(), parsingOptions) let navItems parsedInput = NavigateTo.getNavigableItems parsedInput diff --git a/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs b/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs index 42c673d90b7..ac3c054682a 100644 --- a/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs +++ b/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs @@ -12,7 +12,6 @@ open Microsoft.VisualStudio.Text open Microsoft.FSharp.Compiler.SourceCodeServices open Microsoft.VisualStudio.FSharp.LanguageService.SiteProvider - #nowarn "44" // use of obsolete CheckFileInProjectAllowingStaleCachedResults // @@ -95,7 +94,7 @@ type internal FSharpLanguageServiceBackgroundRequests_DEPRECATED lazy // This portion is executed on the language service thread let timestamp = if source=null then System.DateTime(2000,1,1) else source.OpenedTime // source is null in unit tests let checker = getInteractiveChecker() - let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(fileName, sourceText, timestamp, [| |]) |> Async.RunSynchronously + let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(fileName, Microsoft.FSharp.Compiler.Text.SourceText.ofString sourceText, timestamp, [| |]) |> Async.RunSynchronously let referencedProjectFileNames = [| |] let projectSite = ProjectSitesAndFiles.CreateProjectSiteForScript(fileName, referencedProjectFileNames, checkOptions) { ProjectSite = projectSite diff --git a/vsintegration/src/FSharp.LanguageService/FSharpSource.fs b/vsintegration/src/FSharp.LanguageService/FSharpSource.fs index 3146f5d0ba1..a818b6f3d1b 100644 --- a/vsintegration/src/FSharp.LanguageService/FSharpSource.fs +++ b/vsintegration/src/FSharp.LanguageService/FSharpSource.fs @@ -371,7 +371,7 @@ type internal FSharpSource_DEPRECATED(service:LanguageService_DEPRECATED, textLi Stamp = None } |> ic.GetParsingOptionsFromProjectOptions - ic.ParseFile(fileName, source.GetText(), co) |> Async.RunSynchronously + ic.ParseFile(fileName, Microsoft.FSharp.Compiler.Text.SourceText.ofString (source.GetText()), co) |> Async.RunSynchronously override source.GetCommentFormat() = let mutable info = new CommentInfo() diff --git a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs index a2650c519e4..d442b9a8963 100644 --- a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs +++ b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs @@ -53,7 +53,7 @@ module GoToDefinitionServiceTests = let textLinePos = sourceText.Lines.GetLinePosition position let fcsTextLineNumber = Line.fromZ textLinePos.Line let! lexerSymbol = Tokenizer.getSymbolAtPosition(documentKey, sourceText, position, filePath, defines, SymbolLookupKind.Greedy, false) - let! _, _, checkFileResults = checker.ParseAndCheckDocument (filePath, textVersionHash, sourceText.ToString(), options, LanguageServicePerformanceOptions.Default, userOpName=userOpName) |> Async.RunSynchronously + let! _, _, checkFileResults = checker.ParseAndCheckDocument (filePath, textVersionHash, sourceText, options, LanguageServicePerformanceOptions.Default, userOpName=userOpName) |> Async.RunSynchronously let declarations = checkFileResults.GetDeclarationLocation (fcsTextLineNumber, lexerSymbol.Ident.idRange.EndColumn, textLine.ToString(), lexerSymbol.FullIsland, false, userOpName=userOpName) |> Async.RunSynchronously diff --git a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs index 3c5b8f40147..c53e0421277 100644 --- a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs @@ -33,7 +33,7 @@ type SemanticClassificationServiceTests() = let getRanges (sourceText: string) : (Range.range * SemanticClassificationType) list = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, sourceText, projectOptions, perfOptions, "") + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString sourceText, projectOptions, perfOptions, "") return checkFileResults.GetSemanticClassification(None) } |> Async.RunSynchronously From 1f0a4a830dba2836305bd876eae3589f43964898 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 15:45:22 -0800 Subject: [PATCH 04/33] Fixing tests --- tests/service/AssemblyReaderShim.fs | 2 +- tests/service/Common.fs | 12 ++--- tests/service/EditorTests.fs | 4 +- tests/service/MultiProjectAnalysisTests.fs | 4 +- tests/service/PerfTests.fs | 2 +- tests/service/ProjectAnalysisTests.fs | 53 +++++++++++++--------- tests/service/ProjectOptionsTests.fs | 14 +++--- vsintegration/tests/Salsa/salsa.fs | 4 +- 8 files changed, 52 insertions(+), 43 deletions(-) diff --git a/tests/service/AssemblyReaderShim.fs b/tests/service/AssemblyReaderShim.fs index e7fd0a48ba3..51664dd32e6 100644 --- a/tests/service/AssemblyReaderShim.fs +++ b/tests/service/AssemblyReaderShim.fs @@ -29,5 +29,5 @@ let x = 123 """ let fileName, options = Common.mkTestFileAndOptions source [| |] - Common.checker.ParseAndCheckFileInProject(fileName, 0, source, options) |> Async.RunSynchronously |> ignore + Common.checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, options) |> Async.RunSynchronously |> ignore gotRequest |> should be True diff --git a/tests/service/Common.fs b/tests/service/Common.fs index 081a24bbc33..36d6d6c156d 100644 --- a/tests/service/Common.fs +++ b/tests/service/Common.fs @@ -57,13 +57,13 @@ type TempFile(ext, contents) = let getBackgroundParseResultsForScriptText (input) = use file = new TempFile("fsx", input) - let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, input) |> Async.RunSynchronously + let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) |> Async.RunSynchronously checker.GetBackgroundParseResultsForFileInProject(file.Name, checkOptions) |> Async.RunSynchronously let getBackgroundCheckResultsForScriptText (input) = use file = new TempFile("fsx", input) - let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, input) |> Async.RunSynchronously + let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file.Name, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) |> Async.RunSynchronously checker.GetBackgroundCheckResultsForFileInProject(file.Name, checkOptions) |> Async.RunSynchronously @@ -167,7 +167,7 @@ let mkTestFileAndOptions source additionalArgs = fileName, options let parseAndCheckFile fileName source options = - match checker.ParseAndCheckFileInProject(fileName, 0, source, options) |> Async.RunSynchronously with + match checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, options) |> Async.RunSynchronously with | parseResults, FSharpCheckFileAnswer.Succeeded(checkResults) -> parseResults, checkResults | _ -> failwithf "Parsing aborted unexpectedly..." @@ -181,11 +181,11 @@ let parseAndCheckScript (file, input) = let projectOptions = checker.GetProjectOptionsFromCommandLineArgs (projName, args) #else - let projectOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, input) |> Async.RunSynchronously + let projectOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) |> Async.RunSynchronously printfn "projectOptions = %A" projectOptions #endif - let parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, input, projectOptions) |> Async.RunSynchronously + let parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString input, projectOptions) |> Async.RunSynchronously // if parseResult.Errors.Length > 0 then // printfn "---> Parse Input = %A" input @@ -204,7 +204,7 @@ let parseSourceCode (name: string, code: string) = let dllPath = Path.Combine(location, name + ".dll") let args = mkProjectCommandLineArgs(dllPath, [filePath]) let options, errors = checker.GetParsingOptionsFromCommandLineArgs(List.ofArray args) - let parseResults = checker.ParseFile(filePath, code, options) |> Async.RunSynchronously + let parseResults = checker.ParseFile(filePath, Microsoft.FSharp.Compiler.Text.SourceText.ofString code, options) |> Async.RunSynchronously parseResults.ParseTree /// Extract range info diff --git a/tests/service/EditorTests.fs b/tests/service/EditorTests.fs index 5edcd2da94c..35adea1a9ab 100644 --- a/tests/service/EditorTests.fs +++ b/tests/service/EditorTests.fs @@ -116,8 +116,8 @@ let ``Basic cancellation test`` () = let file = "/home/user/Test.fsx" async { checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() - let! checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, input) - let! parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, input, checkOptions) + let! checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(file, Microsoft.FSharp.Compiler.Text.SourceText.ofString input) + let! parseResult, typedRes = checker.ParseAndCheckFileInProject(file, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString input, checkOptions) return parseResult, typedRes } |> Async.RunSynchronously |> ignore diff --git a/tests/service/MultiProjectAnalysisTests.fs b/tests/service/MultiProjectAnalysisTests.fs index 0e4d38a3321..4dfe416858b 100644 --- a/tests/service/MultiProjectAnalysisTests.fs +++ b/tests/service/MultiProjectAnalysisTests.fs @@ -913,7 +913,7 @@ let ``Type provider project references should not throw exceptions`` () = //printfn "options: %A" options let fileName = __SOURCE_DIRECTORY__ + @"/data/TypeProviderConsole/Program.fs" let fileSource = File.ReadAllText(fileName) - let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, fileSource, options) |> Async.RunSynchronously + let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource, options) |> Async.RunSynchronously let fileCheckResults = match fileCheckAnswer with | FSharpCheckFileAnswer.Succeeded(res) -> res @@ -1010,7 +1010,7 @@ let ``Projects creating generated types should not utilize cross-project-referen //printfn "options: %A" options let fileName = __SOURCE_DIRECTORY__ + @"/data/TypeProvidersBug/TestConsole/Program.fs" let fileSource = File.ReadAllText(fileName) - let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, fileSource, options) |> Async.RunSynchronously + let fileParseResults, fileCheckAnswer = checker.ParseAndCheckFileInProject(fileName, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource, options) |> Async.RunSynchronously let fileCheckResults = match fileCheckAnswer with | FSharpCheckFileAnswer.Succeeded(res) -> res diff --git a/tests/service/PerfTests.fs b/tests/service/PerfTests.fs index 662e00c5645..ad544f39499 100644 --- a/tests/service/PerfTests.fs +++ b/tests/service/PerfTests.fs @@ -31,7 +31,7 @@ module internal Project1 = let projFileName = Path.ChangeExtension(base2, ".fsproj") let fileSources = [ for (i,f) in fileNamesI -> (f, "module M" + string i) ] for (f,text) in fileSources do File.WriteAllText(f, text) - let fileSources2 = [ for (i,f) in fileSources -> f ] + let fileSources2 = [ for (i,f) in fileSources -> Microsoft.FSharp.Compiler.Text.SourceText.ofString f ] let fileNames = [ for (_,f) in fileNamesI -> f ] let args = mkProjectCommandLineArgs (dllName, fileNames) diff --git a/tests/service/ProjectAnalysisTests.fs b/tests/service/ProjectAnalysisTests.fs index 5ea55a40505..0913c673988 100644 --- a/tests/service/ProjectAnalysisTests.fs +++ b/tests/service/ProjectAnalysisTests.fs @@ -28,7 +28,7 @@ module internal Project1 = let fileName2 = Path.ChangeExtension(base2, ".fs") let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module M type C() = @@ -39,9 +39,10 @@ let fff () = xxx + xxx type CAbbrev = C """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) - let fileSource2 = """ + let fileSource2Text = """ module N open M @@ -83,7 +84,8 @@ let mmmm1 : M.C = new M.C() // note, these don't count as uses of CA let mmmm2 : M.CAbbrev = new M.CAbbrev() // note, these don't count as uses of C """ - File.WriteAllText(fileName2, fileSource2) + let fileSource2 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource2Text + File.WriteAllText(fileName2, fileSource2Text) let fileNames = [fileName1; fileName2] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -2411,7 +2413,7 @@ module internal Project16 = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module Impl type C() = @@ -2427,9 +2429,10 @@ and F = { Field1 : int; Field2 : int } and G = Case1 | Case2 of int """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) - let sigFileSource1 = """ + let sigFileSource1Text = """ module Impl type C = @@ -2448,7 +2451,8 @@ and F = { Field1 : int; Field2 : int } and G = Case1 | Case2 of int """ - File.WriteAllText(sigFileName1, sigFileSource1) + let sigFileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString sigFileSource1Text + File.WriteAllText(sigFileName1, sigFileSource1Text) let cleanFileName a = if a = fileName1 then "file1" elif a = sigFileName1 then "sig1" else "??" let fileNames = [sigFileName1; fileName1] @@ -4506,11 +4510,12 @@ module internal Project35b = open System.IO let fileName1 = Path.ChangeExtension(Path.GetTempFileName(), ".fsx") - let fileSource1 = """ + let fileSource1Text = """ #r "System.dll" #r "notexist.dll" """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let cleanFileName a = if a = fileName1 then "file1" else "??" let fileNames = [fileName1] @@ -5153,7 +5158,7 @@ module internal ProjectBig = let projFileName = Path.ChangeExtension(base2, ".fsproj") let fileSources = [ for (i,f) in fileNamesI -> (f, "module M" + string i) ] for (f,text) in fileSources do File.WriteAllText(f, text) - let fileSources2 = [ for (i,f) in fileSources -> f ] + let fileSources2 = [ for (i,f) in fileSources -> Microsoft.FSharp.Compiler.Text.SourceText.ofString f ] let fileNames = [ for (_,f) in fileNamesI -> f ] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5286,14 +5291,14 @@ module internal ProjectLineDirectives = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module M # 10 "Test.fsy" let x = (1 = 3.0) """ - - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) let options = checker.GetProjectOptionsFromCommandLineArgs (projFileName, args) @@ -5329,11 +5334,12 @@ let ``ParseAndCheckFileResults contains ImplFile list if FSharpChecker is create let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ type A(i:int) = member x.Value = i """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5386,7 +5392,7 @@ let ``Unused opens in rec module smoke test 1``() = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module rec Module open System.Collections // unused @@ -5423,7 +5429,8 @@ type UseTheThings(i:int) = member x.UseSomeUsedModuleContainingExtensionMember() = (3).Q member x.UseSomeUsedModuleContainingUnion() = A """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5458,7 +5465,7 @@ let ``Unused opens in non rec module smoke test 1``() = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ module Module open System.Collections // unused @@ -5495,7 +5502,8 @@ type UseTheThings(i:int) = member x.UseSomeUsedModuleContainingExtensionMember() = (3).Q member x.UseSomeUsedModuleContainingUnion() = A """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) @@ -5530,7 +5538,7 @@ let ``Unused opens smoke test auto open``() = let base2 = Path.GetTempFileName() let dllName = Path.ChangeExtension(base2, ".dll") let projFileName = Path.ChangeExtension(base2, ".fsproj") - let fileSource1 = """ + let fileSource1Text = """ open System.Collections // unused open System.Collections.Generic // used, should not appear open FSharp.Control // unused @@ -5567,7 +5575,8 @@ type UseTheThings(i:int) = member x.UseSomeUsedModuleContainingExtensionMember() = (3).Q member x.UseSomeUsedModuleContainingUnion() = A """ - File.WriteAllText(fileName1, fileSource1) + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + File.WriteAllText(fileName1, fileSource1Text) let fileNames = [fileName1] let args = mkProjectCommandLineArgs (dllName, fileNames) diff --git a/tests/service/ProjectOptionsTests.fs b/tests/service/ProjectOptionsTests.fs index b390f53e62b..5c5d86db3ba 100644 --- a/tests/service/ProjectOptionsTests.fs +++ b/tests/service/ProjectOptionsTests.fs @@ -508,7 +508,7 @@ let ``Test SourceFiles order for GetProjectOptionsFromScript`` () = // See #594 let scriptPath = __SOURCE_DIRECTORY__ + @"/data/ScriptProject/" + scriptName + ".fsx" let scriptSource = File.ReadAllText scriptPath let projOpts, _diagnostics = - checker.GetProjectOptionsFromScript(scriptPath, scriptSource) + checker.GetProjectOptionsFromScript(scriptPath, Microsoft.FSharp.Compiler.Text.SourceText.ofString scriptSource) |> Async.RunSynchronously projOpts.SourceFiles |> Array.map Path.GetFileNameWithoutExtension @@ -527,21 +527,21 @@ let ``Script load closure project`` () = let fileName1 = Path.GetTempPath() + Path.DirectorySeparatorChar.ToString() + "Impl.fs" let fileName2 = Path.ChangeExtension(Path.GetTempFileName(), ".fsx") - let fileSource1 = """ + let fileSource1Text = """ module ImplFile #if INTERACTIVE let x = 42 #endif """ - - let fileSource2 = """ + let fileSource1 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource1Text + let fileSource2Text = """ #load "Impl.fs" ImplFile.x """ - - File.WriteAllText(fileName1, fileSource1) - File.WriteAllText(fileName2, fileSource2) + let fileSource2 = Microsoft.FSharp.Compiler.Text.SourceText.ofString fileSource2Text + File.WriteAllText(fileName1, fileSource1Text) + File.WriteAllText(fileName2, fileSource2Text) printfn "------Starting Script load closure project----" printfn "Getting project options..." diff --git a/vsintegration/tests/Salsa/salsa.fs b/vsintegration/tests/Salsa/salsa.fs index 1dc47629103..418387f487d 100644 --- a/vsintegration/tests/Salsa/salsa.fs +++ b/vsintegration/tests/Salsa/salsa.fs @@ -1099,7 +1099,7 @@ module internal Salsa = member file.GetFileName() = filename member file.GetProjectOptionsOfScript() = - project.Solution.Vs.LanguageService.FSharpChecker.GetProjectOptionsFromScript(filename, file.CombinedLines, System.DateTime(2000,1,1), [| |]) + project.Solution.Vs.LanguageService.FSharpChecker.GetProjectOptionsFromScript(filename, Microsoft.FSharp.Compiler.Text.SourceText.ofString file.CombinedLines, System.DateTime(2000,1,1), [| |]) |> Async.RunSynchronously |> fst // drop diagnostics @@ -1111,7 +1111,7 @@ module internal Salsa = member file.OnIdle() = while file.Source.NeedsVisualRefresh do file.OnIdleTypeCheck() - member file.CombinedLines = + member file.CombinedLines : string = if combinedLines = null then combinedLines<-String.Join("\n",lines) combinedLines From 159d8a6ee3a6ddb82a9354cba9ad27338132d755 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 15:47:09 -0800 Subject: [PATCH 05/33] We need to use addNewLine for tests to pass --- src/fsharp/service/service.fs | 5 ----- src/utils/prim-lexing.fs | 7 ++++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index 064999d75ad..ee371f05538 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -1537,11 +1537,6 @@ module internal Parser = let tokenizer = LexFilter.LexFilter(lightSyntaxStatus, options.CompilingFsLib, Lexer.token lexargs true, lexbuf) tokenizer.Lexer - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - let createLexbuf (sourceText: ISourceText) = UnicodeLexing.SourceTextAsLexbuf(sourceText) diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 669c14b2672..da638f4f35a 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -100,7 +100,12 @@ type StringSourceText(str: string) = module SourceText = - let ofString str = StringSourceText(str) :> ISourceText + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let private addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + + let ofString str = StringSourceText(addNewLine str) :> ISourceText namespace Internal.Utilities.Text.Lexing From e365da17fa552d4d1275039fe3bb4fd0f58553fd Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 17:10:18 -0800 Subject: [PATCH 06/33] Added test for SourceText.ofString --- src/utils/prim-lexing.fs | 57 ++++++++++++++----- .../FSharp.Compiler.UnitTests.fsproj | 1 + tests/service/AssemblyContentProviderTests.fs | 2 +- .../SemanticColorizationServiceTests.fs | 2 +- .../tests/UnitTests/UnusedOpensTests.fs | 2 +- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index da638f4f35a..fd785a34093 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -51,25 +51,54 @@ type ISourceText = [] type StringTextLineCollection(str: string) = - let getLines (str: string) = + let isAnyLineBreakCharacter c = + c = '\n' || c = '\r' || c = '\u0085' || c = '\u2028' || c = '\u2029' + + let lineStarts = lazy - use reader = new StringReader(str) - [| - let line = ref (reader.ReadLine()) - while not (isNull !line) do - yield !line - line := reader.ReadLine() - if str.EndsWith("\n", StringComparison.Ordinal) then - // last trailing space not returned - // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak - yield String.Empty - |] + if str.Length = 0 then ResizeArray() + else + + let lineStarts = ResizeArray() + lineStarts.Add(0) + + let mutable index = 0 + while index < str.Length do + let c = str.[index] + index <- index + 1 + + // Common case - ASCII & not a line break + // if (c > '\r' && c <= 127) + // if (c >= ('\r'+1) && c <= 127) + let bias = uint32 '\r' + 1u + if uint32 c - bias <= (127u - bias) then () + else + + // Assumes that only 2-char line break sequence is CR+LF + if c = '\r' then + if index < str.Length && str.[index] = '\n' then + index <- index + 1 + lineStarts.Add(index) + elif isAnyLineBreakCharacter c then + lineStarts.Add(index) + + lineStarts interface ITextLineCollection with - member __.Item with get index = TextSpan(0, (getLines str).Value.[index].Length) + member __.Item + with get index = + let lineStarts = lineStarts.Value + + let start = lineStarts.[index] + if index = lineStarts.Count - 1 then + let length = str.Length - start + TextSpan(start, length) + else + let length = lineStarts.[index + 1] - start + TextSpan(start, length) - member __.Count = (getLines str).Value.Length + member __.Count = lineStarts.Value.Count [] type StringSourceText(str: string) = diff --git a/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj b/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj index 4d00bae0a1f..2f6a1624fbb 100644 --- a/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj +++ b/tests/FSharp.Compiler.UnitTests/FSharp.Compiler.UnitTests.fsproj @@ -53,6 +53,7 @@ + diff --git a/tests/service/AssemblyContentProviderTests.fs b/tests/service/AssemblyContentProviderTests.fs index 30f11f4cdd5..b98eaf33a0c 100644 --- a/tests/service/AssemblyContentProviderTests.fs +++ b/tests/service/AssemblyContentProviderTests.fs @@ -43,7 +43,7 @@ let (=>) (source: string) (expected: string list) = // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak yield "" |] - let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, source, projectOptions) |> Async.RunSynchronously + let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, projectOptions) |> Async.RunSynchronously let checkFileResults = match checkFileAnswer with diff --git a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs index c53e0421277..4bac0dff22a 100644 --- a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs @@ -33,7 +33,7 @@ type SemanticClassificationServiceTests() = let getRanges (sourceText: string) : (Range.range * SemanticClassificationType) list = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString sourceText, projectOptions, perfOptions, "") + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(sourceText), projectOptions, perfOptions, "") return checkFileResults.GetSemanticClassification(None) } |> Async.RunSynchronously diff --git a/vsintegration/tests/UnitTests/UnusedOpensTests.fs b/vsintegration/tests/UnitTests/UnusedOpensTests.fs index d23f5ed4e18..64f3fa2e8a3 100644 --- a/vsintegration/tests/UnitTests/UnusedOpensTests.fs +++ b/vsintegration/tests/UnitTests/UnusedOpensTests.fs @@ -29,7 +29,7 @@ let private checker = FSharpChecker.Create() let (=>) (source: string) (expectedRanges: ((*line*)int * ((*start column*)int * (*end column*)int)) list) = let sourceLines = source.Split ([|"\r\n"; "\n"; "\r"|], StringSplitOptions.None) - let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, source, projectOptions) |> Async.RunSynchronously + let _, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, 0, Microsoft.FSharp.Compiler.Text.SourceText.ofString source, projectOptions) |> Async.RunSynchronously let checkFileResults = match checkFileAnswer with From f649d0c3d11ab1e000497ecdb8908da48d7fe3cd Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 17:36:17 -0800 Subject: [PATCH 07/33] Trying to fix tests --- .../SourceTextTests.fs | 25 +++++++++++++++++++ .../src/FSharp.Editor/Common/Extensions.fs | 3 +-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 tests/FSharp.Compiler.UnitTests/SourceTextTests.fs diff --git a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs new file mode 100644 index 00000000000..0715697a46d --- /dev/null +++ b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Compiler.UnitTests + +open System +open NUnit.Framework + +open Microsoft.FSharp.Compiler.Text + +[] +module SourceTextTests = + + [] + let StringText () = + let text = "test\ntest2\r\ntest3\n\ntest4\ntest5\rtest6\n" + let sourceText = SourceText.ofString text + let lines = sourceText.Lines + + Assert.AreEqual("test\n", sourceText.GetTextString(lines.[0])) + Assert.AreEqual("test2\r\n", sourceText.GetTextString(lines.[1])) + Assert.AreEqual("test3\n", sourceText.GetTextString(lines.[2])) + Assert.AreEqual("\n", sourceText.GetTextString(lines.[3])) + Assert.AreEqual("test4\n", sourceText.GetTextString(lines.[4])) + Assert.AreEqual("test5\r", sourceText.GetTextString(lines.[5])) + Assert.AreEqual("test6\n", sourceText.GetTextString(lines.[6])) \ No newline at end of file diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index 6ed75ff8dbe..6bdfbc6fc0e 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -44,7 +44,7 @@ type Document with type TextLine with - member this.ToFSharpTextSpan() = TextSpan(this.Span.Start, this.Span.End) + member this.ToFSharpTextSpan() = TextSpan(this.SpanIncludingLineBreak.Start, this.SpanIncludingLineBreak.Length) type TextLineCollection with @@ -56,7 +56,6 @@ type TextLineCollection with member __.Count = this.Count } - type SourceText with member this.ToFSharpSourceText() = From 4f8eb3933afd5738664e11096c41d896c63a169a Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 20:30:37 -0800 Subject: [PATCH 08/33] Simplified ISourceText API. Added RoslynSourceTextTests --- src/fsharp/service/ServiceXmlDocParser.fs | 7 +- src/fsharp/service/service.fs | 2 +- src/utils/prim-lexing.fs | 121 ++++-------------- src/utils/prim-lexing.fsi | 25 +--- .../SourceTextTests.fs | 22 ++-- .../src/FSharp.Editor/Common/Extensions.fs | 57 +++++---- .../tests/UnitTests/RoslynSourceTextTests.fs | 33 +++++ .../UnitTests/VisualFSharp.UnitTests.fsproj | 1 + 8 files changed, 114 insertions(+), 154 deletions(-) create mode 100644 vsintegration/tests/UnitTests/RoslynSourceTextTests.fs diff --git a/src/fsharp/service/ServiceXmlDocParser.fs b/src/fsharp/service/ServiceXmlDocParser.fs index 4554db327e2..cc9223f4ca6 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fs +++ b/src/fsharp/service/ServiceXmlDocParser.fs @@ -189,9 +189,4 @@ module XmlDocParser = /// Get the list of Xml documentation from current source code let getXmlDocables (sourceText: ISourceText, input) = - let sourceCodeLinesOfTheFile = - [| - for i = 0 to sourceText.Lines.Count - 1 do - yield sourceText.GetTextString(sourceText.Lines.[i]) - |] - XmlDocParsing.getXmlDocablesImpl (sourceCodeLinesOfTheFile, input) \ No newline at end of file + XmlDocParsing.getXmlDocablesImpl (sourceText.GetLines(), input) \ No newline at end of file diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index ee371f05538..95977c1799f 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -1477,7 +1477,7 @@ module internal Parser = let mutable errorCount = 0 // We'll need number of lines for adjusting error messages at EOF - let fileInfo = (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Length) + let fileInfo = sourceText.GetLastCharacterPosition() // This function gets called whenever an error happens during parsing or checking let diagnosticSink sev (exn: PhasedDiagnostic) = diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index fd785a34093..c1e005b8052 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -7,103 +7,38 @@ namespace Microsoft.FSharp.Compiler.Text open System open System.IO -// This could just be a normal Span someday. -[] -type TextSpan = - - val Start : int - - val Length : int - - member this.End = this.Start + this.Length - - new (start, length) = - if start < 0 then - raise (ArgumentOutOfRangeException("start")) - - if start + length < start then - raise (ArgumentOutOfRangeException("length")) - - { Start = start; Length = length } - -type ITextLineCollection = - - abstract Item : int -> TextSpan with get - - abstract Count : int - type ISourceText = abstract Item : int -> char with get - abstract Lines : ITextLineCollection + abstract GetLines : unit -> string [] + + abstract GetLastCharacterPosition : unit -> int * int abstract Length : int abstract ContentEquals : ISourceText -> bool - abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit - - abstract GetTextString : unit -> string - - abstract GetTextString : TextSpan -> string - -[] -type StringTextLineCollection(str: string) = - - let isAnyLineBreakCharacter c = - c = '\n' || c = '\r' || c = '\u0085' || c = '\u2028' || c = '\u2029' - - let lineStarts = - lazy - if str.Length = 0 then ResizeArray() - else - - let lineStarts = ResizeArray() - lineStarts.Add(0) - - let mutable index = 0 - while index < str.Length do - let c = str.[index] - index <- index + 1 - - // Common case - ASCII & not a line break - // if (c > '\r' && c <= 127) - // if (c >= ('\r'+1) && c <= 127) - let bias = uint32 '\r' + 1u - if uint32 c - bias <= (127u - bias) then () - else - - // Assumes that only 2-char line break sequence is CR+LF - if c = '\r' then - if index < str.Length && str.[index] = '\n' then - index <- index + 1 - lineStarts.Add(index) - elif isAnyLineBreakCharacter c then - lineStarts.Add(index) - - lineStarts - - interface ITextLineCollection with - - member __.Item - with get index = - let lineStarts = lineStarts.Value - - let start = lineStarts.[index] - if index = lineStarts.Count - 1 then - let length = str.Length - start - TextSpan(start, length) - else - let length = lineStarts.[index + 1] - start - TextSpan(start, length) - - member __.Count = lineStarts.Value.Count + abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit [] -type StringSourceText(str: string) = - - let lines = StringTextLineCollection(str) :> ITextLineCollection +type StringText(str: string) = + + let getLines (str: string) = + use reader = new StringReader(str) + [| + let line = ref (reader.ReadLine()) + while not (isNull !line) do + yield !line + line := reader.ReadLine() + if str.EndsWith("\n", StringComparison.Ordinal) then + // last trailing space not returned + // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak + yield String.Empty + |] + + let getLines = + lazy getLines str member __.String = str @@ -111,22 +46,22 @@ type StringSourceText(str: string) = member __.Item with get index = str.[index] - member __.Lines = lines + member __.GetLastCharacterPosition() = + let lines = getLines.Value + (lines.Length, lines.[lines.Length - 1].Length) + + member __.GetLines() = getLines.Value member __.Length = str.Length member this.ContentEquals(sourceText) = match sourceText with - | :? StringSourceText as sourceText when sourceText = this || sourceText.String = str -> true + | :? StringText as sourceText when sourceText = this || sourceText.String = str -> true | _ -> false member __.CopyTo(sourceIndex, destination, destinationIndex, count) = str.CopyTo(sourceIndex, destination, destinationIndex, count) - member __.GetTextString() = str - - member __.GetTextString(textSpan) = str.Substring(textSpan.Start, textSpan.Length) - module SourceText = // Adding this new-line character at the end of the source seems odd but is required for some unit tests @@ -134,7 +69,7 @@ module SourceText = let private addNewLine (source: string) = if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - let ofString str = StringSourceText(addNewLine str) :> ISourceText + let ofString str = StringText(addNewLine str) :> ISourceText namespace Internal.Utilities.Text.Lexing diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index 60134cf42b2..fb6db3c911a 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -5,28 +5,13 @@ namespace Microsoft.FSharp.Compiler.Text -[] -type TextSpan = - - val Start : int - - val Length : int - - member End : int - - new : start: int * length: int -> TextSpan - -type ITextLineCollection = - - abstract Item : int -> TextSpan with get - - abstract Count : int - type ISourceText = abstract Item : int -> char with get - abstract Lines : ITextLineCollection + abstract GetLastCharacterPosition : unit -> int * int + + abstract GetLines : unit -> string [] abstract Length : int @@ -34,10 +19,6 @@ type ISourceText = abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit - abstract GetTextString : unit -> string - - abstract GetTextString : TextSpan -> string - module SourceText = val ofString : string -> ISourceText diff --git a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs index 0715697a46d..deed63d7a45 100644 --- a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs +++ b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs @@ -14,12 +14,18 @@ module SourceTextTests = let StringText () = let text = "test\ntest2\r\ntest3\n\ntest4\ntest5\rtest6\n" let sourceText = SourceText.ofString text - let lines = sourceText.Lines + let lines = sourceText.GetLines() - Assert.AreEqual("test\n", sourceText.GetTextString(lines.[0])) - Assert.AreEqual("test2\r\n", sourceText.GetTextString(lines.[1])) - Assert.AreEqual("test3\n", sourceText.GetTextString(lines.[2])) - Assert.AreEqual("\n", sourceText.GetTextString(lines.[3])) - Assert.AreEqual("test4\n", sourceText.GetTextString(lines.[4])) - Assert.AreEqual("test5\r", sourceText.GetTextString(lines.[5])) - Assert.AreEqual("test6\n", sourceText.GetTextString(lines.[6])) \ No newline at end of file + Assert.AreEqual("test", lines.[0]) + Assert.AreEqual("test2", lines.[1]) + Assert.AreEqual("test3", lines.[2]) + Assert.AreEqual("", lines.[3]) + Assert.AreEqual("test4", lines.[4]) + Assert.AreEqual("test5", lines.[5]) + Assert.AreEqual("test6", lines.[6]) + Assert.AreEqual("", lines.[7]) + Assert.AreEqual(8, lines.Length) + + let (count, length) = sourceText.GetLastCharacterPosition() + Assert.AreEqual(8, count) + Assert.AreEqual(0, length) \ No newline at end of file diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index 6bdfbc6fc0e..09b3b046a4b 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -42,43 +42,52 @@ type Document with languageServices.GetService<'T>() |> Some -type TextLine with +module private SourceText = - member this.ToFSharpTextSpan() = TextSpan(this.SpanIncludingLineBreak.Start, this.SpanIncludingLineBreak.Length) + open System.Runtime.CompilerServices -type TextLineCollection with - - member this.ToFSharpTextLineCollection() = - { new ITextLineCollection with - - member __.Item with get index = this.[index].ToFSharpTextSpan() - - member __.Count = this.Count - } + let weakTable = ConditionalWeakTable() type SourceText with member this.ToFSharpSourceText() = - { new ISourceText with + match SourceText.weakTable.TryGetValue(this) with + | true, sourceText -> sourceText + | _ -> + let getLines = + lazy + [| + for i = 0 to this.Lines.Count - 1 do + yield this.Lines.[i].ToString() + |] + + let getLastCharacterPosition = + lazy + (this.Lines.Count, this.Lines.[this.Lines.Count - 1].Span.Length) + + let sourceText = + { new ISourceText with - member __.Item with get index = this.[index] + member __.Item with get index = this.[index] + + member __.GetLines() = getLines.Value - member __.Lines = this.Lines.ToFSharpTextLineCollection() + member __.GetLastCharacterPosition() = getLastCharacterPosition.Value - member __.ContentEquals(sourceText) = - match sourceText with - | :? SourceText as sourceText -> this.ContentEquals(sourceText) - | _ -> false + member __.ContentEquals(sourceText) = + match sourceText with + | :? SourceText as sourceText -> this.ContentEquals(sourceText) + | _ -> false - member __.Length = this.Length + member __.Length = this.Length - member __.CopyTo(sourceIndex, destination, destinationIndex, count) = - this.CopyTo(sourceIndex, destination, destinationIndex, count) + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + this.CopyTo(sourceIndex, destination, destinationIndex, count) + } - member __.GetTextString() = this.ToString() + SourceText.weakTable.Add(this, sourceText) - member __.GetTextString(textSpan) = this.ToString(Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(textSpan.Start, textSpan.End)) - } + sourceText type FSharpNavigationDeclarationItem with member x.RoslynGlyph : Glyph = diff --git a/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs new file mode 100644 index 00000000000..efd14fd576e --- /dev/null +++ b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace Microsoft.VisualStudio.FSharp.Editor.Tests.Roslyn + +open System +open NUnit.Framework + +open Microsoft.VisualStudio.FSharp.Editor +open Microsoft.FSharp.Compiler.Text +open Microsoft.CodeAnalysis.Text + +[] +module RoslynSourceTextTests = + + [] + let SourceText () = + let text = "test\ntest2\r\ntest3\n\ntest4\ntest5\rtest6\n" + let sourceText = SourceText.From(text).ToFSharpSourceText() + let lines = sourceText.GetLines() + + Assert.AreEqual("test", lines.[0]) + Assert.AreEqual("test2", lines.[1]) + Assert.AreEqual("test3", lines.[2]) + Assert.AreEqual("", lines.[3]) + Assert.AreEqual("test4", lines.[4]) + Assert.AreEqual("test5", lines.[5]) + Assert.AreEqual("test6", lines.[6]) + Assert.AreEqual("", lines.[7]) + Assert.AreEqual(8, lines.Length) + + let (count, length) = sourceText.GetLastCharacterPosition() + Assert.AreEqual(8, count) + Assert.AreEqual(0, length) \ No newline at end of file diff --git a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj index ea8b5992d1f..997471f88ce 100644 --- a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj +++ b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj @@ -122,6 +122,7 @@ --> + Roslyn\IndentationServiceTests.fs From 56db86791d4752860da09db613ba3252609b7a5b Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 21:14:58 -0800 Subject: [PATCH 09/33] Trying to get the build working again --- fcs/samples/EditorService/Program.fs | 1 + fcs/samples/UntypedTree/Program.fs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fcs/samples/EditorService/Program.fs b/fcs/samples/EditorService/Program.fs index 5a2cb624f0c..17fc133e6e3 100644 --- a/fcs/samples/EditorService/Program.fs +++ b/fcs/samples/EditorService/Program.fs @@ -8,6 +8,7 @@ open Microsoft.FSharp.Compiler.QuickParse let checker = FSharpChecker.Create() let parseWithTypeInfo (file, input) = + let input = Microsoft.FSharp.Compiler.Text.SourceText.ofString input let checkOptions, _errors = checker.GetProjectOptionsFromScript(file, input) |> Async.RunSynchronously let parsingOptions, _errors = checker.GetParsingOptionsFromProjectOptions(checkOptions) let untypedRes = checker.ParseFile(file, input, parsingOptions) |> Async.RunSynchronously diff --git a/fcs/samples/UntypedTree/Program.fs b/fcs/samples/UntypedTree/Program.fs index 3334b093bad..8d4986125e7 100644 --- a/fcs/samples/UntypedTree/Program.fs +++ b/fcs/samples/UntypedTree/Program.fs @@ -11,7 +11,7 @@ let checker = FSharpChecker.Create() // Get untyped tree for a specified input let getUntypedTree (file, input) = let parsingOptions = { FSharpParsingOptions.Default with SourceFiles = [| file |] } - let untypedRes = checker.ParseFile(file, input, parsingOptions) |> Async.RunSynchronously + let untypedRes = checker.ParseFile(file, Microsoft.FSharp.Compiler.Text.SourceText.ofString input, parsingOptions) |> Async.RunSynchronously match untypedRes.ParseTree with | Some tree -> tree | None -> failwith "Something went wrong during parsing!" From ee7e0f12210c4494aed82a8cbfd793eeba4feb29 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 11 Dec 2018 21:45:49 -0800 Subject: [PATCH 10/33] Re-organize prim-lexing.fsi --- src/utils/prim-lexing.fsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index fb6db3c911a..02001acd338 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -9,10 +9,10 @@ type ISourceText = abstract Item : int -> char with get - abstract GetLastCharacterPosition : unit -> int * int - abstract GetLines : unit -> string [] + abstract GetLastCharacterPosition : unit -> int * int + abstract Length : int abstract ContentEquals : ISourceText -> bool From 83682c2dbb17f2718a6f71ae2e9a0cf4f4ea2c6e Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 12 Dec 2018 18:22:47 -0800 Subject: [PATCH 11/33] Handling format strings --- src/fsharp/CheckFormatStrings.fs | 20 ++++++++-------- src/fsharp/NameResolution.fs | 23 ++++++++++--------- src/fsharp/NameResolution.fsi | 7 +++--- src/fsharp/TypeChecker.fs | 2 +- src/fsharp/service/service.fs | 2 +- src/utils/prim-lexing.fs | 17 ++++++++++++++ src/utils/prim-lexing.fsi | 6 ++++- .../src/FSharp.Editor/Common/Extensions.fs | 6 +++++ 8 files changed, 57 insertions(+), 26 deletions(-) diff --git a/src/fsharp/CheckFormatStrings.fs b/src/fsharp/CheckFormatStrings.fs index 9d7fd72d7a0..94a96642eda 100644 --- a/src/fsharp/CheckFormatStrings.fs +++ b/src/fsharp/CheckFormatStrings.fs @@ -54,15 +54,17 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe let (offset, fmt) = match context with | Some context -> - let length = context.Source.Length - if m.EndLine < context.LineStartPositions.Length then - let startIndex = context.LineStartPositions.[m.StartLine-1] + m.StartColumn - let endIndex = context.LineStartPositions.[m.EndLine-1] + m.EndColumn - 1 - if startIndex < length-3 && context.Source.[startIndex..startIndex+2] = "\"\"\"" then - (3, context.Source.[startIndex+3..endIndex-3]) - elif startIndex < length-2 && context.Source.[startIndex..startIndex+1] = "@\"" then - (2, context.Source.[startIndex+2..endIndex-1]) - else (1, context.Source.[startIndex+1..endIndex-1]) + let sourceText = context.SourceText + let lineStartPositions = context.LineStartPositions + let length = sourceText.Length + if m.EndLine < lineStartPositions.Length then + let startIndex = lineStartPositions.[m.StartLine-1] + m.StartColumn + let endIndex = lineStartPositions.[m.EndLine-1] + m.EndColumn - 1 + if startIndex < length-3 && sourceText.SubTextEquals("\"\"\"", startIndex) then + (3, sourceText.GetSubTextString(startIndex + 3, endIndex - startIndex)) //context.SourceText.[startIndex+3..endIndex-3]) + elif startIndex < length-2 && sourceText.SubTextEquals("@\"", startIndex) then + (2, sourceText.GetSubTextString(startIndex + 2, endIndex + 1 - startIndex)) //context.SourceText.[startIndex+2..endIndex-1]) + else (1, sourceText.GetSubTextString(startIndex + 1, endIndex - startIndex)) //context.SourceText.[startIndex+1..endIndex-1]) else (1, fmt) | None -> (1, fmt) diff --git a/src/fsharp/NameResolution.fs b/src/fsharp/NameResolution.fs index 43c15418490..d3d1c7db881 100644 --- a/src/fsharp/NameResolution.fs +++ b/src/fsharp/NameResolution.fs @@ -5,7 +5,8 @@ module internal Microsoft.FSharp.Compiler.NameResolution open Internal.Utilities -open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.Range open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.ErrorLogger @@ -1251,7 +1252,7 @@ type OpenDeclaration = IsOwnNamespace = isOwnNamespace } type FormatStringCheckContext = - { Source: string + { SourceText: ISourceText LineStartPositions: int[] } /// An abstract type for reporting the results of name resolution and type checking. @@ -1261,7 +1262,7 @@ type ITypecheckResultsSink = abstract NotifyNameResolution : pos * Item * Item * TyparInst * ItemOccurence * Tastops.DisplayEnv * NameResolutionEnv * AccessorDomain * range * bool -> unit abstract NotifyFormatSpecifierLocation : range * int -> unit abstract NotifyOpenDeclaration : OpenDeclaration -> unit - abstract CurrentSource : string option + abstract CurrentSourceText : ISourceText option abstract FormatStringCheckContext : FormatStringCheckContext option let (|ValRefOfProp|_|) (pi : PropInfo) = pi.ArbitraryValRef @@ -1513,7 +1514,7 @@ type TcSymbolUses(g, capturedNameResolutions : ResizeArray() let capturedExprTypings = ResizeArray<_>() let capturedNameResolutions = ResizeArray<_>() @@ -1537,18 +1538,18 @@ type TcResultsSinkImpl(g, ?source: string) = let formatStringCheckContext = lazy - source |> Option.map (fun source -> + sourceText |> Option.map (fun sourceText -> let positions = [| yield 0 - for i in 1..source.Length do - let c = source.[i-1] - if c = '\r' && i < source.Length && source.[i] = '\n' then () + for i in 1..sourceText.Length do + let c = sourceText.[i-1] + if c = '\r' && i < sourceText.Length && sourceText.[i] = '\n' then () elif c = '\r' then yield i elif c = '\n' then yield i - yield source.Length + yield sourceText.Length |] - { Source = source + { SourceText = sourceText LineStartPositions = positions }) member this.GetResolutions() = @@ -1603,7 +1604,7 @@ type TcResultsSinkImpl(g, ?source: string) = member sink.NotifyOpenDeclaration(openDeclaration) = capturedOpenDeclarations.Add(openDeclaration) - member sink.CurrentSource = source + member sink.CurrentSourceText = sourceText member sink.FormatStringCheckContext = formatStringCheckContext.Value diff --git a/src/fsharp/NameResolution.fsi b/src/fsharp/NameResolution.fsi index 10fe03ff0b1..8912c8946a5 100755 --- a/src/fsharp/NameResolution.fsi +++ b/src/fsharp/NameResolution.fsi @@ -3,6 +3,7 @@ module internal Microsoft.FSharp.Compiler.NameResolution open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Text open Microsoft.FSharp.Compiler.AccessibilityLogic open Microsoft.FSharp.Compiler.Ast open Microsoft.FSharp.Compiler.Infos @@ -347,7 +348,7 @@ type internal OpenDeclaration = /// Source text and an array of line end positions, used for format string parsing type FormatStringCheckContext = { /// Source text - Source: string + SourceText: ISourceText /// Array of line start positions LineStartPositions: int[] } @@ -370,7 +371,7 @@ type ITypecheckResultsSink = abstract NotifyOpenDeclaration : OpenDeclaration -> unit /// Get the current source - abstract CurrentSource : string option + abstract CurrentSourceText : ISourceText option /// Cached line-end normalized source text and an array of line end positions, used for format string parsing abstract FormatStringCheckContext : FormatStringCheckContext option @@ -379,7 +380,7 @@ type ITypecheckResultsSink = type internal TcResultsSinkImpl = /// Create a TcResultsSinkImpl - new : tcGlobals : TcGlobals * ?source:string -> TcResultsSinkImpl + new : tcGlobals : TcGlobals * ?sourceText: ISourceText -> TcResultsSinkImpl /// Get all the resolutions reported to the sink member GetResolutions : unit -> TcResolutions diff --git a/src/fsharp/TypeChecker.fs b/src/fsharp/TypeChecker.fs index 495e0d4c470..3b9c9e07647 100755 --- a/src/fsharp/TypeChecker.fs +++ b/src/fsharp/TypeChecker.fs @@ -1860,7 +1860,7 @@ let MakeAndPublishSimpleVals cenv env m names mergeNamesInOneNameresEnv = member this.NotifyExprHasType(_, _, _, _, _, _) = assert false // no expr typings in MakeSimpleVals member this.NotifyFormatSpecifierLocation(_, _) = () member this.NotifyOpenDeclaration(_) = () - member this.CurrentSource = None + member this.CurrentSourceText = None member this.FormatStringCheckContext = None } use _h = WithNewTypecheckResultsSink(sink, cenv.tcSink) diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index 95977c1799f..9c3470eaab0 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -1710,7 +1710,7 @@ module internal Parser = tcState.NiceNameGenerator.Reset() // Typecheck the real input. - let sink = TcResultsSinkImpl(tcGlobals) + let sink = TcResultsSinkImpl(tcGlobals, sourceText = sourceText) let! ct = Async.CancellationToken diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index c1e005b8052..a336bed57fa 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -15,6 +15,10 @@ type ISourceText = abstract GetLastCharacterPosition : unit -> int * int + abstract GetSubTextString : start: int * length: int -> string + + abstract SubTextEquals : target: string * startIndex: int -> bool + abstract Length : int abstract ContentEquals : ISourceText -> bool @@ -52,6 +56,19 @@ type StringText(str: string) = member __.GetLines() = getLines.Value + member __.GetSubTextString(start, length) = + str.Substring(start, length) + + member __.SubTextEquals(target, startIndex) = + if startIndex < 0 then + raise (ArgumentOutOfRangeException("startIndex")) + + if String.IsNullOrEmpty(target) then + raise (ArgumentException("Target is null or empty.", "target")) + + if startIndex + target.Length > str.Length then false + else str.IndexOf(target, startIndex, target.Length) <> -1 + member __.Length = str.Length member this.ContentEquals(sourceText) = diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index 02001acd338..7503f2e05f4 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -13,11 +13,15 @@ type ISourceText = abstract GetLastCharacterPosition : unit -> int * int + abstract GetSubTextString : start: int * length: int -> string + + abstract SubTextEquals : target: string * startIndex: int -> bool + abstract Length : int abstract ContentEquals : ISourceText -> bool - abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit + abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit module SourceText = diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index 09b3b046a4b..a1ce23b8f1c 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -74,6 +74,12 @@ type SourceText with member __.GetLastCharacterPosition() = getLastCharacterPosition.Value + member __.GetSubTextString(start, length) = + this.GetSubText(TextSpan(start, length)).ToString() + + member __.SubTextEquals(target, startIndex) = + this.GetSubText(TextSpan(startIndex, target.Length)).ContentEquals(SourceText.From(target)) + member __.ContentEquals(sourceText) = match sourceText with | :? SourceText as sourceText -> this.ContentEquals(sourceText) From 012806e51be95380c0b1503cc9e864b944463cf8 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 12 Dec 2018 22:46:20 -0800 Subject: [PATCH 12/33] Trying to get tests to pass --- src/fsharp/CheckFormatStrings.fs | 6 +-- .../UnitTests/BraceMatchingServiceTests.fs | 11 ++++-- .../UnitTests/BreakpointResolutionService.fs | 7 +++- .../UnitTests/CompletionProviderTests.fs | 38 ++++++++++++------- .../DocumentDiagnosticAnalyzerTests.fs | 9 ++++- .../DocumentHighlightsServiceTests.fs | 9 ++++- .../UnitTests/EditorFormattingServiceTests.fs | 13 +++++-- .../UnitTests/GoToDefinitionServiceTests.fs | 7 +++- .../UnitTests/IndentationServiceTests.fs | 9 ++++- .../LanguageDebugInfoServiceTests.fs | 8 +++- .../SemanticColorizationServiceTests.fs | 11 ++++-- .../UnitTests/SignatureHelpProviderTests.fs | 7 +++- .../SyntacticColorizationServiceTests.fs | 7 +++- 13 files changed, 105 insertions(+), 37 deletions(-) diff --git a/src/fsharp/CheckFormatStrings.fs b/src/fsharp/CheckFormatStrings.fs index 94a96642eda..37295ac987c 100644 --- a/src/fsharp/CheckFormatStrings.fs +++ b/src/fsharp/CheckFormatStrings.fs @@ -61,10 +61,10 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe let startIndex = lineStartPositions.[m.StartLine-1] + m.StartColumn let endIndex = lineStartPositions.[m.EndLine-1] + m.EndColumn - 1 if startIndex < length-3 && sourceText.SubTextEquals("\"\"\"", startIndex) then - (3, sourceText.GetSubTextString(startIndex + 3, endIndex - startIndex)) //context.SourceText.[startIndex+3..endIndex-3]) + (3, sourceText.GetSubTextString(startIndex + 3, endIndex - startIndex)) elif startIndex < length-2 && sourceText.SubTextEquals("@\"", startIndex) then - (2, sourceText.GetSubTextString(startIndex + 2, endIndex + 1 - startIndex)) //context.SourceText.[startIndex+2..endIndex-1]) - else (1, sourceText.GetSubTextString(startIndex + 1, endIndex - startIndex)) //context.SourceText.[startIndex+1..endIndex-1]) + (2, sourceText.GetSubTextString(startIndex + 2, endIndex + 1 - startIndex)) + else (1, sourceText.GetSubTextString(startIndex + 1, endIndex - startIndex)) else (1, fmt) | None -> (1, fmt) diff --git a/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs b/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs index e323cc8db20..07fd79be3aa 100644 --- a/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs +++ b/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs @@ -32,8 +32,13 @@ type BraceMatchingServiceTests() = Stamp = None } + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + member private this.VerifyNoBraceMatch(fileContents: string, marker: string) = - let sourceText = SourceText.From(fileContents) + let sourceText = SourceText.From(addNewLine fileContents) let position = fileContents.IndexOf(marker) Assert.IsTrue(position >= 0, "Cannot find marker '{0}' in file contents", marker) @@ -43,7 +48,7 @@ type BraceMatchingServiceTests() = | Some(left, right) -> Assert.Fail("Found match for brace '{0}'", marker) member private this.VerifyBraceMatch(fileContents: string, startMarker: string, endMarker: string) = - let sourceText = SourceText.From(fileContents) + let sourceText = SourceText.From(addNewLine fileContents) let startMarkerPosition = fileContents.IndexOf(startMarker) let endMarkerPosition = fileContents.IndexOf(endMarker) @@ -169,7 +174,7 @@ let main argv = []\nlet a7 = 70", [|0;1;22|])>] [] member this.DoNotMatchOnInnerSide(fileContents: string, matchingPositions: int[]) = - let sourceText = SourceText.From(fileContents) + let sourceText = SourceText.From(addNewLine fileContents) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions for position in matchingPositions do diff --git a/vsintegration/tests/UnitTests/BreakpointResolutionService.fs b/vsintegration/tests/UnitTests/BreakpointResolutionService.fs index 655fc02986a..c142a90bf2d 100644 --- a/vsintegration/tests/UnitTests/BreakpointResolutionService.fs +++ b/vsintegration/tests/UnitTests/BreakpointResolutionService.fs @@ -57,6 +57,11 @@ let main argv = 0 // return an integer exit code " + + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source static member private testCases: Object[][] = [| [| "This is a comment"; None |] @@ -73,7 +78,7 @@ let main argv = let searchPosition = code.IndexOf(searchToken) Assert.IsTrue(searchPosition >= 0, "SearchToken '{0}' is not found in code", searchToken) - let sourceText = SourceText.From(code) + let sourceText = SourceText.From(addNewLine code) let searchSpan = TextSpan.FromBounds(searchPosition, searchPosition + searchToken.Length) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions let actualResolutionOption = FSharpBreakpointResolutionService.GetBreakpointLocation(checker, sourceText, fileName, searchSpan, parsingOptions) |> Async.RunSynchronously diff --git a/vsintegration/tests/UnitTests/CompletionProviderTests.fs b/vsintegration/tests/UnitTests/CompletionProviderTests.fs index f73c57ce899..26e485d136f 100644 --- a/vsintegration/tests/UnitTests/CompletionProviderTests.fs +++ b/vsintegration/tests/UnitTests/CompletionProviderTests.fs @@ -53,10 +53,16 @@ let internal projectOptions = { let formatCompletions(completions : string seq) = "\n\t" + String.Join("\n\t", completions) +// Adding this new-line character at the end of the source seems odd but is required for some unit tests +// Todo: fix tests +let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + let VerifyCompletionList(fileContents: string, marker: string, expected: string list, unexpected: string list) = let caretPosition = fileContents.IndexOf(marker) + marker.Length let results = - FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) + // addNewLine - make tests pass + FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(addNewLine fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) |> Async.RunSynchronously |> Option.defaultValue (ResizeArray()) |> Seq.map(fun result -> result.DisplayText) @@ -104,7 +110,8 @@ let VerifyCompletionListExactly(fileContents: string, marker: string, expected: let caretPosition = fileContents.IndexOf(marker) + marker.Length let actual = - FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) + // addNewLine - make tests pass + FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(addNewLine fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) |> Async.RunSynchronously |> Option.defaultValue (ResizeArray()) |> Seq.toList @@ -134,6 +141,11 @@ let ShouldTriggerCompletionAtCorrectMarkers() = ("System.", true) ("Console.", true) ] + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + for (marker: string, shouldBeTriggered: bool) in testCases do let fileContents = """ let x = 1 @@ -144,7 +156,7 @@ System.Console.WriteLine(x + y) let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.AreEqual(shouldBeTriggered, triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should compute the correct result") [] @@ -154,7 +166,7 @@ let ShouldNotTriggerCompletionAfterAnyTriggerOtherThanInsertion() = let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, triggerKind, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, triggerKind, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -163,7 +175,7 @@ let ShouldNotTriggerCompletionInStringLiterals() = let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -177,7 +189,7 @@ System.Console.WriteLine() let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -190,7 +202,7 @@ System.Console.WriteLine() let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -203,7 +215,7 @@ let f() = let caretPosition = fileContents.IndexOf("|.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger on operators") [] @@ -216,7 +228,7 @@ module Foo = module end let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger on Attributes.") [] @@ -229,7 +241,7 @@ printfn "%d" !f let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an identifier that follows a dereference operator (!).") [] @@ -243,7 +255,7 @@ use ptr = fixed &p let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an identifier that follows an addressOf operator (&).") [] @@ -267,7 +279,7 @@ xVal**y let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an identifier that follows a mathematical operation") [] @@ -279,7 +291,7 @@ l""" let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an Insertion character at the top of the file, e.g. a function definition in a new script file.") [] diff --git a/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs b/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs index 99f0380abbc..1f6284c7ee2 100644 --- a/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs +++ b/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs @@ -39,11 +39,16 @@ type DocumentDiagnosticAnalyzerTests() = Stamp = None } + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + let getDiagnostics (fileContents: string) = async { let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions - let! syntacticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Syntax) - let! semanticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Semantic) + let! syntacticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(addNewLine fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Syntax) + let! semanticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(addNewLine fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Semantic) return syntacticDiagnostics.AddRange(semanticDiagnostics) } |> Async.RunSynchronously diff --git a/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs b/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs index 2b39f4749d5..d6ba09d7e06 100644 --- a/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs +++ b/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs @@ -63,6 +63,11 @@ let private span sourceText isDefinition (startLine, startCol) (endLine, endCol) { IsDefinition = isDefinition TextSpan = RoslynHelpers.FSharpRangeToTextSpan(sourceText, range) } +// Adding this new-line character at the end of the source seems odd but is required for some unit tests +// Todo: fix tests +let private addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + [] let ShouldHighlightAllSimpleLocalSymbolReferences() = let fileContents = """ @@ -70,7 +75,7 @@ let ShouldHighlightAllSimpleLocalSymbolReferences() = x + x let y = foo 2 """ - let sourceText = SourceText.From(fileContents) + let sourceText = SourceText.From(addNewLine fileContents) let caretPosition = fileContents.IndexOf("foo") + 1 let spans = getSpans sourceText caretPosition @@ -86,7 +91,7 @@ let ShouldHighlightAllQualifiedSymbolReferences() = let x = System.DateTime.Now let y = System.DateTime.MaxValue """ - let sourceText = SourceText.From(fileContents) + let sourceText = SourceText.From(addNewLine fileContents) let caretPosition = fileContents.IndexOf("DateTime") + 1 let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) diff --git a/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs b/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs index c1d043377d5..abf61785ba5 100644 --- a/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs +++ b/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs @@ -64,6 +64,11 @@ marker2 marker3 marker4""" + + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source [] [] @@ -74,7 +79,7 @@ marker4""" let position = indentTemplate.IndexOf(marker) Assert.IsTrue(position >= 0, "Precondition failed: unable to find marker in template") - let sourceText = SourceText.From(indentTemplate) + let sourceText = SourceText.From(addNewLine indentTemplate) let lineNumber = sourceText.Lines |> Seq.findIndex (fun line -> line.Span.Contains position) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions @@ -117,7 +122,7 @@ let foo = somethingElseHere """ - let sourceText = SourceText.From(start.Replace("$", clipboard)) + let sourceText = SourceText.From(start.Replace("$", clipboard) |> addNewLine) let span = TextSpan(start.IndexOf '$', clipboard.Length) let formattingOptions = { FormatOnPaste = enabled } @@ -166,7 +171,7 @@ let foo = somethingElseHere """ - let sourceText = SourceText.From(start.Replace("$", clipboard)) + let sourceText = SourceText.From(start.Replace("$", clipboard) |> addNewLine) let span = TextSpan(start.IndexOf '$', clipboard.Length) let formattingOptions = { FormatOnPaste = true } @@ -211,7 +216,7 @@ let foo = somethingElseHere """ - let sourceText = SourceText.From(start.Replace("$", clipboard)) + let sourceText = SourceText.From(start.Replace("$", clipboard) |> addNewLine) // If we're pasting on an empty line which has been automatically indented, // then the pasted span includes this automatic indentation. Check that we diff --git a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs index d442b9a8963..dd4e8072973 100644 --- a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs +++ b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs @@ -96,6 +96,11 @@ let _ = Module1.foo 1 [ ("let _ = Module", Some (2, 2, 7, 14)) ]) ] + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + for fileContents, testCases in manyTestCases do for caretMarker, expected in testCases do @@ -121,7 +126,7 @@ let _ = Module1.foo 1 let caretPosition = fileContents.IndexOf(caretMarker) + caretMarker.Length - 1 // inside the marker let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let actual = - findDefinition(checker, documentId, SourceText.From(fileContents), filePath, caretPosition, [], options, 0) + findDefinition(checker, documentId, SourceText.From(addNewLine fileContents), filePath, caretPosition, [], options, 0) |> Option.map (fun range -> (range.StartLine, range.EndLine, range.StartColumn, range.EndColumn)) if actual <> expected then diff --git a/vsintegration/tests/UnitTests/IndentationServiceTests.fs b/vsintegration/tests/UnitTests/IndentationServiceTests.fs index 5139e313a48..8633274be5b 100644 --- a/vsintegration/tests/UnitTests/IndentationServiceTests.fs +++ b/vsintegration/tests/UnitTests/IndentationServiceTests.fs @@ -170,10 +170,15 @@ while true do |> Array.map (fun (lineNumber, expectedIndentation) -> ( Some(expectedIndentation), lineNumber, autoIndentTemplate )) + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + [] member this.TestIndentation() = for (expectedIndentation, lineNumber, template) in testCases do - let sourceText = SourceText.From(template) + let sourceText = SourceText.From(addNewLine template) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions let actualIndentation = FSharpIndentationService.GetDesiredIndentation(documentId, sourceText, filePath, lineNumber, tabSize, indentStyle, Some (parsingOptions, projectOptions)) @@ -186,7 +191,7 @@ while true do for (expectedIndentation, lineNumber, template) in autoIndentTestCases do - let sourceText = SourceText.From(template) + let sourceText = SourceText.From(addNewLine template) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions let actualIndentation = FSharpIndentationService.GetDesiredIndentation(documentId, sourceText, filePath, lineNumber, tabSize, indentStyle, Some (parsingOptions, projectOptions)) diff --git a/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs b/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs index 7578f34bb9f..ee0baff43a6 100644 --- a/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs +++ b/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs @@ -39,6 +39,12 @@ let main argv = 0 // return an integer exit code " + + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + static member private testCases: Object[][] = [| [| "123456"; None |] // Numeric literals are not interesting [| "is a string"; Some("\"This is a string\"") |] @@ -52,7 +58,7 @@ let main argv = let searchPosition = code.IndexOf(searchToken) Assert.IsTrue(searchPosition >= 0, "SearchToken '{0}' is not found in code", searchToken) - let sourceText = SourceText.From(code) + let sourceText = SourceText.From(addNewLine code) let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let classifiedSpans = Tokenizer.getClassifiedSpans(documentId, sourceText, TextSpan.FromBounds(0, sourceText.Length), Some(fileName), defines, CancellationToken.None) let actualDataTipSpanOption = FSharpLanguageDebugInfoService.GetDataTipInformation(sourceText, searchPosition, classifiedSpans) diff --git a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs index 4bac0dff22a..18561e8e11a 100644 --- a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs @@ -27,13 +27,18 @@ type SemanticClassificationServiceTests() = Stamp = None } + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + let checker = FSharpChecker.Create() let perfOptions = { LanguageServicePerformanceOptions.Default with AllowStaleCompletionResults = false } let getRanges (sourceText: string) : (Range.range * SemanticClassificationType) list = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(sourceText), projectOptions, perfOptions, "") + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(addNewLine sourceText), projectOptions, perfOptions, "") return checkFileResults.GetSemanticClassification(None) } |> Async.RunSynchronously @@ -41,7 +46,7 @@ type SemanticClassificationServiceTests() = |> List.collect Array.toList let verifyClassificationAtEndOfMarker(fileContents: string, marker: string, classificationType: string) = - let text = SourceText.From fileContents + let text = SourceText.From(addNewLine fileContents) let ranges = getRanges fileContents let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1) let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1) @@ -50,7 +55,7 @@ type SemanticClassificationServiceTests() = | Some(_, ty) -> Assert.AreEqual(classificationType, FSharpClassificationTypes.getClassificationTypeName ty, "Classification data doesn't match for end of marker") let verifyNoClassificationDataAtEndOfMarker(fileContents: string, marker: string, classificationType: string) = - let text = SourceText.From fileContents + let text = SourceText.From(addNewLine fileContents) let ranges = getRanges fileContents let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1) let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1) diff --git a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs index ee651357715..a3921cca25d 100644 --- a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs +++ b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs @@ -129,6 +129,11 @@ type foo5 = N1.T [("let _ = System.DateTime(", Some ("[8..24)", 0, 0, None)) ]) ] + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + let sb = StringBuilder() for (fileContents, testCases) in manyTestCases do printfn "Test case: fileContents = %s..." fileContents.[2..4] @@ -146,7 +151,7 @@ type foo5 = N1.T } let triggerChar = if marker = "," then Some ',' elif marker = "(" then Some '(' elif marker = "<" then Some '<' else None - let triggered = FSharpSignatureHelpProvider.ProvideMethodsAsyncAux(checker, documentationProvider, SourceText.From(fileContents), caretPosition, projectOptions, triggerChar, filePath, 0) |> Async.RunSynchronously + let triggered = FSharpSignatureHelpProvider.ProvideMethodsAsyncAux(checker, documentationProvider, SourceText.From(addNewLine fileContents), caretPosition, projectOptions, triggerChar, filePath, 0) |> Async.RunSynchronously checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() let actual = match triggered with diff --git a/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs index 05b0d39e7aa..c1c35618801 100644 --- a/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs @@ -14,11 +14,16 @@ open Microsoft.VisualStudio.FSharp.Editor [][] type SyntacticClassificationServiceTests() = + // Adding this new-line character at the end of the source seems odd but is required for some unit tests + // Todo: fix tests + let addNewLine (source: string) = + if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source + member private this.ExtractMarkerData(fileContents: string, marker: string, defines: string list, isScriptFile: Option) = let textSpan = TextSpan(0, fileContents.Length) let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) - let tokens = Tokenizer.getClassifiedSpans(documentId, SourceText.From(fileContents), textSpan, Some(fileName), defines, CancellationToken.None) + let tokens = Tokenizer.getClassifiedSpans(documentId, SourceText.From(addNewLine fileContents), textSpan, Some(fileName), defines, CancellationToken.None) let markerPosition = fileContents.IndexOf(marker) Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) (tokens, markerPosition) From c8ea8f4056c80e22c792d0bd60d62563e8a57a95 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 13 Dec 2018 01:18:04 -0800 Subject: [PATCH 13/33] Trying to fix tests --- .../tests/UnitTests/SignatureHelpProviderTests.fs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs index a3921cca25d..ee651357715 100644 --- a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs +++ b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs @@ -129,11 +129,6 @@ type foo5 = N1.T [("let _ = System.DateTime(", Some ("[8..24)", 0, 0, None)) ]) ] - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - let sb = StringBuilder() for (fileContents, testCases) in manyTestCases do printfn "Test case: fileContents = %s..." fileContents.[2..4] @@ -151,7 +146,7 @@ type foo5 = N1.T } let triggerChar = if marker = "," then Some ',' elif marker = "(" then Some '(' elif marker = "<" then Some '<' else None - let triggered = FSharpSignatureHelpProvider.ProvideMethodsAsyncAux(checker, documentationProvider, SourceText.From(addNewLine fileContents), caretPosition, projectOptions, triggerChar, filePath, 0) |> Async.RunSynchronously + let triggered = FSharpSignatureHelpProvider.ProvideMethodsAsyncAux(checker, documentationProvider, SourceText.From(fileContents), caretPosition, projectOptions, triggerChar, filePath, 0) |> Async.RunSynchronously checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() let actual = match triggered with From 4714be2c0a72a08ad060136e5a7dcc4dce9e5c2c Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 13 Dec 2018 15:46:52 -0800 Subject: [PATCH 14/33] Ignoring test --- vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs index ee651357715..18cf31ad2cd 100644 --- a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs +++ b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs @@ -49,7 +49,7 @@ let internal projectOptions = { Stamp = None } -[] +[] let ShouldGiveSignatureHelpAtCorrectMarkers() = let manyTestCases = [ (""" From dbc89560ee75565e1d8d6a6cba9baeb2ab79b71b Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 13 Dec 2018 18:00:45 -0800 Subject: [PATCH 15/33] unignoring test --- src/fsharp/pars.fsy | 6 +++--- src/fsharp/service/ServiceParamInfoLocations.fs | 2 +- src/fsharp/service/ServiceParseTreeWalk.fs | 2 ++ vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/fsharp/pars.fsy b/src/fsharp/pars.fsy index 9e75787d29f..de42111b414 100644 --- a/src/fsharp/pars.fsy +++ b/src/fsharp/pars.fsy @@ -3857,9 +3857,9 @@ parenExpr: arbExpr("parenExpr2rbcs", lhsm) } | LPAREN OBLOCKEND_COMING_SOON - { reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnmatchedParen()) - let lhsm = unionRangeWithPos (rhs parseState 1) (rhs parseState 2).Start - arbExpr("parenExpr2obecs", lhsm) } + { let lparenRange = (rhs parseState 1) + reportParseErrorAt lparenRange (FSComp.SR.parsUnmatchedParen()) + SynExpr.Paren(arbExpr("parenExpr2obecs", lparenRange.EndRange), lparenRange, None, lparenRange) } | LPAREN recover %prec prec_atomexpr_lparen_error { reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnmatchedParen()) diff --git a/src/fsharp/service/ServiceParamInfoLocations.fs b/src/fsharp/service/ServiceParamInfoLocations.fs index e0e9affd2d5..7295e2e08e6 100755 --- a/src/fsharp/service/ServiceParamInfoLocations.fs +++ b/src/fsharp/service/ServiceParamInfoLocations.fs @@ -83,7 +83,7 @@ module internal NoteworthyParamInfoLocationsImpl = let inner = traverseSynExpr synExpr match inner with | None -> - if AstTraversal.rangeContainsPosEdgesExclusive parenRange pos then + if AstTraversal.rangeContainsPosLeftEdgeExclusiveAndRightEdgeInclusive parenRange pos then Found (parenRange.Start, [(parenRange.End, getNamedParamName synExpr)], rpRangeOpt.IsSome), None else NotFound, None diff --git a/src/fsharp/service/ServiceParseTreeWalk.fs b/src/fsharp/service/ServiceParseTreeWalk.fs index 660528e03ee..f84f84bb993 100755 --- a/src/fsharp/service/ServiceParseTreeWalk.fs +++ b/src/fsharp/service/ServiceParseTreeWalk.fs @@ -27,6 +27,8 @@ module public AstTraversal = // treat ranges as though they are fully open: (,) let rangeContainsPosEdgesExclusive (m1:range) p = posGt p m1.Start && posGt m1.End p + let rangeContainsPosLeftEdgeExclusiveAndRightEdgeInclusive (m1:range) p = posGt p m1.Start && posGeq m1.End p + /// used to track route during traversal AST [] type TraverseStep = diff --git a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs index 18cf31ad2cd..ee651357715 100644 --- a/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs +++ b/vsintegration/tests/UnitTests/SignatureHelpProviderTests.fs @@ -49,7 +49,7 @@ let internal projectOptions = { Stamp = None } -[] +[] let ShouldGiveSignatureHelpAtCorrectMarkers() = let manyTestCases = [ (""" From 069c926ee3c7b6471bff1f30b38d1615cde75e85 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 13 Dec 2018 20:25:44 -0800 Subject: [PATCH 16/33] Fixed weak table --- .../src/FSharp.Editor/Common/Extensions.fs | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index a1ce23b8f1c..a8ef14e91c7 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -48,52 +48,50 @@ module private SourceText = let weakTable = ConditionalWeakTable() -type SourceText with - - member this.ToFSharpSourceText() = - match SourceText.weakTable.TryGetValue(this) with - | true, sourceText -> sourceText - | _ -> - let getLines = - lazy - [| - for i = 0 to this.Lines.Count - 1 do - yield this.Lines.[i].ToString() - |] - - let getLastCharacterPosition = - lazy - (this.Lines.Count, this.Lines.[this.Lines.Count - 1].Span.Length) - - let sourceText = - { new ISourceText with + let create (sourceText: SourceText) = + let getLines = + lazy + [| + for i = 0 to sourceText.Lines.Count - 1 do + yield sourceText.Lines.[i].ToString() + |] + + let getLastCharacterPosition = + lazy + (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) + + let sourceText = + { new ISourceText with - member __.Item with get index = this.[index] + member __.Item with get index = sourceText.[index] - member __.GetLines() = getLines.Value + member __.GetLines() = getLines.Value - member __.GetLastCharacterPosition() = getLastCharacterPosition.Value + member __.GetLastCharacterPosition() = getLastCharacterPosition.Value - member __.GetSubTextString(start, length) = - this.GetSubText(TextSpan(start, length)).ToString() + member __.GetSubTextString(start, length) = + sourceText.GetSubText(TextSpan(start, length)).ToString() - member __.SubTextEquals(target, startIndex) = - this.GetSubText(TextSpan(startIndex, target.Length)).ContentEquals(SourceText.From(target)) + member __.SubTextEquals(target, startIndex) = + sourceText.GetSubText(TextSpan(startIndex, target.Length)).ContentEquals(SourceText.From(target)) - member __.ContentEquals(sourceText) = - match sourceText with - | :? SourceText as sourceText -> this.ContentEquals(sourceText) - | _ -> false + member __.ContentEquals(sourceText) = + match sourceText with + | :? SourceText as sourceText -> sourceText.ContentEquals(sourceText) + | _ -> false - member __.Length = this.Length + member __.Length = sourceText.Length - member __.CopyTo(sourceIndex, destination, destinationIndex, count) = - this.CopyTo(sourceIndex, destination, destinationIndex, count) - } + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + sourceText.CopyTo(sourceIndex, destination, destinationIndex, count) + } - SourceText.weakTable.Add(this, sourceText) + sourceText - sourceText +type SourceText with + + member this.ToFSharpSourceText() = + SourceText.weakTable.GetValue(this, Runtime.CompilerServices.ConditionalWeakTable<_,_>.CreateValueCallback(SourceText.create)) type FSharpNavigationDeclarationItem with member x.RoslynGlyph : Glyph = From 95d04937b7bf9239cbe9b459f9e1e7eb24f67b1c Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 13 Dec 2018 20:55:02 -0800 Subject: [PATCH 17/33] Removing addNewLine in sourcetext --- src/utils/prim-lexing.fs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index a336bed57fa..a1d77524dba 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -81,12 +81,7 @@ type StringText(str: string) = module SourceText = - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let private addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - - let ofString str = StringText(addNewLine str) :> ISourceText + let ofString str = StringText(str) :> ISourceText namespace Internal.Utilities.Text.Lexing From fd6486cf0ddbde2176a6fe344dc72ff09ee29b0f Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 13 Dec 2018 22:24:52 -0800 Subject: [PATCH 18/33] Fixing interactive checker tests --- tests/service/InteractiveCheckerTests.fs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/service/InteractiveCheckerTests.fs b/tests/service/InteractiveCheckerTests.fs index fba9adcb639..f3ee91b0f12 100644 --- a/tests/service/InteractiveCheckerTests.fs +++ b/tests/service/InteractiveCheckerTests.fs @@ -45,8 +45,11 @@ let internal identsAndRanges (input: Ast.ParsedInput) = | Ast.SynModuleDecl.Attributes(_attrs, _range) -> failwith "Not implemented yet" | Ast.SynModuleDecl.HashDirective(_, _range) -> failwith "Not implemented yet" | Ast.SynModuleDecl.NamespaceFragment(moduleOrNamespace) -> extractFromModuleOrNamespace moduleOrNamespace - and extractFromModuleOrNamespace (Ast.SynModuleOrNamespace(longIdent, _, _, moduleDecls, _, _, _, range)) = - (identAndRange (longIdentToString longIdent) range) :: (moduleDecls |> List.collect extractFromModuleDecl) + and extractFromModuleOrNamespace (Ast.SynModuleOrNamespace(longIdent, _, _, moduleDecls, _, _, _, _)) = + let xs = moduleDecls |> List.collect extractFromModuleDecl + if longIdent.IsEmpty then xs + else + (identAndRange (longIdentToString longIdent) (longIdent |> List.map (fun id -> id.idRange) |> List.reduce Range.unionRanges)) :: xs match input with | Ast.ParsedInput.ImplFile(Ast.ParsedImplFileInput(_, _, _, _, _, modulesOrNamespaces, _)) -> @@ -71,7 +74,7 @@ let input = let ``Test ranges - namespace`` () = let res = parseAndExtractRanges input printfn "Test ranges - namespace, res = %A" res - res |> shouldEqual [("N", ((4, 4), (6, 0))); ("Sample", ((4, 9), (4, 15)))] + res |> shouldEqual [("N", ((2, 14), (2, 15))); ("Sample", ((4, 9), (4, 15)))] let input2 = """ @@ -84,7 +87,7 @@ let input2 = let ``Test ranges - module`` () = let res = parseAndExtractRanges input2 printfn "Test ranges - module, res = %A" res - res |> shouldEqual [("M", ((2, 4), (4, 26))); ("Sample", ((4, 9), (4, 15)))] + res |> shouldEqual [("M", ((2, 11), (2, 12))); ("Sample", ((4, 9), (4, 15)))] let input3 = """ @@ -97,4 +100,4 @@ let input3 = let ``Test ranges - global namespace`` () = let res = parseAndExtractRanges input3 printfn "Test ranges - global namespace, res = %A" res - res |> shouldEqual [("", ((4, 4), (6, 0))); ("Sample", ((4, 9), (4, 15)))] + res |> shouldEqual [("Sample", ((4, 9), (4, 15)))] From d9d2c57a3a1304367065214bb4e574ed27d1166a Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 13 Dec 2018 22:41:49 -0800 Subject: [PATCH 19/33] Fixing more tests --- src/utils/prim-lexing.fs | 5 ++++- tests/service/StructureTests.fs | 2 +- vsintegration/src/FSharp.Editor/Common/Extensions.fs | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index a1d77524dba..c3bb53a23b8 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -52,7 +52,10 @@ type StringText(str: string) = member __.GetLastCharacterPosition() = let lines = getLines.Value - (lines.Length, lines.[lines.Length - 1].Length) + if lines.Length > 0 then + (lines.Length, lines.[lines.Length - 1].Length) + else + (0, 0) member __.GetLines() = getLines.Value diff --git a/tests/service/StructureTests.fs b/tests/service/StructureTests.fs index d87c3c18e78..9229ab6ccb5 100644 --- a/tests/service/StructureTests.fs +++ b/tests/service/StructureTests.fs @@ -60,7 +60,7 @@ let (=>) (source: string) (expectedRanges: (Range * Range) list) = reraise() [] -let ``empty file``() = "" => [ (1, 0, 2, 0), (1, 0, 2, 0) ] +let ``empty file``() = "" => [ ] [] let ``nested module``() = diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index a8ef14e91c7..d3e3294f1c2 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -58,7 +58,10 @@ module private SourceText = let getLastCharacterPosition = lazy - (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) + if sourceText.Lines.Count > 0 then + (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) + else + (0, 0) let sourceText = { new ISourceText with From a1c503177868a7364ffe05945413dea94f470cad Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 10:08:25 -0800 Subject: [PATCH 20/33] Removed addNewLine --- .../UnitTests/SemanticColorizationServiceTests.fs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs index 18561e8e11a..7fdacfe7489 100644 --- a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs @@ -27,18 +27,13 @@ type SemanticClassificationServiceTests() = Stamp = None } - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - let checker = FSharpChecker.Create() let perfOptions = { LanguageServicePerformanceOptions.Default with AllowStaleCompletionResults = false } let getRanges (sourceText: string) : (Range.range * SemanticClassificationType) list = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(addNewLine sourceText), projectOptions, perfOptions, "") + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(sourceText), projectOptions, perfOptions, "") return checkFileResults.GetSemanticClassification(None) } |> Async.RunSynchronously @@ -46,7 +41,7 @@ type SemanticClassificationServiceTests() = |> List.collect Array.toList let verifyClassificationAtEndOfMarker(fileContents: string, marker: string, classificationType: string) = - let text = SourceText.From(addNewLine fileContents) + let text = SourceText.From(fileContents) let ranges = getRanges fileContents let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1) let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1) @@ -55,7 +50,7 @@ type SemanticClassificationServiceTests() = | Some(_, ty) -> Assert.AreEqual(classificationType, FSharpClassificationTypes.getClassificationTypeName ty, "Classification data doesn't match for end of marker") let verifyNoClassificationDataAtEndOfMarker(fileContents: string, marker: string, classificationType: string) = - let text = SourceText.From(addNewLine fileContents) + let text = SourceText.From(fileContents) let ranges = getRanges fileContents let line = text.Lines.GetLinePosition (fileContents.IndexOf(marker) + marker.Length - 1) let markerPos = Range.mkPos (Range.Line.fromZ line.Line) (line.Character + marker.Length - 1) From 3fcfdcd66f13ec98d8d72bbac16524a31e81ee85 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 10:11:59 -0800 Subject: [PATCH 21/33] Removed addNewLine --- .../tests/UnitTests/SyntacticColorizationServiceTests.fs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs index c1c35618801..05b0d39e7aa 100644 --- a/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SyntacticColorizationServiceTests.fs @@ -14,16 +14,11 @@ open Microsoft.VisualStudio.FSharp.Editor [][] type SyntacticClassificationServiceTests() = - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - member private this.ExtractMarkerData(fileContents: string, marker: string, defines: string list, isScriptFile: Option) = let textSpan = TextSpan(0, fileContents.Length) let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) - let tokens = Tokenizer.getClassifiedSpans(documentId, SourceText.From(addNewLine fileContents), textSpan, Some(fileName), defines, CancellationToken.None) + let tokens = Tokenizer.getClassifiedSpans(documentId, SourceText.From(fileContents), textSpan, Some(fileName), defines, CancellationToken.None) let markerPosition = fileContents.IndexOf(marker) Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) (tokens, markerPosition) From 8892081e2b440ff17e6de3b524f7fbad6f1e94ab Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 10:25:20 -0800 Subject: [PATCH 22/33] Removed addNewLine --- .../tests/UnitTests/BraceMatchingServiceTests.fs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs b/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs index 07fd79be3aa..e323cc8db20 100644 --- a/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs +++ b/vsintegration/tests/UnitTests/BraceMatchingServiceTests.fs @@ -32,13 +32,8 @@ type BraceMatchingServiceTests() = Stamp = None } - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - member private this.VerifyNoBraceMatch(fileContents: string, marker: string) = - let sourceText = SourceText.From(addNewLine fileContents) + let sourceText = SourceText.From(fileContents) let position = fileContents.IndexOf(marker) Assert.IsTrue(position >= 0, "Cannot find marker '{0}' in file contents", marker) @@ -48,7 +43,7 @@ type BraceMatchingServiceTests() = | Some(left, right) -> Assert.Fail("Found match for brace '{0}'", marker) member private this.VerifyBraceMatch(fileContents: string, startMarker: string, endMarker: string) = - let sourceText = SourceText.From(addNewLine fileContents) + let sourceText = SourceText.From(fileContents) let startMarkerPosition = fileContents.IndexOf(startMarker) let endMarkerPosition = fileContents.IndexOf(endMarker) @@ -174,7 +169,7 @@ let main argv = []\nlet a7 = 70", [|0;1;22|])>] [] member this.DoNotMatchOnInnerSide(fileContents: string, matchingPositions: int[]) = - let sourceText = SourceText.From(addNewLine fileContents) + let sourceText = SourceText.From(fileContents) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions for position in matchingPositions do From 2519591c1c1dd34b35282b9d96207178926a6848 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 10:28:56 -0800 Subject: [PATCH 23/33] Removed addNewLine --- vsintegration/tests/UnitTests/IndentationServiceTests.fs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/vsintegration/tests/UnitTests/IndentationServiceTests.fs b/vsintegration/tests/UnitTests/IndentationServiceTests.fs index 8633274be5b..5139e313a48 100644 --- a/vsintegration/tests/UnitTests/IndentationServiceTests.fs +++ b/vsintegration/tests/UnitTests/IndentationServiceTests.fs @@ -170,15 +170,10 @@ while true do |> Array.map (fun (lineNumber, expectedIndentation) -> ( Some(expectedIndentation), lineNumber, autoIndentTemplate )) - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - [] member this.TestIndentation() = for (expectedIndentation, lineNumber, template) in testCases do - let sourceText = SourceText.From(addNewLine template) + let sourceText = SourceText.From(template) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions let actualIndentation = FSharpIndentationService.GetDesiredIndentation(documentId, sourceText, filePath, lineNumber, tabSize, indentStyle, Some (parsingOptions, projectOptions)) @@ -191,7 +186,7 @@ while true do for (expectedIndentation, lineNumber, template) in autoIndentTestCases do - let sourceText = SourceText.From(addNewLine template) + let sourceText = SourceText.From(template) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions let actualIndentation = FSharpIndentationService.GetDesiredIndentation(documentId, sourceText, filePath, lineNumber, tabSize, indentStyle, Some (parsingOptions, projectOptions)) From 31f5850525b27998ef8665d2761fd5731721bbe7 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 10:59:21 -0800 Subject: [PATCH 24/33] Removed addNewLine --- .../tests/UnitTests/BreakpointResolutionService.fs | 7 +------ .../tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs | 9 ++------- .../tests/UnitTests/LanguageDebugInfoServiceTests.fs | 7 +------ 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/vsintegration/tests/UnitTests/BreakpointResolutionService.fs b/vsintegration/tests/UnitTests/BreakpointResolutionService.fs index c142a90bf2d..655fc02986a 100644 --- a/vsintegration/tests/UnitTests/BreakpointResolutionService.fs +++ b/vsintegration/tests/UnitTests/BreakpointResolutionService.fs @@ -57,11 +57,6 @@ let main argv = 0 // return an integer exit code " - - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source static member private testCases: Object[][] = [| [| "This is a comment"; None |] @@ -78,7 +73,7 @@ let main argv = let searchPosition = code.IndexOf(searchToken) Assert.IsTrue(searchPosition >= 0, "SearchToken '{0}' is not found in code", searchToken) - let sourceText = SourceText.From(addNewLine code) + let sourceText = SourceText.From(code) let searchSpan = TextSpan.FromBounds(searchPosition, searchPosition + searchToken.Length) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions let actualResolutionOption = FSharpBreakpointResolutionService.GetBreakpointLocation(checker, sourceText, fileName, searchSpan, parsingOptions) |> Async.RunSynchronously diff --git a/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs b/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs index 1f6284c7ee2..99f0380abbc 100644 --- a/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs +++ b/vsintegration/tests/UnitTests/DocumentDiagnosticAnalyzerTests.fs @@ -39,16 +39,11 @@ type DocumentDiagnosticAnalyzerTests() = Stamp = None } - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - let getDiagnostics (fileContents: string) = async { let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions - let! syntacticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(addNewLine fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Syntax) - let! semanticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(addNewLine fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Semantic) + let! syntacticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Syntax) + let! semanticDiagnostics = FSharpDocumentDiagnosticAnalyzer.GetDiagnostics(checker, filePath, SourceText.From(fileContents), 0, parsingOptions, projectOptions, DiagnosticsType.Semantic) return syntacticDiagnostics.AddRange(semanticDiagnostics) } |> Async.RunSynchronously diff --git a/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs b/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs index ee0baff43a6..93c135ad8bf 100644 --- a/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs +++ b/vsintegration/tests/UnitTests/LanguageDebugInfoServiceTests.fs @@ -40,11 +40,6 @@ let main argv = 0 // return an integer exit code " - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - static member private testCases: Object[][] = [| [| "123456"; None |] // Numeric literals are not interesting [| "is a string"; Some("\"This is a string\"") |] @@ -58,7 +53,7 @@ let main argv = let searchPosition = code.IndexOf(searchToken) Assert.IsTrue(searchPosition >= 0, "SearchToken '{0}' is not found in code", searchToken) - let sourceText = SourceText.From(addNewLine code) + let sourceText = SourceText.From(code) let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let classifiedSpans = Tokenizer.getClassifiedSpans(documentId, sourceText, TextSpan.FromBounds(0, sourceText.Length), Some(fileName), defines, CancellationToken.None) let actualDataTipSpanOption = FSharpLanguageDebugInfoService.GetDataTipInformation(sourceText, searchPosition, classifiedSpans) From dde3ef3f8bd547ab28773c0950860885524f2e3e Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 12:46:09 -0800 Subject: [PATCH 25/33] Removed addNewLine --- .../UnitTests/CompletionProviderTests.fs | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/vsintegration/tests/UnitTests/CompletionProviderTests.fs b/vsintegration/tests/UnitTests/CompletionProviderTests.fs index 26e485d136f..f0a4f6a20ab 100644 --- a/vsintegration/tests/UnitTests/CompletionProviderTests.fs +++ b/vsintegration/tests/UnitTests/CompletionProviderTests.fs @@ -53,16 +53,10 @@ let internal projectOptions = { let formatCompletions(completions : string seq) = "\n\t" + String.Join("\n\t", completions) -// Adding this new-line character at the end of the source seems odd but is required for some unit tests -// Todo: fix tests -let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - let VerifyCompletionList(fileContents: string, marker: string, expected: string list, unexpected: string list) = let caretPosition = fileContents.IndexOf(marker) + marker.Length let results = - // addNewLine - make tests pass - FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(addNewLine fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) + FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) |> Async.RunSynchronously |> Option.defaultValue (ResizeArray()) |> Seq.map(fun result -> result.DisplayText) @@ -110,8 +104,7 @@ let VerifyCompletionListExactly(fileContents: string, marker: string, expected: let caretPosition = fileContents.IndexOf(marker) + marker.Length let actual = - // addNewLine - make tests pass - FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(addNewLine fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) + FSharpCompletionProvider.ProvideCompletionsAsyncAux(checker, SourceText.From(fileContents), caretPosition, projectOptions, filePath, 0, (fun _ -> []), LanguageServicePerformanceOptions.Default, IntelliSenseOptions.Default) |> Async.RunSynchronously |> Option.defaultValue (ResizeArray()) |> Seq.toList @@ -166,7 +159,7 @@ let ShouldNotTriggerCompletionAfterAnyTriggerOtherThanInsertion() = let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, triggerKind, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, triggerKind, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -175,7 +168,7 @@ let ShouldNotTriggerCompletionInStringLiterals() = let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -189,7 +182,7 @@ System.Console.WriteLine() let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -202,7 +195,7 @@ System.Console.WriteLine() let caretPosition = fileContents.IndexOf("System.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger") [] @@ -215,7 +208,7 @@ let f() = let caretPosition = fileContents.IndexOf("|.") let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsFalse(triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should not trigger on operators") [] @@ -228,7 +221,7 @@ module Foo = module end let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger on Attributes.") [] @@ -241,7 +234,7 @@ printfn "%d" !f let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an identifier that follows a dereference operator (!).") [] @@ -255,7 +248,7 @@ use ptr = fixed &p let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an identifier that follows an addressOf operator (&).") [] @@ -279,7 +272,7 @@ xVal**y let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an identifier that follows a mathematical operation") [] @@ -291,7 +284,7 @@ l""" let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.IsTrue(triggered, "Completion should trigger after typing an Insertion character at the top of the file, e.g. a function definition in a new script file.") [] From eac57f8eb4ab3203f9b556b9d70f0a2c2eb3f525 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 12:53:19 -0800 Subject: [PATCH 26/33] Removed addNewLine --- vsintegration/tests/UnitTests/CompletionProviderTests.fs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vsintegration/tests/UnitTests/CompletionProviderTests.fs b/vsintegration/tests/UnitTests/CompletionProviderTests.fs index f0a4f6a20ab..f73c57ce899 100644 --- a/vsintegration/tests/UnitTests/CompletionProviderTests.fs +++ b/vsintegration/tests/UnitTests/CompletionProviderTests.fs @@ -134,11 +134,6 @@ let ShouldTriggerCompletionAtCorrectMarkers() = ("System.", true) ("Console.", true) ] - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - for (marker: string, shouldBeTriggered: bool) in testCases do let fileContents = """ let x = 1 @@ -149,7 +144,7 @@ System.Console.WriteLine(x + y) let caretPosition = fileContents.IndexOf(marker) + marker.Length let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let getInfo() = documentId, filePath, [] - let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(addNewLine fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) + let triggered = FSharpCompletionProvider.ShouldTriggerCompletionAux(SourceText.From(fileContents), caretPosition, CompletionTriggerKind.Insertion, getInfo, IntelliSenseOptions.Default) Assert.AreEqual(shouldBeTriggered, triggered, "FSharpCompletionProvider.ShouldTriggerCompletionAux() should compute the correct result") [] From 641b070d039bb3cc08f708daa8ba33d0b34bec44 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 12:55:31 -0800 Subject: [PATCH 27/33] Removed addNewLine --- .../tests/UnitTests/GoToDefinitionServiceTests.fs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs index dd4e8072973..d442b9a8963 100644 --- a/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs +++ b/vsintegration/tests/UnitTests/GoToDefinitionServiceTests.fs @@ -96,11 +96,6 @@ let _ = Module1.foo 1 [ ("let _ = Module", Some (2, 2, 7, 14)) ]) ] - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - for fileContents, testCases in manyTestCases do for caretMarker, expected in testCases do @@ -126,7 +121,7 @@ let _ = Module1.foo 1 let caretPosition = fileContents.IndexOf(caretMarker) + caretMarker.Length - 1 // inside the marker let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) let actual = - findDefinition(checker, documentId, SourceText.From(addNewLine fileContents), filePath, caretPosition, [], options, 0) + findDefinition(checker, documentId, SourceText.From(fileContents), filePath, caretPosition, [], options, 0) |> Option.map (fun range -> (range.StartLine, range.EndLine, range.StartColumn, range.EndColumn)) if actual <> expected then From 160984def57cf514504f874906c68e3d0e56e7a5 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 13:04:17 -0800 Subject: [PATCH 28/33] Removed addNewLine --- .../tests/UnitTests/DocumentHighlightsServiceTests.fs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs b/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs index d6ba09d7e06..2b39f4749d5 100644 --- a/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs +++ b/vsintegration/tests/UnitTests/DocumentHighlightsServiceTests.fs @@ -63,11 +63,6 @@ let private span sourceText isDefinition (startLine, startCol) (endLine, endCol) { IsDefinition = isDefinition TextSpan = RoslynHelpers.FSharpRangeToTextSpan(sourceText, range) } -// Adding this new-line character at the end of the source seems odd but is required for some unit tests -// Todo: fix tests -let private addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source - [] let ShouldHighlightAllSimpleLocalSymbolReferences() = let fileContents = """ @@ -75,7 +70,7 @@ let ShouldHighlightAllSimpleLocalSymbolReferences() = x + x let y = foo 2 """ - let sourceText = SourceText.From(addNewLine fileContents) + let sourceText = SourceText.From(fileContents) let caretPosition = fileContents.IndexOf("foo") + 1 let spans = getSpans sourceText caretPosition @@ -91,7 +86,7 @@ let ShouldHighlightAllQualifiedSymbolReferences() = let x = System.DateTime.Now let y = System.DateTime.MaxValue """ - let sourceText = SourceText.From(addNewLine fileContents) + let sourceText = SourceText.From(fileContents) let caretPosition = fileContents.IndexOf("DateTime") + 1 let documentId = DocumentId.CreateNewId(ProjectId.CreateNewId()) From cc2b567c7da35ed49a97aa43983178c9f30c71a1 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 13:06:21 -0800 Subject: [PATCH 29/33] Removing last addNewLine. It's done --- .../tests/UnitTests/EditorFormattingServiceTests.fs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs b/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs index abf61785ba5..c1d043377d5 100644 --- a/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs +++ b/vsintegration/tests/UnitTests/EditorFormattingServiceTests.fs @@ -64,11 +64,6 @@ marker2 marker3 marker4""" - - // Adding this new-line character at the end of the source seems odd but is required for some unit tests - // Todo: fix tests - let addNewLine (source: string) = - if source.Length = 0 || not (source.[source.Length - 1] = '\n') then source + "\n" else source [] [] @@ -79,7 +74,7 @@ marker4""" let position = indentTemplate.IndexOf(marker) Assert.IsTrue(position >= 0, "Precondition failed: unable to find marker in template") - let sourceText = SourceText.From(addNewLine indentTemplate) + let sourceText = SourceText.From(indentTemplate) let lineNumber = sourceText.Lines |> Seq.findIndex (fun line -> line.Span.Contains position) let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions projectOptions @@ -122,7 +117,7 @@ let foo = somethingElseHere """ - let sourceText = SourceText.From(start.Replace("$", clipboard) |> addNewLine) + let sourceText = SourceText.From(start.Replace("$", clipboard)) let span = TextSpan(start.IndexOf '$', clipboard.Length) let formattingOptions = { FormatOnPaste = enabled } @@ -171,7 +166,7 @@ let foo = somethingElseHere """ - let sourceText = SourceText.From(start.Replace("$", clipboard) |> addNewLine) + let sourceText = SourceText.From(start.Replace("$", clipboard)) let span = TextSpan(start.IndexOf '$', clipboard.Length) let formattingOptions = { FormatOnPaste = true } @@ -216,7 +211,7 @@ let foo = somethingElseHere """ - let sourceText = SourceText.From(start.Replace("$", clipboard) |> addNewLine) + let sourceText = SourceText.From(start.Replace("$", clipboard)) // If we're pasting on an empty line which has been automatically indented, // then the pasted span includes this automatic indentation. Check that we From 7f212e5b079249046bdef288be0b71535d56423a Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 18:09:08 -0800 Subject: [PATCH 30/33] Better tests and small optimizations --- src/fsharp/service/ServiceXmlDocParser.fs | 6 +-- src/utils/prim-lexing.fs | 24 +++++++--- src/utils/prim-lexing.fsi | 6 ++- .../SourceTextTests.fs | 36 +++++++++----- .../src/FSharp.Editor/Common/Extensions.fs | 47 ++++++++++++------- .../tests/UnitTests/RoslynSourceTextTests.fs | 36 +++++++++----- 6 files changed, 104 insertions(+), 51 deletions(-) diff --git a/src/fsharp/service/ServiceXmlDocParser.fs b/src/fsharp/service/ServiceXmlDocParser.fs index cc9223f4ca6..9191637baa9 100644 --- a/src/fsharp/service/ServiceXmlDocParser.fs +++ b/src/fsharp/service/ServiceXmlDocParser.fs @@ -40,10 +40,10 @@ module XmlDocParsing = | SynPat.InstanceMember _ | SynPat.FromParseError _ -> [] - let getXmlDocablesImpl(sourceCodeLinesOfTheFile: string [], input: ParsedInput option) = + let getXmlDocablesImpl(sourceText: ISourceText, input: ParsedInput option) = let indentOf (lineNum: int) = let mutable i = 0 - let line = sourceCodeLinesOfTheFile.[lineNum-1] // -1 because lineNum reported by xmldocs are 1-based, but array is 0-based + let line = sourceText.GetLineString(lineNum-1) // -1 because lineNum reported by xmldocs are 1-based, but array is 0-based while i < line.Length && line.Chars(i) = ' ' do i <- i + 1 i @@ -189,4 +189,4 @@ module XmlDocParser = /// Get the list of Xml documentation from current source code let getXmlDocables (sourceText: ISourceText, input) = - XmlDocParsing.getXmlDocablesImpl (sourceText.GetLines(), input) \ No newline at end of file + XmlDocParsing.getXmlDocablesImpl (sourceText, input) \ No newline at end of file diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index c3bb53a23b8..4df354f6779 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -11,7 +11,9 @@ type ISourceText = abstract Item : int -> char with get - abstract GetLines : unit -> string [] + abstract GetLineString : lineIndex: int -> string + + abstract GetLineCount : unit -> int abstract GetLastCharacterPosition : unit -> int * int @@ -21,7 +23,7 @@ type ISourceText = abstract Length : int - abstract ContentEquals : ISourceText -> bool + abstract ContentEquals : sourceText: ISourceText -> bool abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit @@ -57,20 +59,30 @@ type StringText(str: string) = else (0, 0) - member __.GetLines() = getLines.Value + member __.GetLineString(lineIndex) = + // This requires allocating and getting all the lines. + // However, it is only called in ServiceXmlDocParser which is rarely called + // and most likely whoever is calling it is using a different implementation of ISourceText + // So, it's ok that we do this for now. + getLines.Value.[lineIndex] + + member __.GetLineCount() = getLines.Value.Length member __.GetSubTextString(start, length) = str.Substring(start, length) member __.SubTextEquals(target, startIndex) = - if startIndex < 0 then + if startIndex < 0 || startIndex >= str.Length then raise (ArgumentOutOfRangeException("startIndex")) if String.IsNullOrEmpty(target) then raise (ArgumentException("Target is null or empty.", "target")) - if startIndex + target.Length > str.Length then false - else str.IndexOf(target, startIndex, target.Length) <> -1 + let lastIndex = startIndex + target.Length + if lastIndex <= startIndex || lastIndex >= str.Length then + raise (ArgumentException("Target is too big.", "target")) + + str.IndexOf(target, startIndex, target.Length) <> -1 member __.Length = str.Length diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index 7503f2e05f4..7ae521cf8a6 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -9,7 +9,9 @@ type ISourceText = abstract Item : int -> char with get - abstract GetLines : unit -> string [] + abstract GetLineString : lineIndex: int -> string + + abstract GetLineCount : unit -> int abstract GetLastCharacterPosition : unit -> int * int @@ -19,7 +21,7 @@ type ISourceText = abstract Length : int - abstract ContentEquals : ISourceText -> bool + abstract ContentEquals : sourceText: ISourceText -> bool abstract CopyTo : sourceIndex: int * destination: char [] * destinationIndex: int * count: int -> unit diff --git a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs index deed63d7a45..3ccdeca03a7 100644 --- a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs +++ b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs @@ -14,18 +14,30 @@ module SourceTextTests = let StringText () = let text = "test\ntest2\r\ntest3\n\ntest4\ntest5\rtest6\n" let sourceText = SourceText.ofString text - let lines = sourceText.GetLines() - - Assert.AreEqual("test", lines.[0]) - Assert.AreEqual("test2", lines.[1]) - Assert.AreEqual("test3", lines.[2]) - Assert.AreEqual("", lines.[3]) - Assert.AreEqual("test4", lines.[4]) - Assert.AreEqual("test5", lines.[5]) - Assert.AreEqual("test6", lines.[6]) - Assert.AreEqual("", lines.[7]) - Assert.AreEqual(8, lines.Length) + + Assert.AreEqual("test", sourceText.GetLineString(0)) + Assert.AreEqual("test2", sourceText.GetLineString(1)) + Assert.AreEqual("test3", sourceText.GetLineString(2)) + Assert.AreEqual("", sourceText.GetLineString(3)) + Assert.AreEqual("test4", sourceText.GetLineString(4)) + Assert.AreEqual("test5", sourceText.GetLineString(5)) + Assert.AreEqual("test6", sourceText.GetLineString(6)) + Assert.AreEqual("", sourceText.GetLineString(7)) + Assert.AreEqual(8, sourceText.GetLineCount()) let (count, length) = sourceText.GetLastCharacterPosition() Assert.AreEqual(8, count) - Assert.AreEqual(0, length) \ No newline at end of file + Assert.AreEqual(0, length) + + Assert.True(sourceText.SubTextEquals("test", 0)) + Assert.True(sourceText.SubTextEquals("test2", 5)) + Assert.True(sourceText.SubTextEquals("test3", 12)) + + Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("", 0) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals(text + text, 0) |> ignore) |> ignore + + Assert.False(sourceText.SubTextEquals("test", 1)) + Assert.False(sourceText.SubTextEquals("test", 4)) + Assert.False(sourceText.SubTextEquals("test", 11)) \ No newline at end of file diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index d3e3294f1c2..ba5407491f6 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -49,34 +49,49 @@ module private SourceText = let weakTable = ConditionalWeakTable() let create (sourceText: SourceText) = - let getLines = - lazy - [| - for i = 0 to sourceText.Lines.Count - 1 do - yield sourceText.Lines.[i].ToString() - |] - - let getLastCharacterPosition = - lazy - if sourceText.Lines.Count > 0 then - (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) - else - (0, 0) let sourceText = { new ISourceText with member __.Item with get index = sourceText.[index] - member __.GetLines() = getLines.Value + member __.GetLineString(lineIndex) = + sourceText.Lines.[lineIndex].ToString() - member __.GetLastCharacterPosition() = getLastCharacterPosition.Value + member __.GetLineCount() = + sourceText.Lines.Count + + member __.GetLastCharacterPosition() = + if sourceText.Lines.Count > 0 then + (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) + else + (0, 0) member __.GetSubTextString(start, length) = sourceText.GetSubText(TextSpan(start, length)).ToString() member __.SubTextEquals(target, startIndex) = - sourceText.GetSubText(TextSpan(startIndex, target.Length)).ContentEquals(SourceText.From(target)) + if startIndex < 0 || startIndex >= sourceText.Length then + raise (ArgumentOutOfRangeException("startIndex")) + + if String.IsNullOrEmpty(target) then + raise (ArgumentException("Target is null or empty.", "target")) + + let lastIndex = startIndex + target.Length + if lastIndex <= startIndex || lastIndex >= sourceText.Length then + raise (ArgumentException("Target is too big.", "target")) + + let mutable finished = false + let mutable didEqual = true + let mutable i = 0 + while not finished && i < target.Length do + if target.[i] <> sourceText.[startIndex + i] then + didEqual <- false + finished <- true // bail out early + else + i <- i + 1 + + didEqual member __.ContentEquals(sourceText) = match sourceText with diff --git a/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs index efd14fd576e..a2354ceba3a 100644 --- a/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs +++ b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs @@ -16,18 +16,30 @@ module RoslynSourceTextTests = let SourceText () = let text = "test\ntest2\r\ntest3\n\ntest4\ntest5\rtest6\n" let sourceText = SourceText.From(text).ToFSharpSourceText() - let lines = sourceText.GetLines() - - Assert.AreEqual("test", lines.[0]) - Assert.AreEqual("test2", lines.[1]) - Assert.AreEqual("test3", lines.[2]) - Assert.AreEqual("", lines.[3]) - Assert.AreEqual("test4", lines.[4]) - Assert.AreEqual("test5", lines.[5]) - Assert.AreEqual("test6", lines.[6]) - Assert.AreEqual("", lines.[7]) - Assert.AreEqual(8, lines.Length) + + Assert.AreEqual("test", sourceText.GetLineString(0)) + Assert.AreEqual("test2", sourceText.GetLineString(1)) + Assert.AreEqual("test3", sourceText.GetLineString(2)) + Assert.AreEqual("", sourceText.GetLineString(3)) + Assert.AreEqual("test4", sourceText.GetLineString(4)) + Assert.AreEqual("test5", sourceText.GetLineString(5)) + Assert.AreEqual("test6", sourceText.GetLineString(6)) + Assert.AreEqual("", sourceText.GetLineString(7)) + Assert.AreEqual(8, sourceText.GetLineCount()) let (count, length) = sourceText.GetLastCharacterPosition() Assert.AreEqual(8, count) - Assert.AreEqual(0, length) \ No newline at end of file + Assert.AreEqual(0, length) + + Assert.True(sourceText.SubTextEquals("test", 0)) + Assert.True(sourceText.SubTextEquals("test2", 5)) + Assert.True(sourceText.SubTextEquals("test3", 12)) + + Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("", 0) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals(text + text, 0) |> ignore) |> ignore + + Assert.False(sourceText.SubTextEquals("test", 1)) + Assert.False(sourceText.SubTextEquals("test", 4)) + Assert.False(sourceText.SubTextEquals("test", 11)) \ No newline at end of file From 45579eb65ba5d2a639f2b62c7f0f58084074cbdf Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 14 Dec 2018 18:20:25 -0800 Subject: [PATCH 31/33] Adjusting comment --- src/utils/prim-lexing.fs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 4df354f6779..47e3b5bb10c 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -44,6 +44,9 @@ type StringText(str: string) = |] let getLines = + // This requires allocating and getting all the lines. + // However, likely whoever is calling it is using a different implementation of ISourceText + // So, it's ok that we do this for now. lazy getLines str member __.String = str @@ -60,10 +63,6 @@ type StringText(str: string) = (0, 0) member __.GetLineString(lineIndex) = - // This requires allocating and getting all the lines. - // However, it is only called in ServiceXmlDocParser which is rarely called - // and most likely whoever is calling it is using a different implementation of ISourceText - // So, it's ok that we do this for now. getLines.Value.[lineIndex] member __.GetLineCount() = getLines.Value.Length From 5081e1c19753afeee484db410a824af870c203bf Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 16 Dec 2018 14:44:17 -0800 Subject: [PATCH 32/33] Updating CompilerServiceBenchmarks --- .../CompilerServiceBenchmarks.fsproj | 5 ++ .../CompilerServiceBenchmarks/Program.fs | 82 +++++++++++++++++-- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj b/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj index 8b88e55342b..f642d6431c4 100644 --- a/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj +++ b/benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj @@ -12,12 +12,17 @@ $(SystemValueTuplePackageVersion) + + AnyCPU + + + diff --git a/benchmarks/CompilerServiceBenchmarks/Program.fs b/benchmarks/CompilerServiceBenchmarks/Program.fs index dbf5eaecfc2..e69d6d6ab50 100644 --- a/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -4,9 +4,80 @@ open BenchmarkDotNet.Attributes open BenchmarkDotNet.Running open Microsoft.FSharp.Compiler.ErrorLogger open Microsoft.FSharp.Compiler.SourceCodeServices +open Microsoft.FSharp.Compiler.Text open System.Text +open Microsoft.CodeAnalysis.Text -[] +module private SourceText = + + open System.Runtime.CompilerServices + + let weakTable = ConditionalWeakTable() + + let create (sourceText: SourceText) = + + let sourceText = + { new ISourceText with + + member __.Item with get index = sourceText.[index] + + member __.GetLineString(lineIndex) = + sourceText.Lines.[lineIndex].ToString() + + member __.GetLineCount() = + sourceText.Lines.Count + + member __.GetLastCharacterPosition() = + if sourceText.Lines.Count > 0 then + (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length) + else + (0, 0) + + member __.GetSubTextString(start, length) = + sourceText.GetSubText(TextSpan(start, length)).ToString() + + member __.SubTextEquals(target, startIndex) = + if startIndex < 0 || startIndex >= sourceText.Length then + raise (ArgumentOutOfRangeException("startIndex")) + + if String.IsNullOrEmpty(target) then + raise (ArgumentException("Target is null or empty.", "target")) + + let lastIndex = startIndex + target.Length + if lastIndex <= startIndex || lastIndex >= sourceText.Length then + raise (ArgumentException("Target is too big.", "target")) + + let mutable finished = false + let mutable didEqual = true + let mutable i = 0 + while not finished && i < target.Length do + if target.[i] <> sourceText.[startIndex + i] then + didEqual <- false + finished <- true // bail out early + else + i <- i + 1 + + didEqual + + member __.ContentEquals(sourceText) = + match sourceText with + | :? SourceText as sourceText -> sourceText.ContentEquals(sourceText) + | _ -> false + + member __.Length = sourceText.Length + + member __.CopyTo(sourceIndex, destination, destinationIndex, count) = + sourceText.CopyTo(sourceIndex, destination, destinationIndex, count) + } + + sourceText + +type SourceText with + + member this.ToFSharpSourceText() = + SourceText.weakTable.GetValue(this, Runtime.CompilerServices.ConditionalWeakTable<_,_>.CreateValueCallback(SourceText.create)) + +[] type CompilerServiceParsing() = let mutable checkerOpt = None @@ -32,8 +103,7 @@ type CompilerServiceParsing() = match sourceOpt with | None -> - let source = File.ReadAllText("""..\..\..\..\..\src\fsharp\TypeChecker.fs""") - sourceOpt <- Some(source) + sourceOpt <- Some <| SourceText.From(File.OpenRead("""..\..\..\..\..\src\fsharp\TypeChecker.fs"""), Encoding.Default, SourceHashAlgorithm.Sha1, true) | _ -> () [] @@ -43,7 +113,7 @@ type CompilerServiceParsing() = | Some(checker) -> checker.InvalidateAll() checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() - checker.ParseFile("dummy.fs", "dummy", parsingOptions) |> Async.RunSynchronously |> ignore + checker.ParseFile("dummy.fs", SourceText.ofString "dummy", parsingOptions) |> Async.RunSynchronously |> ignore [] member __.Parsing() = @@ -51,10 +121,10 @@ type CompilerServiceParsing() = | None, _ -> failwith "no checker" | _, None -> failwith "no source" | Some(checker), Some(source) -> - let results = checker.ParseFile("TypeChecker.fs", source, parsingOptions) |> Async.RunSynchronously + let results = checker.ParseFile("TypeChecker.fs", source.ToFSharpSourceText(), parsingOptions) |> Async.RunSynchronously if results.ParseHadErrors then failwithf "parse had errors: %A" results.Errors [] let main argv = let _ = BenchmarkRunner.Run() - 0 + 0 \ No newline at end of file From 52314bd735cc857875c4aba56c5b6102793237b9 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 17 Dec 2018 11:37:30 -0800 Subject: [PATCH 33/33] Updated nits --- src/fsharp/UnicodeLexing.fs | 2 +- src/fsharp/service/service.fs | 34 +++++++++---------- src/utils/prim-lexing.fs | 14 ++++---- .../SourceTextTests.fs | 4 +-- .../src/FSharp.Editor/Common/Extensions.fs | 7 ++-- .../tests/UnitTests/RoslynSourceTextTests.fs | 4 +-- .../SemanticColorizationServiceTests.fs | 4 +-- 7 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/fsharp/UnicodeLexing.fs b/src/fsharp/UnicodeLexing.fs index aad63513d00..9dfdaf32df7 100644 --- a/src/fsharp/UnicodeLexing.fs +++ b/src/fsharp/UnicodeLexing.fs @@ -21,7 +21,7 @@ let StringAsLexbuf (s:string) : Lexbuf = let FunctionAsLexbuf (bufferFiller: char[] * int * int -> int) : Lexbuf = LexBuffer<_>.FromFunction bufferFiller -let SourceTextAsLexbuf (sourceText) = +let SourceTextAsLexbuf sourceText = LexBuffer.FromSourceText(sourceText) // The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index b272e470695..7a0e9f2f69d 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -1537,7 +1537,7 @@ module internal Parser = let tokenizer = LexFilter.LexFilter(lightSyntaxStatus, options.CompilingFsLib, Lexer.token lexargs true, lexbuf) tokenizer.Lexer - let createLexbuf (sourceText: ISourceText) = + let createLexbuf sourceText = UnicodeLexing.SourceTextAsLexbuf(sourceText) let matchBraces(sourceText, fileName, options: FSharpParsingOptions, userOpName: string) = @@ -2614,7 +2614,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC } /// Type-check the result obtained by parsing, but only if the antecedent type checking context is available. - member bc.CheckFileInProjectAllowingStaleCachedResults(parseResults: FSharpParseFileResults, filename, fileVersion, source, options, textSnapshotInfo: obj option, userOpName) = + member bc.CheckFileInProjectAllowingStaleCachedResults(parseResults: FSharpParseFileResults, filename, fileVersion, sourceText, options, textSnapshotInfo: obj option, userOpName) = let execWithReactorAsync action = reactor.EnqueueAndAwaitOpAsync(userOpName, "CheckFileInProjectAllowingStaleCachedResults ", filename, action) async { try @@ -2629,7 +2629,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC match incrementalBuildersCache.TryGetAny (ctok, options) with | Some (Some builder, creationErrors, _) -> - match bc.GetCachedCheckFileResult(builder, filename, source, options) with + match bc.GetCachedCheckFileResult(builder, filename, sourceText, options) with | Some (_, checkResults) -> return Some (builder, creationErrors, Some (FSharpCheckFileAnswer.Succeeded checkResults)) | _ -> return Some (builder, creationErrors, None) | _ -> return None // the builder wasn't ready @@ -2649,7 +2649,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC match tcPrior with | Some tcPrior -> - let! checkResults = bc.CheckOneFileImpl(parseResults, source, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) + let! checkResults = bc.CheckOneFileImpl(parseResults, sourceText, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) return Some checkResults | None -> return None // the incremental builder was not up to date finally @@ -2657,7 +2657,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC } /// Type-check the result obtained by parsing. Force the evaluation of the antecedent type checking context if needed. - member bc.CheckFileInProject(parseResults: FSharpParseFileResults, filename, fileVersion, source, options, textSnapshotInfo, userOpName) = + member bc.CheckFileInProject(parseResults: FSharpParseFileResults, filename, fileVersion, sourceText, options, textSnapshotInfo, userOpName) = let execWithReactorAsync action = reactor.EnqueueAndAwaitOpAsync(userOpName, "CheckFileInProject", filename, action) async { try @@ -2669,7 +2669,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC | None -> return FSharpCheckFileAnswer.Succeeded (MakeCheckFileResultsEmpty(filename, creationErrors)) | Some builder -> // Check the cache. We can only use cached results when there is no work to do to bring the background builder up-to-date - let cachedResults = bc.GetCachedCheckFileResult(builder, filename, source, options) + let cachedResults = bc.GetCachedCheckFileResult(builder, filename, sourceText, options) match cachedResults with | Some (_, checkResults) -> return FSharpCheckFileAnswer.Succeeded checkResults @@ -2678,14 +2678,14 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC let! tcPrior = execWithReactorAsync <| fun ctok -> builder.GetCheckResultsBeforeFileInProject (ctok, filename) let parseTreeOpt = parseResults.ParseTree |> Option.map builder.DeduplicateParsedInputModuleNameInProject let parseResultsAterDeDuplication = FSharpParseFileResults(parseResults.Errors, parseTreeOpt, parseResults.ParseHadErrors, parseResults.DependencyFiles) - let! checkAnswer = bc.CheckOneFileImpl(parseResultsAterDeDuplication, source, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) + let! checkAnswer = bc.CheckOneFileImpl(parseResultsAterDeDuplication, sourceText, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) return checkAnswer finally bc.ImplicitlyStartCheckProjectInBackground(options, userOpName) } /// Parses and checks the source file and returns untyped AST and check results. - member bc.ParseAndCheckFileInProject (filename:string, fileVersion, source, options:FSharpProjectOptions, textSnapshotInfo, userOpName) = + member bc.ParseAndCheckFileInProject (filename:string, fileVersion, sourceText, options:FSharpProjectOptions, textSnapshotInfo, userOpName) = let execWithReactorAsync action = reactor.EnqueueAndAwaitOpAsync(userOpName, "ParseAndCheckFileInProject", filename, action) async { try @@ -2706,7 +2706,7 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC return (parseResults, FSharpCheckFileAnswer.Aborted) | Some builder -> - let cachedResults = bc.GetCachedCheckFileResult(builder, filename, source, options) + let cachedResults = bc.GetCachedCheckFileResult(builder, filename, sourceText, options) match cachedResults with | Some (parseResults, checkResults) -> @@ -2720,10 +2720,10 @@ type BackgroundCompiler(legacyReferenceResolver, projectCacheSize, keepAssemblyC // Do the parsing. let parsingOptions = FSharpParsingOptions.FromTcConfig(builder.TcConfig, Array.ofList (builder.SourceFiles), options.UseScriptResolutionRules) - let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (source, filename, parsingOptions, userOpName) + let parseErrors, parseTreeOpt, anyErrors = Parser.parseFile (sourceText, filename, parsingOptions, userOpName) let parseTreeOpt = parseTreeOpt |> Option.map builder.DeduplicateParsedInputModuleNameInProject let parseResults = FSharpParseFileResults(parseErrors, parseTreeOpt, anyErrors, builder.AllDependenciesDeprecated) - let! checkResults = bc.CheckOneFileImpl(parseResults, source, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) + let! checkResults = bc.CheckOneFileImpl(parseResults, sourceText, filename, options, textSnapshotInfo, fileVersion, builder, tcPrior, creationErrors, userOpName) Logger.LogBlockMessageStop (filename + strGuid + "-Successful") LogCompilerFunctionId.Service_ParseAndCheckFileInProject @@ -3008,14 +3008,14 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten member ic.ReferenceResolver = legacyReferenceResolver - member ic.MatchBraces(filename, source, options: FSharpParsingOptions, ?userOpName: string) = + member ic.MatchBraces(filename, sourceText, options: FSharpParsingOptions, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" async { - match braceMatchCache.TryGet(AssumeAnyCallerThreadWithoutEvidence(), (filename, source, options)) with + match braceMatchCache.TryGet(AssumeAnyCallerThreadWithoutEvidence(), (filename, sourceText, options)) with | Some res -> return res | None -> - let res = Parser.matchBraces(source, filename, options, userOpName) - braceMatchCache.Set(AssumeAnyCallerThreadWithoutEvidence(), (filename, source, options), res) + let res = Parser.matchBraces(sourceText, filename, options, userOpName) + braceMatchCache.Set(AssumeAnyCallerThreadWithoutEvidence(), (filename, sourceText, options), res) return res } @@ -3029,10 +3029,10 @@ type FSharpChecker(legacyReferenceResolver, projectCacheSize, keepAssemblyConten let argv = List.ofArray options.OtherOptions ic.GetParsingOptionsFromCommandLineArgs(sourceFiles, argv, options.UseScriptResolutionRules) - member ic.ParseFile(filename, source, options, ?userOpName: string) = + member ic.ParseFile(filename, sourceText, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" ic.CheckMaxMemoryReached() - backgroundCompiler.ParseFile(filename, source, options, userOpName) + backgroundCompiler.ParseFile(filename, sourceText, options, userOpName) member ic.ParseFileInProject(filename, source: string, options, ?userOpName: string) = let userOpName = defaultArg userOpName "Unknown" diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 13c0c0c216c..87012685b3f 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -33,10 +33,10 @@ type StringText(str: string) = let getLines (str: string) = use reader = new StringReader(str) [| - let line = ref (reader.ReadLine()) - while not (isNull !line) do - yield !line - line := reader.ReadLine() + let mutable line = reader.ReadLine() + while not (isNull line) do + yield line + line <- reader.ReadLine() if str.EndsWith("\n", StringComparison.Ordinal) then // last trailing space not returned // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak @@ -72,14 +72,14 @@ type StringText(str: string) = member __.SubTextEquals(target, startIndex) = if startIndex < 0 || startIndex >= str.Length then - raise (ArgumentOutOfRangeException("startIndex")) + invalidArg "startIndex" "Out of range." if String.IsNullOrEmpty(target) then - raise (ArgumentException("Target is null or empty.", "target")) + invalidArg "target" "Is null or empty." let lastIndex = startIndex + target.Length if lastIndex <= startIndex || lastIndex >= str.Length then - raise (ArgumentException("Target is too big.", "target")) + invalidArg "target" "Too big." str.IndexOf(target, startIndex, target.Length) <> -1 diff --git a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs index 3ccdeca03a7..3754e75ae2d 100644 --- a/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs +++ b/tests/FSharp.Compiler.UnitTests/SourceTextTests.fs @@ -33,8 +33,8 @@ module SourceTextTests = Assert.True(sourceText.SubTextEquals("test2", 5)) Assert.True(sourceText.SubTextEquals("test3", 12)) - Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore - Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore Assert.Throws(fun () -> sourceText.SubTextEquals("", 0) |> ignore) |> ignore Assert.Throws(fun () -> sourceText.SubTextEquals(text + text, 0) |> ignore) |> ignore diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index ba5407491f6..04245876425 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -49,7 +49,6 @@ module private SourceText = let weakTable = ConditionalWeakTable() let create (sourceText: SourceText) = - let sourceText = { new ISourceText with @@ -72,14 +71,14 @@ module private SourceText = member __.SubTextEquals(target, startIndex) = if startIndex < 0 || startIndex >= sourceText.Length then - raise (ArgumentOutOfRangeException("startIndex")) + invalidArg "startIndex" "Out of range." if String.IsNullOrEmpty(target) then - raise (ArgumentException("Target is null or empty.", "target")) + invalidArg "target" "Is null or empty." let lastIndex = startIndex + target.Length if lastIndex <= startIndex || lastIndex >= sourceText.Length then - raise (ArgumentException("Target is too big.", "target")) + invalidArg "target" "Too big." let mutable finished = false let mutable didEqual = true diff --git a/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs index a2354ceba3a..23b1e6702a0 100644 --- a/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs +++ b/vsintegration/tests/UnitTests/RoslynSourceTextTests.fs @@ -35,8 +35,8 @@ module RoslynSourceTextTests = Assert.True(sourceText.SubTextEquals("test2", 5)) Assert.True(sourceText.SubTextEquals("test3", 12)) - Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore - Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", -1) |> ignore) |> ignore + Assert.Throws(fun () -> sourceText.SubTextEquals("test", text.Length) |> ignore) |> ignore Assert.Throws(fun () -> sourceText.SubTextEquals("", 0) |> ignore) |> ignore Assert.Throws(fun () -> sourceText.SubTextEquals(text + text, 0) |> ignore) |> ignore diff --git a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs index 7fdacfe7489..8ba79f5aad6 100644 --- a/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs +++ b/vsintegration/tests/UnitTests/SemanticColorizationServiceTests.fs @@ -30,10 +30,10 @@ type SemanticClassificationServiceTests() = let checker = FSharpChecker.Create() let perfOptions = { LanguageServicePerformanceOptions.Default with AllowStaleCompletionResults = false } - let getRanges (sourceText: string) : (Range.range * SemanticClassificationType) list = + let getRanges (source: string) : (Range.range * SemanticClassificationType) list = asyncMaybe { - let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(sourceText), projectOptions, perfOptions, "") + let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, 0, SourceText.From(source), projectOptions, perfOptions, "") return checkFileResults.GetSemanticClassification(None) } |> Async.RunSynchronously