Add ISourceText to language service and use Roslyn's SourceText for FSharp.Editor (dotnet#6001)

* Initial ISourceText implementation (does not work yet)

* Lexbuffer works

* Removing Source. Now using only ISourceText. Added SourceText.ofString.

* Fixing tests

* We need to use addNewLine for tests to pass

* Added test for SourceText.ofString

* Trying to fix tests

* Simplified ISourceText API. Added RoslynSourceTextTests

* Trying to get the build working again

* Re-organize prim-lexing.fsi

* Handling format strings

* Trying to get tests to pass

* Trying to fix tests

* Ignoring test

* unignoring test

* Fixed weak table

* Removing addNewLine in sourcetext

* Fixing interactive checker tests

* Fixing more tests

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removing last addNewLine. It's done

* Better tests and small optimizations

* Adjusting comment

* Updating CompilerServiceBenchmarks

* Updated nits

Author: TIHan

Commands/HelpContextService.fs

@@ -25,7 +25,7 @@ type internal FSharpHelpContextService
     static let userOpName = "ImplementInterfaceCodeFix"
     static member GetHelpTerm(checker: FSharpChecker, sourceText : SourceText, fileName, options, span: TextSpan, tokens: List<ClassifiedSpan>, textVersion, perfOptions) : Async<string option> = 
         asyncMaybe {
-            let! _, _, check = checker.ParseAndCheckDocument(fileName, textVersion, sourceText.ToString(), options, perfOptions, userOpName = userOpName)
+            let! _, _, check = checker.ParseAndCheckDocument(fileName, textVersion, sourceText, options, perfOptions, userOpName = userOpName)
             let textLines = sourceText.Lines
             let lineInfo = textLines.GetLineFromPosition(span.Start)
             let line = lineInfo.LineNumber
@@ -34,12 +34,11 @@ type internal FSharpHelpContextService
             let caretColumn = textLines.GetLinePosition(span.Start).Character
 
             let shouldTryToFindSurroundingIdent (token : ClassifiedSpan) =
-                let span = token.TextSpan
-                let content = sourceText.ToString().Substring(span.Start, span.End - span.Start)
+                let content = sourceText.GetSubText(token.TextSpan)
                 match token.ClassificationType with
                 | ClassificationTypeNames.Text
                 | ClassificationTypeNames.WhiteSpace -> true
-                | (ClassificationTypeNames.Operator|ClassificationTypeNames.Punctuation)when content = "." -> true
+                | (ClassificationTypeNames.Operator|ClassificationTypeNames.Punctuation)when content.Length > 0 && content.[0] = '.' -> true
                 | _ -> false
 
             let tokenInformation, col =

Commands/XmlDocCommandService.fs

@@ -68,9 +68,9 @@ type internal XmlDocCommandFilter
                                 let curLineNum = wpfTextView.Caret.Position.BufferPosition.GetContainingLine().LineNumber + 1
                                 let! document = document.Value
                                 let! parsingOptions, _options = projectInfoManager.TryGetOptionsForEditingDocumentOrProject(document, CancellationToken.None)
-                                let sourceText = wpfTextView.TextBuffer.CurrentSnapshot.GetText()
+                                let! sourceText = document.GetTextAsync(CancellationToken.None)
                                 let! parsedInput = checker.ParseDocument(document, parsingOptions, sourceText, userOpName)
-                                let xmlDocables = XmlDocParser.getXmlDocables (sourceText, Some parsedInput) 
+                                let xmlDocables = XmlDocParser.getXmlDocables (sourceText.ToFSharpSourceText(), Some parsedInput) 
                                 let xmlDocablesBelowThisLine = 
                                     // +1 because looking below current line for e.g. a 'member'
                                     xmlDocables |> List.filter (fun (XmlDocable(line,_indent,_paramNames)) -> line = curLineNum+1) 

Common/Extensions.fs

@@ -6,7 +6,9 @@ module internal Microsoft.VisualStudio.FSharp.Editor.Extensions
 open System
 open System.IO
 open Microsoft.CodeAnalysis
+open Microsoft.CodeAnalysis.Text
 open Microsoft.CodeAnalysis.Host
+open Microsoft.FSharp.Compiler.Text
 open Microsoft.FSharp.Compiler.Ast
 open Microsoft.FSharp.Compiler.SourceCodeServices
 
@@ -40,6 +42,74 @@ type Document with
                 languageServices.GetService<'T>()
                 |> Some
 
+module private SourceText =
+
+    open System.Runtime.CompilerServices
+
+    let weakTable = ConditionalWeakTable<SourceText, ISourceText>()
+
+    let create (sourceText: SourceText) =
+        let sourceText =
+            { new ISourceText with
+            
+                member __.Item with get index = sourceText.[index]
+
+                member __.GetLineString(lineIndex) =
+                    sourceText.Lines.[lineIndex].ToString()
+
+                member __.GetLineCount() =
+                    sourceText.Lines.Count
+
+                member __.GetLastCharacterPosition() =
+                    if sourceText.Lines.Count > 0 then
+                        (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length)
+                    else
+                        (0, 0)
+
+                member __.GetSubTextString(start, length) =
+                    sourceText.GetSubText(TextSpan(start, length)).ToString()
+
+                member __.SubTextEquals(target, startIndex) =
+                    if startIndex < 0 || startIndex >= sourceText.Length then
+                        invalidArg "startIndex" "Out of range."
+
+                    if String.IsNullOrEmpty(target) then
+                        invalidArg "target" "Is null or empty."
+
+                    let lastIndex = startIndex + target.Length
+                    if lastIndex <= startIndex || lastIndex >= sourceText.Length then
+                        invalidArg "target" "Too big."
+
+                    let mutable finished = false
+                    let mutable didEqual = true
+                    let mutable i = 0
+                    while not finished && i < target.Length do
+                        if target.[i] <> sourceText.[startIndex + i] then
+                            didEqual <- false
+                            finished <- true // bail out early                        
+                        else
+                            i <- i + 1
+
+                    didEqual
+
+                member __.ContentEquals(sourceText) =
+                    match sourceText with
+                    | :? SourceText as sourceText -> sourceText.ContentEquals(sourceText)
+                    | _ -> false
+
+                member __.Length = sourceText.Length
+
+                member __.CopyTo(sourceIndex, destination, destinationIndex, count) =
+                    sourceText.CopyTo(sourceIndex, destination, destinationIndex, count)
+            }
+
+        sourceText
+
+type SourceText with
+
+    member this.ToFSharpSourceText() =
+        SourceText.weakTable.GetValue(this, Runtime.CompilerServices.ConditionalWeakTable<_,_>.CreateValueCallback(SourceText.create))
+
 type FSharpNavigationDeclarationItem with
     member x.RoslynGlyph : Glyph =
         match x.Glyph with

Completion/CompletionProvider.fs

@@ -105,7 +105,7 @@ type internal FSharpCompletionProvider
     static member ProvideCompletionsAsyncAux(checker: FSharpChecker, sourceText: SourceText, caretPosition: int, options: FSharpProjectOptions, filePath: string, 
                                              textVersionHash: int, getAllSymbols: FSharpCheckFileResults -> AssemblySymbol list, languageServicePerformanceOptions: LanguageServicePerformanceOptions, intellisenseOptions: IntelliSenseOptions) = 
         asyncMaybe {
-            let! parseResults, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions, userOpName = userOpName)
+            let! parseResults, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions, userOpName = userOpName)
 
             let textLines = sourceText.Lines
             let caretLinePos = textLines.GetLinePosition(caretPosition)

Completion/SignatureHelp.fs

@@ -36,7 +36,7 @@ type internal FSharpSignatureHelpProvider
 
     // Unit-testable core routine
     static member internal ProvideMethodsAsyncAux(checker: FSharpChecker, documentationBuilder: IDocumentationBuilder, sourceText: SourceText, caretPosition: int, options: FSharpProjectOptions, triggerIsTypedChar: char option, filePath: string, textVersionHash: int) = async {
-        let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToString(), options, userOpName = userOpName)
+        let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToFSharpSourceText(), options, userOpName = userOpName)
         match checkFileAnswer with
         | FSharpCheckFileAnswer.Aborted -> return None
         | FSharpCheckFileAnswer.Succeeded(checkFileResults) -> 

Debugging/BreakpointResolutionService.fs

@@ -37,7 +37,7 @@ type internal FSharpBreakpointResolutionService
             else
                 let textLineColumn = textLinePos.Character
                 let fcsTextLineNumber = Line.fromZ textLinePos.Line // Roslyn line numbers are zero-based, FSharp.Compiler.Service line numbers are 1-based
-                let! parseResults = checker.ParseFile(fileName, sourceText.ToString(), parsingOptions, userOpName = userOpName)
+                let! parseResults = checker.ParseFile(fileName, sourceText.ToFSharpSourceText(), parsingOptions, userOpName = userOpName)
                 return parseResults.ValidateBreakpointLocation(mkPos fcsTextLineNumber textLineColumn)
         }
 

Diagnostics/DocumentDiagnosticAnalyzer.fs

@@ -59,12 +59,13 @@ type internal FSharpDocumentDiagnosticAnalyzer() =
 
     static member GetDiagnostics(checker: FSharpChecker, filePath: string, sourceText: SourceText, textVersionHash: int, parsingOptions: FSharpParsingOptions, options: FSharpProjectOptions, diagnosticType: DiagnosticsType) = 
         async {
-            let! parseResults = checker.ParseFile(filePath, sourceText.ToString(), parsingOptions, userOpName=userOpName) 
+            let fsSourceText = sourceText.ToFSharpSourceText()
+            let! parseResults = checker.ParseFile(filePath, fsSourceText, parsingOptions, userOpName=userOpName) 
             let! errors = 
                 async {
                     match diagnosticType with
                     | DiagnosticsType.Semantic ->
-                        let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, sourceText.ToString(), options, userOpName=userOpName) 
+                        let! checkResultsAnswer = checker.CheckFileInProject(parseResults, filePath, textVersionHash, fsSourceText, options, userOpName=userOpName) 
                         match checkResultsAnswer with
                         | FSharpCheckFileAnswer.Aborted -> return [||]
                         | FSharpCheckFileAnswer.Succeeded results ->

DocumentHighlights/DocumentHighlightsService.fs

@@ -59,7 +59,7 @@ type internal FSharpDocumentHighlightsService [<ImportingConstructor>] (checkerP
             let textLinePos = sourceText.Lines.GetLinePosition(position)
             let fcsTextLineNumber = Line.fromZ textLinePos.Line
             let! symbol = Tokenizer.getSymbolAtPosition(documentKey, sourceText, position, filePath, defines, SymbolLookupKind.Greedy, false)
-            let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText.ToString(), options, languageServicePerformanceOptions,  userOpName = userOpName)
+            let! _, _, checkFileResults = checker.ParseAndCheckDocument(filePath, textVersionHash, sourceText, options, languageServicePerformanceOptions,  userOpName = userOpName)
             let! symbolUse = checkFileResults.GetSymbolUseAtLocation(fcsTextLineNumber, symbol.Ident.idRange.EndColumn, textLine.ToString(), symbol.FullIsland, userOpName=userOpName)
             let! symbolUses = checkFileResults.GetUsesOfSymbolInFile(symbolUse.Symbol) |> liftAsync
             return 

Formatting/BraceMatchingService.fs

@@ -3,6 +3,7 @@
 namespace Microsoft.VisualStudio.FSharp.Editor
 
 open System.ComponentModel.Composition
+open Microsoft.CodeAnalysis.Text
 open Microsoft.CodeAnalysis.Editor
 open Microsoft.FSharp.Compiler.SourceCodeServices
 open System.Runtime.InteropServices
@@ -17,9 +18,9 @@ type internal FSharpBraceMatchingService
 
     static let defaultUserOpName = "BraceMatching"
 
-    static member GetBraceMatchingResult(checker: FSharpChecker, sourceText, fileName, parsingOptions: FSharpParsingOptions, position: int, userOpName: string, [<Optional; DefaultParameterValue(false)>] forFormatting: bool) = 
+    static member GetBraceMatchingResult(checker: FSharpChecker, sourceText: SourceText, fileName, parsingOptions: FSharpParsingOptions, position: int, userOpName: string, [<Optional; DefaultParameterValue(false)>] forFormatting: bool) = 
         async {
-            let! matchedBraces = checker.MatchBraces(fileName, sourceText.ToString(), parsingOptions, userOpName)
+            let! matchedBraces = checker.MatchBraces(fileName, sourceText.ToFSharpSourceText(), parsingOptions, userOpName)
             let isPositionInRange range = 
                 match RoslynHelpers.TryFSharpRangeToTextSpan(sourceText, range) with
                 | None -> false

LanguageService/FSharpCheckerExtensions.fs

@@ -10,20 +10,17 @@ open Microsoft.FSharp.Compiler.Ast
 open Microsoft.FSharp.Compiler.SourceCodeServices
 
 type FSharpChecker with
-    member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: string, userOpName: string) =
+    member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: SourceText, userOpName: string) =
         asyncMaybe {
-            let! fileParseResults = checker.ParseFile(document.FilePath, sourceText, parsingOptions, userOpName=userOpName) |> liftAsync
+            let! fileParseResults = checker.ParseFile(document.FilePath, sourceText.ToFSharpSourceText(), parsingOptions, userOpName=userOpName) |> liftAsync
             return! fileParseResults.ParseTree
         }
 
-    member checker.ParseDocument(document: Document, parsingOptions: FSharpParsingOptions, sourceText: SourceText, userOpName: string) =
-        checker.ParseDocument(document, parsingOptions, sourceText=sourceText.ToString(), userOpName=userOpName)
-
-    member checker.ParseAndCheckDocument(filePath: string, textVersionHash: int, sourceText: string, options: FSharpProjectOptions, languageServicePerformanceOptions: LanguageServicePerformanceOptions, userOpName: string) =
+    member checker.ParseAndCheckDocument(filePath: string, textVersionHash: int, sourceText: SourceText, options: FSharpProjectOptions, languageServicePerformanceOptions: LanguageServicePerformanceOptions, userOpName: string) =
         async {
             let parseAndCheckFile =
                 async {
-                    let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText, options, userOpName=userOpName)
+                    let! parseResults, checkFileAnswer = checker.ParseAndCheckFileInProject(filePath, textVersionHash, sourceText.ToFSharpSourceText(), options, userOpName=userOpName)
                     return
                         match checkFileAnswer with
                         | FSharpCheckFileAnswer.Aborted -> 
@@ -82,21 +79,21 @@ type FSharpChecker with
                 match allowStaleResults with 
                 | Some b -> { document.FSharpOptions.LanguageServicePerformance with AllowStaleCompletionResults = b } 
                 | _ ->  document.FSharpOptions.LanguageServicePerformance
-            return! checker.ParseAndCheckDocument(document.FilePath, textVersion.GetHashCode(), sourceText.ToString(), options, perfOpts, userOpName=userOpName)
+            return! checker.ParseAndCheckDocument(document.FilePath, textVersion.GetHashCode(), sourceText, options, perfOpts, userOpName=userOpName)
         }
 
 
-    member checker.TryParseAndCheckFileInProject (projectOptions, fileName, source, userOpName) = async {
-        let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0, source,projectOptions, userOpName=userOpName)
+    member checker.TryParseAndCheckFileInProject (projectOptions, fileName, sourceText: SourceText, userOpName) = async {
+        let! (parseResults, checkAnswer) = checker.ParseAndCheckFileInProject (fileName,0,sourceText.ToFSharpSourceText(),projectOptions, userOpName=userOpName)
         match checkAnswer with
         | FSharpCheckFileAnswer.Aborted ->  return  None
         | FSharpCheckFileAnswer.Succeeded checkResults -> return Some (parseResults,checkResults)
     }
 
 
-    member checker.GetAllUsesOfAllSymbolsInSourceString (projectOptions, fileName, source: string, checkForUnusedOpens, userOpName) = async {
+    member checker.GetAllUsesOfAllSymbolsInSourceString (projectOptions, fileName, sourceText: SourceText, checkForUnusedOpens, userOpName) = async {
 
-        let! parseAndCheckResults = checker.TryParseAndCheckFileInProject (projectOptions, fileName, source, userOpName=userOpName)
+        let! parseAndCheckResults = checker.TryParseAndCheckFileInProject (projectOptions, fileName, sourceText, userOpName=userOpName)
         match parseAndCheckResults with
         | None -> return [||]
         | Some(_parseResults,checkResults) ->