diff --git a/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj b/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj index 8f849da41b3..71738ed4b77 100644 --- a/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj +++ b/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj @@ -671,6 +671,7 @@ + diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index 56f3a3c1aca..85629fcebc1 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -9,6 +9,7 @@ open System.IO.MemoryMappedFiles open System.Runtime.InteropServices open System.Runtime.CompilerServices open FSharp.NativeInterop +open FSharp.Compiler.AbstractIL.Internal.Library #nowarn "9" @@ -35,6 +36,244 @@ module internal Bytes = let stringAsUnicodeNullTerminated (s:string) = Array.append (System.Text.Encoding.Unicode.GetBytes s) (ofInt32Array [| 0x0;0x0 |]) +type ChunkedArrayForEachDelegate<'T> = delegate of Span<'T> -> unit + +/// Not thread safe. +/// Loosely based on StringBuilder/BlobBuilder +[] +type ChunkedArrayBuilder<'T> private (minChunkSize: int, buffer: 'T []) = + + member val private Buffer = buffer with get, set + member val private ChunkLength = 0 with get, set + member val private NextOrPrevious = Unchecked.defaultof> with get, set + member val private Position = 0 with get, set + member val private IsFrozen = false with get, set + member val private ChunkCount = 1 with get, set + + // [1:first]->[2]->[3:last]<-[4:head] + // ^_______________| + member private x.FirstChunk = x.NextOrPrevious.NextOrPrevious + + member private x.CheckIsFrozen() = + if x.IsFrozen then + invalidOp "Chunked array builder is frozen and cannot add or remove items." + + member x.IsEmpty = x.ChunkCount = 1 && x.ChunkLength = 0 + + member x.Reserve length = + x.CheckIsFrozen() + + if length + x.Position > x.Buffer.Length then + x.ChunkCount <- x.ChunkCount + 1 + let newChunk = + ChunkedArrayBuilder<'T>( + minChunkSize, + Array.zeroCreate<'T> (Math.Max(length, minChunkSize)), + IsFrozen = true) + let newBuffer = newChunk.Buffer + + match box x.NextOrPrevious with + | null -> () + | _ -> + match box x.FirstChunk with + | null -> + let firstChunk = x.NextOrPrevious + firstChunk.NextOrPrevious <- newChunk + newChunk.NextOrPrevious <- firstChunk + | _ -> + let firstChunk = x.FirstChunk + let lastChunk = x.NextOrPrevious + lastChunk.NextOrPrevious <- newChunk + newChunk.NextOrPrevious <- firstChunk + + newChunk.ChunkLength <- x.ChunkLength + newChunk.Buffer <- x.Buffer + x.Buffer <- newBuffer + x.NextOrPrevious <- newChunk + x.ChunkLength <- 0 + x.Position <- 0 + let reserved = Span(x.Buffer, x.Position, length) + x.ChunkLength <- x.ChunkLength + length + x.Position <- x.ChunkLength + reserved + + member x.AddSpan(data: ReadOnlySpan<'T>) = + let reserved = x.Reserve data.Length + data.CopyTo(reserved) + + member x.Add(data: 'T) = + let reserved = x.Reserve 1 + reserved.[0] <- data + + member x.ForEachBuilder f = + match box x.NextOrPrevious with + | null -> () + | _ -> + match box x.FirstChunk with + | null -> + f x.NextOrPrevious + | _ -> + let firstChunk = x.FirstChunk + f firstChunk + let mutable chunk = firstChunk.NextOrPrevious + while chunk <> firstChunk do + f chunk + chunk <- chunk.NextOrPrevious + + if x.ChunkLength > 0 then + f x + + member x.ForEachChunk (f: ChunkedArrayForEachDelegate<'T>) = + x.ForEachBuilder (fun builder -> + // Note: This could happen due to removing items. + // It might be better to remove the node itself instead of checking for the length in the for loop. + if builder.ChunkLength > 0 then + f.Invoke(Span(builder.Buffer, 0, builder.ChunkLength))) + + member x.RemoveAll predicate = + x.CheckIsFrozen() + x.ForEachBuilder(fun builder -> + let removeQueue = Collections.Generic.Queue() + + for i = 0 to builder.ChunkLength - 1 do + let item = builder.Buffer.[i] + if predicate item then + removeQueue.Enqueue i + + while removeQueue.Count > 0 do + let indexToRemove = removeQueue.Dequeue() + + if indexToRemove = builder.ChunkLength - 1 then + builder.Buffer.[indexToRemove] <- Unchecked.defaultof<_> + else + for i = indexToRemove to builder.ChunkLength - 2 do + builder.Buffer.[i] <- builder.Buffer.[i + 1] + builder.Buffer.[builder.ChunkLength - 1] <- Unchecked.defaultof<_> + + builder.ChunkLength <- builder.ChunkLength - 1 + + assert (builder.ChunkLength >= 0) + + if builder.ChunkLength = 0 then + x.ChunkCount <- x.ChunkCount - 1) + + static member Create(minChunkSize, startingCapacity) = + ChunkedArrayBuilder( + minChunkSize, + Array.zeroCreate<'T> (Math.Max(startingCapacity, minChunkSize))) + +[] +type ChunkedArray<'T>(builder: ChunkedArrayBuilder<'T>) = + + member _.Length = + match box builder with + | null -> 0 + | _ -> + let mutable total = 0 + builder.ForEachChunk (fun chunk -> total <- total + chunk.Length) + total + + member _.ForEachChunk f = + match box builder with + | null -> () + | _ -> + builder.ForEachChunk f + + member x.ToArray() = + match box builder with + | null -> [||] + | _ -> + let arr = Array.zeroCreate<'T> x.Length + let mutable total = 0 + builder.ForEachChunk (fun chunk -> + chunk.CopyTo(Span(arr, total, chunk.Length)) + total <- total + chunk.Length) + arr + + member x.IsEmpty = + match box builder with + | null -> true + | _ -> builder.IsEmpty + + static member Empty = + ChunkedArray<'T> Unchecked.defaultof<_> + +type ChunkedArrayBuilder<'T> with + + member x.ToChunkedArray() = + x.IsFrozen <- true + ChunkedArray x + +[] +module ChunkedArray = + + [] + let MinChunkSize = 16 + + [] + let DefaultChunkSize = 256 + + let iter f (chunkedArr: ChunkedArray<_>) = + chunkedArr.ForEachChunk(fun chunk -> + for item in chunk do + f item) + + let map f (chunkedArr: ChunkedArray<_>) = + if chunkedArr.IsEmpty then ChunkedArray<_>.Empty + else + let res = ChunkedArrayBuilder<_>.Create(MinChunkSize, DefaultChunkSize) + chunkedArr.ForEachChunk(fun chunk -> + for item in chunk do + res.Add (f item)) + res.ToChunkedArray() + + let filter predicate (chunkedArr: ChunkedArray<_>) = + if chunkedArr.IsEmpty then ChunkedArray<_>.Empty + else + let res = ChunkedArrayBuilder<_>.Create(MinChunkSize, DefaultChunkSize) + chunkedArr.ForEachChunk(fun chunk -> + for item in chunk do + if predicate item then + res.Add item) + res.ToChunkedArray() + + let choose chooser (chunkedArr: ChunkedArray<_>) = + if chunkedArr.IsEmpty then ChunkedArray<_>.Empty + else + let res = ChunkedArrayBuilder<_>.Create(MinChunkSize, DefaultChunkSize) + chunkedArr.ForEachChunk(fun chunk -> + for item in chunk do + match chooser item with + | Some mappedItem -> res.Add mappedItem + | _ -> ()) + res.ToChunkedArray() + + let distinctBy projection (chunkedArr: ChunkedArray<_>) = + if chunkedArr.IsEmpty then ChunkedArray<_>.Empty + else + let hashSet = Collections.Generic.HashSet<_>(HashIdentity.Structural<_>) + let res = ChunkedArrayBuilder<_>.Create(MinChunkSize, DefaultChunkSize) + chunkedArr.ForEachChunk(fun chunk -> + for item in chunk do + if hashSet.Add(projection item) then + res.Add item) + res.ToChunkedArray() + + let concat (chunkedArrs: ChunkedArray<_> seq) = + let res = ChunkedArrayBuilder<_>.Create(MinChunkSize, DefaultChunkSize) + chunkedArrs + |> Seq.iter (fun chunkedArr -> chunkedArr |> iter res.Add) + res.ToChunkedArray() + + let toArray (chunkedArr: ChunkedArray<_>) = + chunkedArr.ToArray() + + let toReversedList (chunkedArr: ChunkedArray<_>) = + let mutable res = [] + chunkedArr + |> iter (fun item -> res <- item :: res) + res + [] type ByteMemory () = @@ -54,6 +293,8 @@ type ByteMemory () = abstract CopyTo: Stream -> unit + abstract CopyTo: Span -> unit + abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit abstract ToArray: unit -> byte[] @@ -101,9 +342,12 @@ type ByteArrayMemory(bytes: byte[], offset, length) = override _.Slice(pos, count) = ByteArrayMemory(bytes, offset + pos, count) :> ByteMemory - override _.CopyTo stream = + override _.CopyTo(stream: Stream) = stream.Write(bytes, offset, length) + override _.CopyTo(span: Span) = + ReadOnlySpan(bytes, offset, length).CopyTo span + override _.Copy(srcOffset, dest, destOffset, count) = Array.blit bytes (offset + srcOffset) dest destOffset count @@ -166,10 +410,13 @@ type RawByteMemory(addr: nativeptr, length: int, hold: obj) = check (pos + count - 1) RawByteMemory(NativePtr.add addr pos, count, hold) :> ByteMemory - override x.CopyTo stream = + override x.CopyTo(stream: Stream) = use stream2 = x.AsStream() stream2.CopyTo stream + override x.CopyTo(span: Span) = + ReadOnlySpan(NativePtr.toVoidPtr addr, length).CopyTo span + override x.Copy(srcOffset, dest, destOffset, count) = check srcOffset Marshal.Copy(NativePtr.toNativeInt addr + nativeint srcOffset, dest, destOffset, count) @@ -208,7 +455,10 @@ type ReadOnlyByteMemory(bytes: ByteMemory) = member _.Slice(pos, count) = bytes.Slice(pos, count) |> ReadOnlyByteMemory [] - member _.CopyTo stream = bytes.CopyTo stream + member _.CopyTo(stream: Stream) = bytes.CopyTo stream + + [] + member _.CopyTo(span: Span) = bytes.CopyTo span [] member _.Copy(srcOffset, dest, destOffset, count) = bytes.Copy(srcOffset, dest, destOffset, count) @@ -340,70 +590,61 @@ type internal ByteStream = member b.Skip = b.pos <- b.pos + n #endif +[] +module ByteBufferConstants = + + // From System.Reflection.Metadata.BlobBuilder as the DefaultChunkSize. + [] + let MaxStartingCapacity = 256 + + // From System.Reflection.Metadata.BlobBuilder + // Reasoning was the smallest atomic data was a Guid. + // Therefore, it should be a reasonable minimum value. + [] + let MinChunkSize = 16 type internal ByteBuffer = - { mutable bbArray: byte[] + { mutable bbArray: ChunkedArrayBuilder mutable bbCurrent: int } - member buf.Ensure newSize = - let oldBufSize = buf.bbArray.Length - if newSize > oldBufSize then - let old = buf.bbArray - buf.bbArray <- Bytes.zeroCreate (max newSize (oldBufSize * 2)) - Bytes.blit old 0 buf.bbArray 0 buf.bbCurrent + member buf.Reserve length = + buf.bbCurrent <- buf.bbCurrent + length + buf.bbArray.Reserve length - member buf.Close () = Bytes.sub buf.bbArray 0 buf.bbCurrent + member buf.Close () = buf.bbArray.ToChunkedArray().ToArray() member buf.EmitIntAsByte (i:int) = - let newSize = buf.bbCurrent + 1 - buf.Ensure newSize - buf.bbArray.[buf.bbCurrent] <- byte i - buf.bbCurrent <- newSize + buf.EmitByte (byte i) - member buf.EmitByte (b:byte) = buf.EmitIntAsByte (int b) + member buf.EmitByte (b:byte) = + (buf.bbArray.Reserve 1).WriteByte (0, b) + buf.bbCurrent <- buf.bbCurrent + 1 member buf.EmitIntsAsBytes (arr:int[]) = let n = arr.Length - let newSize = buf.bbCurrent + n - buf.Ensure newSize - let bbArr = buf.bbArray - let bbBase = buf.bbCurrent for i = 0 to n - 1 do - bbArr.[bbBase + i] <- byte arr.[i] - buf.bbCurrent <- newSize + buf.EmitByte (byte arr.[i]) - member bb.FixupInt32 pos n = - bb.bbArray.[pos] <- (Bytes.b0 n |> byte) - bb.bbArray.[pos + 1] <- (Bytes.b1 n |> byte) - bb.bbArray.[pos + 2] <- (Bytes.b2 n |> byte) - bb.bbArray.[pos + 3] <- (Bytes.b3 n |> byte) + member buf.EmitInt32s (arr: int32[]) = + let n = arr.Length + for i = 0 to n - 1 do + buf.EmitInt32 arr.[i] member buf.EmitInt32 n = - let newSize = buf.bbCurrent + 4 - buf.Ensure newSize - buf.FixupInt32 buf.bbCurrent n - buf.bbCurrent <- newSize + (buf.bbArray.Reserve 4).WriteUInt32(0, uint32 n) + buf.bbCurrent <- buf.bbCurrent + 4 member buf.EmitBytes (i:byte[]) = - let n = i.Length - let newSize = buf.bbCurrent + n - buf.Ensure newSize - Bytes.blit i 0 buf.bbArray buf.bbCurrent n - buf.bbCurrent <- newSize - - member buf.EmitByteMemory (i:ReadOnlyByteMemory) = - let n = i.Length - let newSize = buf.bbCurrent + n - buf.Ensure newSize - i.Copy(0, buf.bbArray, buf.bbCurrent, n) - buf.bbCurrent <- newSize + buf.bbArray.AddSpan(ReadOnlySpan i) + buf.bbCurrent <- buf.bbCurrent + i.Length + + member buf.EmitByteMemory (bytes: ReadOnlyByteMemory) = + bytes.CopyTo(buf.bbArray.Reserve bytes.Length) + buf.bbCurrent <- buf.bbCurrent + bytes.Length member buf.EmitInt32AsUInt16 n = - let newSize = buf.bbCurrent + 2 - buf.Ensure newSize - buf.bbArray.[buf.bbCurrent] <- (Bytes.b0 n |> byte) - buf.bbArray.[buf.bbCurrent + 1] <- (Bytes.b1 n |> byte) - buf.bbCurrent <- newSize + (buf.bbArray.Reserve 2).WriteInt32AsUInt16 (0, n) + buf.bbCurrent <- buf.bbCurrent + 2 member buf.EmitBoolAsByte (b:bool) = buf.EmitIntAsByte (if b then 1 else 0) @@ -415,8 +656,13 @@ type internal ByteBuffer = member buf.Position = buf.bbCurrent - static member Create sz = - { bbArray=Bytes.zeroCreate sz + static member Create startingCapacity = + let startingCapacity = + if startingCapacity > ByteBufferConstants.MaxStartingCapacity then + ByteBufferConstants.MaxStartingCapacity + else + startingCapacity + { bbArray = ChunkedArrayBuilder.Create(ByteBufferConstants.MinChunkSize, startingCapacity) bbCurrent = 0 } diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index 11e7c9f58cc..1e9ad8f9d6a 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -3,12 +3,8 @@ /// Blobs of bytes, cross-compiling namespace FSharp.Compiler.AbstractIL.Internal +open System open System.IO -open Internal.Utilities - -open FSharp.Compiler.AbstractIL -open FSharp.Compiler.AbstractIL.Internal - module internal Bytes = /// returned int will be 0 <= x <= 255 @@ -23,6 +19,59 @@ module internal Bytes = val stringAsUnicodeNullTerminated: string -> byte[] val stringAsUtf8NullTerminated: string -> byte[] +type internal ChunkedArrayForEachDelegate<'T> = delegate of Span<'T> -> unit + +[] +type internal ChunkedArray<'T> = + + member Length: int + + member ForEachChunk: ChunkedArrayForEachDelegate<'T> -> unit + + member ToArray: unit -> 'T[] + + member IsEmpty: bool + + static member Empty: ChunkedArray<'T> + +/// Not thread safe. +[] +type internal ChunkedArrayBuilder<'T> = + + member Reserve: length: int -> Span<'T> + + member AddSpan: data: ReadOnlySpan<'T> -> unit + + member Add: data: 'T -> unit + + member RemoveAll: predicate: ('T -> bool) -> unit + + member ForEachChunk: ChunkedArrayForEachDelegate<'T> -> unit + + /// When the builder is turned into a chunked array, it is considered frozen. + member ToChunkedArray: unit -> ChunkedArray<'T> + + static member Create: minChunkSize: int * startingCapacity: int -> ChunkedArrayBuilder<'T> + +[] +module internal ChunkedArray = + + val iter: ('T -> unit) -> ChunkedArray<'T> -> unit + + val map: ('T -> 'U) -> ChunkedArray<'T> -> ChunkedArray<'U> + + val filter: ('T -> bool) -> ChunkedArray<'T> -> ChunkedArray<'T> + + val choose: ('T -> 'U option) -> ChunkedArray<'T> -> ChunkedArray<'U> + + val distinctBy: ('T -> 'Key) -> ChunkedArray<'T> -> ChunkedArray<'T> when 'Key : equality + + val concat: ChunkedArray<'T> seq -> ChunkedArray<'T> + + val toArray: ChunkedArray<'T> -> 'T[] + + val toReversedList: ChunkedArray<'T> -> 'T list + /// May be backed by managed or unmanaged memory, or memory mapped file. [] type internal ByteMemory = @@ -43,6 +92,8 @@ type internal ByteMemory = abstract CopyTo: Stream -> unit + abstract CopyTo: Span -> unit + abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit abstract ToArray: unit -> byte[] @@ -77,13 +128,18 @@ type internal ReadOnlyByteMemory = member CopyTo: Stream -> unit + member CopyTo: Span -> unit + member Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit member ToArray: unit -> byte[] + /// Get a stream representation of the backing memory. + /// Disposing this will not free up any of the backing memory. + /// Stream cannot be written to. member AsStream: unit -> Stream -type ByteMemory with +type internal ByteMemory with member AsReadOnly: unit -> ReadOnlyByteMemory @@ -106,20 +162,21 @@ type ByteMemory with /// Imperative buffers and streams of byte[] [] type internal ByteBuffer = + member Reserve : int -> Span member Close : unit -> byte[] member EmitIntAsByte : int -> unit member EmitIntsAsBytes : int[] -> unit + member EmitInt32s : int32[] -> unit member EmitByte : byte -> unit member EmitBytes : byte[] -> unit member EmitByteMemory : ReadOnlyByteMemory -> unit member EmitInt32 : int32 -> unit member EmitInt64 : int64 -> unit - member FixupInt32 : pos: int -> value: int32 -> unit member EmitInt32AsUInt16 : int32 -> unit member EmitBoolAsByte : bool -> unit member EmitUInt16 : uint16 -> unit member Position : int - static member Create : int -> ByteBuffer + static member Create : startingCapacity: int -> ByteBuffer [] diff --git a/src/absil/illib.fs b/src/absil/illib.fs index 0a7d471e141..35fd01f5f2a 100644 --- a/src/absil/illib.fs +++ b/src/absil/illib.fs @@ -436,47 +436,33 @@ module List = let mapiFoldSquared f z xss = mapFoldSquared f z (xss |> mapiSquared (fun i j x -> (i, j, x))) -module ResizeArray = - - /// Split a ResizeArray into an array of smaller chunks. - /// This requires `items/chunkSize` Array copies of length `chunkSize` if `items/chunkSize % 0 = 0`, - /// otherwise `items/chunkSize + 1` Array copies. - let chunkBySize chunkSize f (items: ResizeArray<'t>) = - // we could use Seq.chunkBySize here, but that would involve many enumerator.MoveNext() calls that we can sidestep with a bit of math - let itemCount = items.Count - if itemCount = 0 - then [||] - else - let chunksCount = - match itemCount / chunkSize with - | n when itemCount % chunkSize = 0 -> n - | n -> n + 1 // any remainder means we need an additional chunk to store it - - [| for index in 0..chunksCount-1 do - let startIndex = index * chunkSize - let takeCount = min (itemCount - startIndex) chunkSize - - let holder = Array.zeroCreate takeCount - // we take a bounds-check hit here on each access. - // other alternatives here include - // * iterating across an IEnumerator (incurs MoveNext penalty) - // * doing a block copy using `List.CopyTo(index, array, index, count)` (requires more copies to do the mapping) - // none are significantly better. - for i in 0 .. takeCount - 1 do - holder.[i] <- f items.[i] - yield holder |] - - /// Split a large ResizeArray into a series of array chunks that are each under the Large Object Heap limit. - /// This is done to help prevent a stop-the-world collection of the single large array, instead allowing for a greater - /// probability of smaller collections. Stop-the-world is still possible, just less likely. - let mapToSmallArrayChunks f (inp: ResizeArray<'t>) = - let itemSizeBytes = sizeof<'t> - // rounding down here is good because it ensures we don't go over - let maxArrayItemCount = LOH_SIZE_THRESHOLD_BYTES / itemSizeBytes - - /// chunk the provided input into arrays that are smaller than the LOH limit - /// in order to prevent long-term storage of those values - chunkBySize maxArrayItemCount f inp +[] +type SpanExtensions = + + [] + static member inline WriteByte(data: Span, offset, value: byte) = + data.[0 + offset] <- byte value + + [] + static member inline WriteUInt16(data: Span, offset, value: uint16) = + data.[0 + offset] <- byte value + data.[1 + offset] <- byte (value >>> 8) + + [] + static member inline WriteUInt32(data: Span, offset, value: uint32) = + data.[0 + offset] <- byte value + data.[1 + offset] <- byte (value >>> 8) + data.[2 + offset] <- byte (value >>> 16) + data.[3 + offset] <- byte (value >>> 24) + + [] + static member inline WriteInt32(data: Span, offset, value: int32) = + data.WriteUInt32(offset, uint32 value) + + [] + static member inline WriteInt32AsUInt16(data: Span, offset, value: int32) = + data.[0 + offset] <- byte value + data.[1 + offset] <- byte (value >>> 8) module ValueOptionInternal = diff --git a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj index d8f5341a8a2..ced2cd785df 100644 --- a/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj +++ b/src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj @@ -724,6 +724,7 @@ + diff --git a/src/fsharp/NameResolution.fs b/src/fsharp/NameResolution.fs index 389c6e16b04..153b51f4ebb 100644 --- a/src/fsharp/NameResolution.fs +++ b/src/fsharp/NameResolution.fs @@ -1655,10 +1655,10 @@ type CapturedNameResolution(p: pos, i: Item, tpinst, io: ItemOccurence, de: Disp type TcResolutions (capturedEnvs: ResizeArray, capturedExprTypes: ResizeArray, - capturedNameResolutions: ResizeArray, - capturedMethodGroupResolutions: ResizeArray) = + capturedNameResolutions: ChunkedArray, + capturedMethodGroupResolutions: ChunkedArray) = - static let empty = TcResolutions(ResizeArray 0, ResizeArray 0, ResizeArray 0, ResizeArray 0) + static let empty = TcResolutions(ResizeArray 0, ResizeArray 0, ChunkedArray<_>.Empty, ChunkedArray<_>.Empty) member this.CapturedEnvs = capturedEnvs member this.CapturedExpressionTypings = capturedExprTypes @@ -1679,24 +1679,16 @@ type TcSymbolUseData = /// This is a memory-critical data structure - allocations of this data structure and its immediate contents /// is one of the highest memory long-lived data structures in typical uses of IDEs. Not many of these objects /// are allocated (one per file), but they are large because the allUsesOfAllSymbols array is large. -type TcSymbolUses(g, capturedNameResolutions: ResizeArray, formatSpecifierLocations: (range * int)[]) = - - // Make sure we only capture the information we really need to report symbol uses - let allUsesOfSymbols = - capturedNameResolutions - |> ResizeArray.mapToSmallArrayChunks (fun cnr -> { Item=cnr.Item; ItemOccurence=cnr.ItemOccurence; DisplayEnv=cnr.DisplayEnv; Range=cnr.Range }) - - let capturedNameResolutions = () - do ignore capturedNameResolutions // don't capture this! +type TcSymbolUses(g, allUsesOfSymbols: ChunkedArray, formatSpecifierLocations: (range * int)[]) = member this.GetUsesOfSymbol item = - // This member returns what is potentially a very large array, which may approach the size constraints of the Large Object Heap. - // This is unlikely in practice, though, because we filter down the set of all symbol uses to those specifically for the given `item`. - // Consequently we have a much lesser chance of ending up with an array large enough to be promoted to the LOH. - [| for symbolUseChunk in allUsesOfSymbols do - for symbolUse in symbolUseChunk do + let usesBuilder = ChunkedArrayBuilder<_>.Create(64, 128) + allUsesOfSymbols.ForEachChunk(fun chunk -> + for symbolUse in chunk do if protectAssemblyExploration false (fun () -> ItemsAreEffectivelyEqual g item symbolUse.Item) then - yield symbolUse |] + usesBuilder.Add(symbolUse) + ) + usesBuilder.ToChunkedArray() member this.AllUsesOfSymbols = allUsesOfSymbols @@ -1706,7 +1698,7 @@ type TcSymbolUses(g, capturedNameResolutions: ResizeArray() let capturedExprTypings = ResizeArray<_>() - let capturedNameResolutions = ResizeArray<_>() + let capturedNameResolutionsBuilder = ChunkedArrayBuilder<_>.Create(64, 128) let capturedFormatSpecifierLocations = ResizeArray<_>() let capturedNameResolutionIdentifiers = @@ -1721,7 +1713,7 @@ type TcResultsSinkImpl(g, ?sourceText: ISourceText) = member __.GetHashCode ((m, _)) = hash m member __.Equals ((m1, item1), (m2, item2)) = Range.equals m1 m2 && ItemsAreEffectivelyEqual g item1 item2 } ) - let capturedMethodGroupResolutions = ResizeArray<_>() + let capturedMethodGroupResolutionsBuilder = ChunkedArrayBuilder<_>.Create(64, 128) let capturedOpenDeclarations = ResizeArray() let allowedRange (m: range) = not m.IsSynthetic @@ -1741,11 +1733,26 @@ type TcResultsSinkImpl(g, ?sourceText: ISourceText) = { SourceText = sourceText LineStartPositions = positions }) + let capturedNameResolutions = + lazy capturedNameResolutionsBuilder.ToChunkedArray() + + let capturedMethodGroupResolutions = + lazy capturedMethodGroupResolutionsBuilder.ToChunkedArray() + member this.GetResolutions() = - TcResolutions(capturedEnvs, capturedExprTypings, capturedNameResolutions, capturedMethodGroupResolutions) + TcResolutions(capturedEnvs, capturedExprTypings, capturedNameResolutions.Value, capturedMethodGroupResolutions.Value) member this.GetSymbolUses() = - TcSymbolUses(g, capturedNameResolutions, capturedFormatSpecifierLocations.ToArray()) + // Make sure we only capture the information we really need to report symbol uses + let allUsesOfSymbols = + let mappedBuilder = ChunkedArrayBuilder<_>.Create(64, 128) + capturedNameResolutions.Value.ForEachChunk(fun chunk -> + for cnr in chunk do + mappedBuilder.Add { Item=cnr.Item; ItemOccurence=cnr.ItemOccurence; DisplayEnv=cnr.DisplayEnv; Range=cnr.Range } + ) + mappedBuilder.ToChunkedArray() + + TcSymbolUses(g, allUsesOfSymbols, capturedFormatSpecifierLocations.ToArray()) member this.GetOpenDeclarations() = capturedOpenDeclarations |> Seq.distinctBy (fun x -> x.Range, x.AppliedScope, x.IsOwnNamespace) |> Seq.toArray @@ -1765,8 +1772,8 @@ type TcResultsSinkImpl(g, ?sourceText: ISourceText) = // for the same identifier at the same location. if allowedRange m then if replace then - capturedNameResolutions.RemoveAll(fun cnr -> Range.equals cnr.Range m) |> ignore - capturedMethodGroupResolutions.RemoveAll(fun cnr -> Range.equals cnr.Range m) |> ignore + capturedNameResolutionsBuilder.RemoveAll(fun cnr -> Range.equals cnr.Range m) |> ignore + capturedMethodGroupResolutionsBuilder.RemoveAll(fun cnr -> Range.equals cnr.Range m) |> ignore else let alreadyDone = match item with @@ -1784,8 +1791,8 @@ type TcResultsSinkImpl(g, ?sourceText: ISourceText) = | _ -> false if not alreadyDone then - capturedNameResolutions.Add(CapturedNameResolution(endPos, item, tpinst, occurenceType, denv, nenv, ad, m)) - capturedMethodGroupResolutions.Add(CapturedNameResolution(endPos, itemMethodGroup, [], occurenceType, denv, nenv, ad, m)) + capturedNameResolutionsBuilder.Add(CapturedNameResolution(endPos, item, tpinst, occurenceType, denv, nenv, ad, m)) + capturedMethodGroupResolutionsBuilder.Add(CapturedNameResolution(endPos, itemMethodGroup, [], occurenceType, denv, nenv, ad, m)) member sink.NotifyFormatSpecifierLocation(m, numArgs) = capturedFormatSpecifierLocations.Add((m, numArgs)) diff --git a/src/fsharp/NameResolution.fsi b/src/fsharp/NameResolution.fsi index ea3bbd8aeaf..71504d2be0b 100755 --- a/src/fsharp/NameResolution.fsi +++ b/src/fsharp/NameResolution.fsi @@ -12,6 +12,7 @@ open FSharp.Compiler.InfoReader open FSharp.Compiler.Tast open FSharp.Compiler.Tastops open FSharp.Compiler.TcGlobals +open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.Internal.Library open FSharp.Compiler.PrettyNaming open FSharp.Compiler.Text @@ -297,10 +298,10 @@ type internal TcResolutions = member CapturedExpressionTypings : ResizeArray /// Exact name resolutions - member CapturedNameResolutions : ResizeArray + member CapturedNameResolutions : ChunkedArray /// Represents all the resolutions of names to groups of methods. - member CapturedMethodGroupResolutions : ResizeArray + member CapturedMethodGroupResolutions : ChunkedArray /// Represents the empty set of resolutions static member Empty : TcResolutions @@ -318,10 +319,10 @@ type TcSymbolUseData = type internal TcSymbolUses = /// Get all the uses of a particular item within the file - member GetUsesOfSymbol : Item -> TcSymbolUseData[] + member GetUsesOfSymbol : Item -> ChunkedArray /// All the uses of all items within the file - member AllUsesOfSymbols : TcSymbolUseData[][] + member AllUsesOfSymbols : ChunkedArray /// Get the locations of all the printf format specifiers in the file member GetFormatSpecifierLocationsAndArity : unit -> (range * int)[] diff --git a/src/fsharp/TastPickle.fs b/src/fsharp/TastPickle.fs index 5098b16ecd0..ea9de87a324 100644 --- a/src/fsharp/TastPickle.fs +++ b/src/fsharp/TastPickle.fs @@ -512,21 +512,8 @@ let p_option f x st = // lazily. However, a lazy reader is not used in this version because the value may contain the definitions of some // OSGN nodes. let private p_lazy_impl p v st = - let fixupPos1 = st.os.Position // We fix these up after - prim_p_int32 0 st - let fixupPos2 = st.os.Position - prim_p_int32 0 st - let fixupPos3 = st.os.Position - prim_p_int32 0 st - let fixupPos4 = st.os.Position - prim_p_int32 0 st - let fixupPos5 = st.os.Position - prim_p_int32 0 st - let fixupPos6 = st.os.Position - prim_p_int32 0 st - let fixupPos7 = st.os.Position - prim_p_int32 0 st + let fixup = st.os.Reserve (7 * sizeof) let idx1 = st.os.Position let otyconsIdx1 = st.oentities.Size let otyparsIdx1 = st.otypars.Size @@ -535,17 +522,17 @@ let private p_lazy_impl p v st = p v st // Determine and fixup the length of the pickled data let idx2 = st.os.Position - st.os.FixupInt32 fixupPos1 (idx2-idx1) // Determine and fixup the ranges of OSGN nodes defined within the lazy portion let otyconsIdx2 = st.oentities.Size let otyparsIdx2 = st.otypars.Size let ovalsIdx2 = st.ovals.Size - st.os.FixupInt32 fixupPos2 otyconsIdx1 - st.os.FixupInt32 fixupPos3 otyconsIdx2 - st.os.FixupInt32 fixupPos4 otyparsIdx1 - st.os.FixupInt32 fixupPos5 otyparsIdx2 - st.os.FixupInt32 fixupPos6 ovalsIdx1 - st.os.FixupInt32 fixupPos7 ovalsIdx2 + fixup.WriteInt32 (0 , (idx2-idx1)) + fixup.WriteInt32 (1 * sizeof, otyconsIdx1) + fixup.WriteInt32 (2 * sizeof, otyconsIdx2) + fixup.WriteInt32 (3 * sizeof, otyparsIdx1) + fixup.WriteInt32 (4 * sizeof, otyparsIdx2) + fixup.WriteInt32 (5 * sizeof, ovalsIdx1) + fixup.WriteInt32 (6 * sizeof, ovalsIdx2) let p_lazy p x st = p_lazy_impl p (Lazy.force x) st diff --git a/src/fsharp/service/FSharpCheckerResults.fs b/src/fsharp/service/FSharpCheckerResults.fs index ab8ee3d216f..d08c22a0a2c 100644 --- a/src/fsharp/service/FSharpCheckerResults.fs +++ b/src/fsharp/service/FSharpCheckerResults.fs @@ -40,6 +40,7 @@ open FSharp.Compiler.SourceCodeServices.SymbolHelpers open Internal.Utilities open Internal.Utilities.Collections +open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.ILBinaryReader [] @@ -257,7 +258,7 @@ type internal TypeCheckInfo | ResolveOverloads.Yes -> sResolutions.CapturedNameResolutions | ResolveOverloads.No -> sResolutions.CapturedMethodGroupResolutions - let quals = quals |> ResizeArray.filter (fun cnr -> posEq cnr.Pos endOfNamesPos) + let quals = quals |> ChunkedArray.filter (fun cnr -> posEq cnr.Pos endOfNamesPos) quals @@ -268,7 +269,7 @@ type internal TypeCheckInfo let endOfNamesPos = mkPos line colAtEndOfNames // Logic below expects the list to be in reverse order of resolution - let cnrs = GetCapturedNameResolutions endOfNamesPos resolveOverloads |> ResizeArray.toList |> List.rev + let cnrs = GetCapturedNameResolutions endOfNamesPos resolveOverloads |> ChunkedArray.toReversedList match cnrs, membersByResidue with @@ -325,7 +326,7 @@ type internal TypeCheckInfo let TryGetTypeFromNameResolution(line, colAtEndOfNames, membersByResidue, resolveOverloads) = let endOfNamesPos = mkPos line colAtEndOfNames - let items = GetCapturedNameResolutions endOfNamesPos resolveOverloads |> ResizeArray.toList |> List.rev + let items = GetCapturedNameResolutions endOfNamesPos resolveOverloads |> ChunkedArray.toReversedList match items, membersByResidue with | CNR(_,Item.Types(_,(ty::_)),_,_,_,_,_)::_, Some _ -> Some ty @@ -347,7 +348,7 @@ type internal TypeCheckInfo ) let GetNamedParametersAndSettableFields endOfExprPos hasTextChangedSinceLastTypecheck = - let cnrs = GetCapturedNameResolutions endOfExprPos ResolveOverloads.No |> ResizeArray.toList |> List.rev + let cnrs = GetCapturedNameResolutions endOfExprPos ResolveOverloads.No |> ChunkedArray.toReversedList let result = match cnrs with | CNR(_, Item.CtorGroup(_, ((ctor::_) as ctors)), _, denv, nenv, ad, m) ::_ -> @@ -1300,9 +1301,9 @@ type internal TypeCheckInfo match range with | Some range -> sResolutions.CapturedNameResolutions - |> Seq.filter (fun cnr -> rangeContainsPos range cnr.Range.Start || rangeContainsPos range cnr.Range.End) + |> ChunkedArray.filter (fun cnr -> rangeContainsPos range cnr.Range.Start || rangeContainsPos range cnr.Range.End) | None -> - sResolutions.CapturedNameResolutions :> seq<_> + sResolutions.CapturedNameResolutions let isDisposableTy (ty: TType) = protectAssemblyExplorationNoReraise false false (fun () -> Infos.ExistsHeadTypeInEntireHierarchy g amap range0 ty g.tcref_System_IDisposable) @@ -1323,7 +1324,7 @@ type internal TypeCheckInfo || Tastops.isRefCellTy g rfinfo.RecdField.FormalType resolutions - |> Seq.choose (fun cnr -> + |> ChunkedArray.choose (fun cnr -> match cnr with // 'seq' in 'seq { ... }' gets colored as keywords | CNR(_, (Item.Value vref), ItemOccurence.Use, _, _, _, m) when valRefEq g g.seq_vref vref -> @@ -1383,7 +1384,7 @@ type internal TypeCheckInfo | CNR(_, (Item.ActivePatternCase _ | Item.UnionCase _ | Item.ActivePatternResult _), _, _, _, _, m) -> Some (m, SemanticClassificationType.UnionCase) | _ -> None) - |> Seq.toArray + |> ChunkedArray.toArray |> Array.append (sSymbolUses.GetFormatSpecifierLocationsAndArity() |> Array.map (fun m -> fst m, SemanticClassificationType.Printf)) ) (fun msg -> @@ -1935,20 +1936,28 @@ type FSharpCheckFileResults (fun () -> [| |]) (fun scope -> let cenv = scope.SymbolEnv - [| for symbolUseChunk in scope.ScopeSymbolUses.AllUsesOfSymbols do - for symbolUse in symbolUseChunk do + let res = ResizeArray() + scope.ScopeSymbolUses.AllUsesOfSymbols + |> ChunkedArray.iter (fun symbolUse -> if symbolUse.ItemOccurence <> ItemOccurence.RelatedText then let symbol = FSharpSymbol.Create(cenv, symbolUse.Item) - yield FSharpSymbolUse(scope.TcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range) |]) + FSharpSymbolUse(scope.TcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range) + |> res.Add) + res.ToArray()) |> async.Return member __.GetUsesOfSymbolInFile(symbol:FSharpSymbol) = threadSafeOp (fun () -> [| |]) (fun scope -> - [| for symbolUse in scope.ScopeSymbolUses.GetUsesOfSymbol(symbol.Item) |> Seq.distinctBy (fun symbolUse -> symbolUse.ItemOccurence, symbolUse.Range) do - if symbolUse.ItemOccurence <> ItemOccurence.RelatedText then - yield FSharpSymbolUse(scope.TcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range) |]) + let res = ResizeArray() + scope.ScopeSymbolUses.GetUsesOfSymbol(symbol.Item) + |> ChunkedArray.distinctBy (fun symbolUse -> symbolUse.ItemOccurence, symbolUse.Range) + |> ChunkedArray.iter (fun symbolUse -> + if symbolUse.ItemOccurence <> ItemOccurence.RelatedText then + res.Add(FSharpSymbolUse(scope.TcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range)) + ) + res.ToArray()) |> async.Return member __.GetVisibleNamespacesAndModulesAtPoint(pos: pos) = @@ -2146,24 +2155,28 @@ type FSharpCheckProjectResults let (tcGlobals, _tcImports, _thisCcu, _ccuSig, tcSymbolUses, _topAttribs, _tcAssemblyData, _ilAssemRef, _ad, _tcAssemblyExpr, _dependencyFiles) = getDetails() tcSymbolUses - |> Seq.collect (fun r -> r.GetUsesOfSymbol symbol.Item) - |> Seq.distinctBy (fun symbolUse -> symbolUse.ItemOccurence, symbolUse.Range) - |> Seq.filter (fun symbolUse -> symbolUse.ItemOccurence <> ItemOccurence.RelatedText) - |> Seq.map (fun symbolUse -> FSharpSymbolUse(tcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range)) - |> Seq.toArray + |> Seq.map (fun r -> r.GetUsesOfSymbol symbol.Item) + |> ChunkedArray.concat + |> ChunkedArray.distinctBy (fun symbolUse -> symbolUse.ItemOccurence, symbolUse.Range) + |> ChunkedArray.filter (fun symbolUse -> symbolUse.ItemOccurence <> ItemOccurence.RelatedText) + |> ChunkedArray.map (fun symbolUse -> FSharpSymbolUse(tcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range)) + |> ChunkedArray.toArray |> async.Return // Not, this does not have to be a SyncOp, it can be called from any thread member __.GetAllUsesOfAllSymbols() = let (tcGlobals, tcImports, thisCcu, ccuSig, tcSymbolUses, _topAttribs, _tcAssemblyData, _ilAssemRef, _ad, _tcAssemblyExpr, _dependencyFiles) = getDetails() let cenv = SymbolEnv(tcGlobals, thisCcu, Some ccuSig, tcImports) + let res = ResizeArray() - [| for r in tcSymbolUses do - for symbolUseChunk in r.AllUsesOfSymbols do - for symbolUse in symbolUseChunk do + tcSymbolUses + |> List.iter (fun r -> + r.AllUsesOfSymbols + |> ChunkedArray.iter (fun symbolUse -> if symbolUse.ItemOccurence <> ItemOccurence.RelatedText then let symbol = FSharpSymbol.Create(cenv, symbolUse.Item) - yield FSharpSymbolUse(tcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range) |] + res.Add(FSharpSymbolUse(tcGlobals, symbolUse.DisplayEnv, symbol, symbolUse.ItemOccurence, symbolUse.Range)))) + res.ToArray() |> async.Return member __.ProjectContext = diff --git a/vsintegration/Directory.Build.targets b/vsintegration/Directory.Build.targets index bc5c6d7195c..7b6e317d4ff 100644 --- a/vsintegration/Directory.Build.targets +++ b/vsintegration/Directory.Build.targets @@ -15,6 +15,7 @@ +