From 4245e1a13cfac267d9fed9a5a3ffd8b682ba9787 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sat, 14 Dec 2019 23:06:36 -0800 Subject: [PATCH 01/11] Fixed LOH byte array allocations in the IDE. Unified memory handling for IL and FSharp metadata. --- src/absil/bytes.fs | 264 ++++++++++++++++++++++++- src/absil/bytes.fsi | 62 +++++- src/absil/il.fs | 10 +- src/absil/il.fsi | 11 +- src/absil/ilprint.fs | 3 +- src/absil/ilread.fs | 159 ++++----------- src/absil/ilreflect.fs | 7 +- src/absil/ilwrite.fs | 3 +- src/fsharp/CompileOps.fs | 16 +- src/fsharp/CompileOps.fsi | 5 +- src/fsharp/TastPickle.fs | 12 +- src/fsharp/TastPickle.fsi | 2 +- src/fsharp/fsc.fs | 8 +- src/fsharp/service/IncrementalBuild.fs | 3 +- src/fsharp/service/service.fs | 2 +- 15 files changed, 391 insertions(+), 176 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index fd5f14247cf..7864f032a23 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -3,7 +3,14 @@ /// Byte arrays namespace FSharp.Compiler.AbstractIL.Internal +open System +open System.IO +open System.IO.MemoryMappedFiles +open System.Runtime.InteropServices +open System.Runtime.CompilerServices +open FSharp.NativeInterop +#nowarn "9" module internal Bytes = let b0 n = (n &&& 0xFF) @@ -26,10 +33,255 @@ module internal Bytes = Array.append (System.Text.Encoding.UTF8.GetBytes s) (ofInt32Array [| 0x0 |]) let stringAsUnicodeNullTerminated (s:string) = - Array.append (System.Text.Encoding.Unicode.GetBytes s) (ofInt32Array [| 0x0;0x0 |]) + Array.append (System.Text.Encoding.Unicode.GetBytes s) (ofInt32Array [| 0x0;0x0 |]) + +[] +type ByteMemory () = + + abstract Item: int -> byte with get, set + + abstract Length: int + + abstract GetBytes: pos: int * count: int -> byte[] + + abstract GetInt32: pos: int -> int + + abstract GetUInt16: pos: int -> uint16 + + abstract GetUtf8String: pos: int * count: int -> string + + abstract Slice: pos: int * count: int -> ByteMemory + + abstract CopyTo: Stream -> unit + + abstract ToArray: unit -> byte[] + + abstract AsStream: unit -> Stream + +[] +type ByteArrayMemory(bytes: byte[], offset, length) = + inherit ByteMemory() + + let check i = + if i < 0 || i >= length then + failwith "out of range" + + do + if offset < 0 then + failwith "offset is less than zero" + + if length <= 0 then + failwith "length is less than or equal to zero" + + if (offset + length) > bytes.Length then + failwith "span larger than byte array" + + override _.Item + with get i = + check i + bytes.[offset + i] + and set i v = + check i + bytes.[offset + i] <- v + + override _.Length = length + + override _.GetBytes(pos, count) = + Array.sub bytes (offset + pos) count + + override _.GetInt32 pos = + BitConverter.ToInt32(bytes, pos) + + override _.GetUInt16 pos = + BitConverter.ToUInt16(bytes, pos) + + override _.GetUtf8String(pos, count) = + System.Text.Encoding.UTF8.GetString(bytes, pos, count) + + override _.Slice(pos, count) = + ByteArrayMemory(bytes, offset + pos, count) :> ByteMemory + + override _.CopyTo stream = + stream.Write(bytes, offset, length) + + override _.ToArray() = + Array.sub bytes offset length + + override _.AsStream() = + new MemoryStream(bytes, offset, length) :> Stream + +[] +type RawByteMemory(addr: nativeptr, length: int, hold: obj) = + inherit ByteMemory () + + let check i = + if i < 0 || i >= length then + failwith "out of range" + + override _.Item + with get i = + check i + NativePtr.add addr i + |> NativePtr.read + and set i v = + check i + NativePtr.set addr i v + + override _.Length = length + + override _.GetUtf8String(pos, count) = + check pos + check (pos + count - 1) + System.Text.Encoding.UTF8.GetString(NativePtr.add addr pos, count) + + override _.GetBytes(pos, count) = + check pos + check (pos + count - 1) + let res = Bytes.zeroCreate count + Marshal.Copy(NativePtr.toNativeInt addr + nativeint pos, res, 0, count) + res + + override _.GetInt32 pos = + check pos + check (pos + 3) + Marshal.ReadInt32(NativePtr.toNativeInt addr + nativeint pos) + + override _.GetUInt16 pos = + check pos + check (pos + 1) + uint16(Marshal.ReadInt16(NativePtr.toNativeInt addr + nativeint pos)) + + override _.Slice(pos, count) = + check pos + check (pos + count - 1) + RawByteMemory(NativePtr.add addr pos, count, hold) :> ByteMemory + + override x.CopyTo stream = + use stream2 = x.AsStream() + stream2.CopyTo stream + + override _.ToArray() = + let res = Array.zeroCreate length + Marshal.Copy(NativePtr.toNativeInt addr, res, 0, res.Length) + res + + override _.AsStream() = + new UnmanagedMemoryStream(addr, int64 length) :> Stream + +[] +type ReadOnlyByteMemory(bytes: ByteMemory) = + + member _.Item with [] get i = bytes.[i] + + member _.Length with [] get () = bytes.Length + + [] + member _.GetBytes(pos, count) = bytes.GetBytes(pos, count) + + [] + member _.GetInt32 pos = bytes.GetInt32 pos + + [] + member _.GetUInt16 pos = bytes.GetUInt16 pos + + [] + member _.GetUtf8String(pos, count) = bytes.GetUtf8String(pos, count) + + [] + member _.Slice(pos, count) = bytes.Slice(pos, count) |> ReadOnlyByteMemory + + [] + member _.ToArray() = bytes.ToArray() + +type ByteMemory with + + static member CreateMemoryMappedFile(bytes: ByteMemory) = + let length = int64 bytes.Length + let mmf = + let mmf = + MemoryMappedFile.CreateNew( + null, + length, + MemoryMappedFileAccess.ReadWrite, + MemoryMappedFileOptions.None, + HandleInheritability.None) + use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) + bytes.CopyTo stream + mmf + + let accessor = mmf.CreateViewAccessor(0L, length, MemoryMappedFileAccess.ReadWrite) + + let safeHolder = + { new obj() with + override x.Finalize() = + (x :?> IDisposable).Dispose() + interface IDisposable with + member x.Dispose() = + GC.SuppressFinalize x + accessor.Dispose() + mmf.Dispose() } + RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, safeHolder) + + static member FromFile(path, access, ?canShadowCopy: bool) = + let canShadowCopy = defaultArg canShadowCopy false + + let memoryMappedFileAccess = + match access with + | FileAccess.Read -> MemoryMappedFileAccess.Read + | FileAccess.Write -> MemoryMappedFileAccess.Write + | _ -> MemoryMappedFileAccess.ReadWrite + + let mmf, accessor, length = + let fileStream = File.Open(path, FileMode.Open, access, FileShare.Read) + let length = fileStream.Length + let mmf = + if canShadowCopy then + let mmf = + MemoryMappedFile.CreateNew( + null, + length, + MemoryMappedFileAccess.ReadWrite, + MemoryMappedFileOptions.None, + HandleInheritability.None) + use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) + fileStream.CopyTo(stream) + fileStream.Dispose() + mmf + else + MemoryMappedFile.CreateFromFile( + fileStream, + null, + length, + memoryMappedFileAccess, + HandleInheritability.None, + leaveOpen=false) + mmf, mmf.CreateViewAccessor(0L, length, memoryMappedFileAccess), length + + match access with + | FileAccess.Read when not accessor.CanRead -> failwith "Cannot read file" + | FileAccess.Write when not accessor.CanWrite -> failwith "Cannot write file" + | _ when not accessor.CanRead || not accessor.CanWrite -> failwith "Cannot read or write file" + | _ -> () + + let safeHolder = + { new obj() with + override x.Finalize() = + (x :?> IDisposable).Dispose() + interface IDisposable with + member x.Dispose() = + GC.SuppressFinalize x + accessor.Dispose() + mmf.Dispose() } + RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, safeHolder) + + static member FromUnsafePointer(addr, length, hold: obj) = + RawByteMemory(NativePtr.ofNativeInt addr, length, hold) :> ByteMemory + + static member FromArray(bytes, offset, length) = + ByteArrayMemory(bytes, offset, length) :> ByteMemory type internal ByteStream = - { bytes: byte[] + { bytes: ReadOnlyByteMemory mutable pos: int max: int } member b.ReadByte() = @@ -38,18 +290,18 @@ type internal ByteStream = b.pos <- b.pos + 1 res member b.ReadUtf8String n = - let res = System.Text.Encoding.UTF8.GetString(b.bytes,b.pos,n) + let res = b.bytes.GetUtf8String(b.pos,n) b.pos <- b.pos + n; res - static member FromBytes (b:byte[],n,len) = + static member FromBytes (b: ReadOnlyByteMemory,n,len) = if n < 0 || (n+len) > b.Length then failwith "FromBytes" { bytes = b; pos = n; max = n+len } member b.ReadBytes n = if b.pos + n > b.max then failwith "ReadBytes: end of stream" - let res = Bytes.sub b.bytes b.pos n + let res = b.bytes.Slice(b.pos, n) b.pos <- b.pos + n - res + res member b.Position = b.pos #if LAZY_UNPICKLE diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index c611e80c776..190b33ede82 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -3,6 +3,7 @@ /// Blobs of bytes, cross-compiling namespace FSharp.Compiler.AbstractIL.Internal +open System.IO open Internal.Utilities open FSharp.Compiler.AbstractIL @@ -22,6 +23,63 @@ module internal Bytes = val stringAsUnicodeNullTerminated: string -> byte[] val stringAsUtf8NullTerminated: string -> byte[] +/// May be backed by managed or unmanaged memory, or memory mapped file. +[] +type internal ByteMemory = + + abstract Item: int -> byte with get + + abstract Length: int + + abstract GetBytes: pos: int * count: int -> byte[] + + abstract GetInt32: pos: int -> int + + abstract GetUInt16: pos: int -> uint16 + + abstract GetUtf8String: pos: int * count: int -> string + + abstract Slice: pos: int * count: int -> ByteMemory + + abstract CopyTo: Stream -> unit + + abstract ToArray: unit -> byte[] + + abstract AsStream: unit -> Stream + + /// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents. + static member CreateMemoryMappedFile: ByteMemory -> ByteMemory + + /// Creates a ByteMemory object that has a backing memory mapped file. + static member FromFile: path: string * FileAccess * ?canShadowCopy: bool -> ByteMemory + + /// Creates a ByteMemory object that is backed by a raw pointer. + /// Use with care. + static member FromUnsafePointer: addr: nativeint * length: int * hold: obj -> ByteMemory + + /// Creates a ByteMemory object that is backed by a byte array with the specified offset and length. + static member FromArray: bytes: byte[] * offset: int * length: int -> ByteMemory + +[] +type internal ReadOnlyByteMemory = + + new: ByteMemory -> ReadOnlyByteMemory + + member Item: int -> byte with get + + member Length: int + + member GetBytes: pos: int * count: int -> byte[] + + member GetInt32: pos: int -> int + + member GetUInt16: pos: int -> uint16 + + member GetUtf8String: pos: int * count: int -> string + + member Slice: pos: int * count: int -> ReadOnlyByteMemory + + member ToArray: unit -> byte[] /// Imperative buffers and streams of byte[] [] @@ -44,10 +102,10 @@ type internal ByteBuffer = [] type internal ByteStream = member ReadByte : unit -> byte - member ReadBytes : int -> byte[] + member ReadBytes : int -> ReadOnlyByteMemory member ReadUtf8String : int -> string member Position : int - static member FromBytes : byte[] * start:int * length:int -> ByteStream + static member FromBytes : ReadOnlyByteMemory * start:int * length:int -> ByteStream #if LAZY_UNPICKLE member CloneAndSeek : int -> ByteStream diff --git a/src/absil/il.fs b/src/absil/il.fs index fb6e5bc320a..83cceb602fc 100644 --- a/src/absil/il.fs +++ b/src/absil/il.fs @@ -2208,8 +2208,7 @@ type ILResourceAccess = [] type ILResourceLocation = - | LocalIn of string * int * int - | LocalOut of byte[] + | Local of ByteMemory | File of ILModuleRef * int32 | Assembly of ILAssemblyRef @@ -2223,9 +2222,7 @@ type ILResource = /// Read the bytes from a resource local to an assembly member r.GetBytes() = match r.Location with - | ILResourceLocation.LocalIn (file, start, len) -> - File.ReadBinaryChunk(file, start, len) - | ILResourceLocation.LocalOut bytes -> bytes + | ILResourceLocation.Local bytes -> bytes | _ -> failwith "GetBytes" member x.CustomAttrs = x.CustomAttrsStored.GetCustomAttrs x.MetadataIndex @@ -4208,8 +4205,7 @@ and refs_of_exported_types s (tab: ILExportedTypesAndForwarders) = List.iter (re and refs_of_resource_where s x = match x with - | ILResourceLocation.LocalIn _ -> () - | ILResourceLocation.LocalOut _ -> () + | ILResourceLocation.Local _ -> () | ILResourceLocation.File (mref, _) -> refs_of_modref s mref | ILResourceLocation.Assembly aref -> refs_of_assemblyRef s aref diff --git a/src/absil/il.fsi b/src/absil/il.fsi index e981e8e8b54..5a3562899f2 100644 --- a/src/absil/il.fsi +++ b/src/absil/il.fsi @@ -3,6 +3,7 @@ /// The "unlinked" view of .NET metadata and code. Central to the Abstract IL library module public FSharp.Compiler.AbstractIL.IL +open FSharp.Compiler.AbstractIL.Internal open System.Collections.Generic open System.Reflection @@ -1405,11 +1406,9 @@ type ILResourceAccess = [] type ILResourceLocation = - /// Represents a manifest resource that can be read from within the PE file - | LocalIn of string * int * int - - /// Represents a manifest resource that is due to be written to the output PE file - | LocalOut of byte[] + internal + /// Represents a manifest resource that can be read or written to a PE file + | Local of ByteMemory /// Represents a manifest resource in an associated file | File of ILModuleRef * int32 @@ -1429,7 +1428,7 @@ type ILResource = MetadataIndex: int32 } /// Read the bytes from a resource local to an assembly. Will fail for non-local resources. - member GetBytes : unit -> byte[] + member internal GetBytes : unit -> ByteMemory member CustomAttrs: ILAttributes diff --git a/src/absil/ilprint.fs b/src/absil/ilprint.fs index 7f408d48cbc..2515634395f 100644 --- a/src/absil/ilprint.fs +++ b/src/absil/ilprint.fs @@ -1024,8 +1024,7 @@ let goutput_resource env os r = output_string os " { " goutput_custom_attrs env os r.CustomAttrs match r.Location with - | ILResourceLocation.LocalIn _ - | ILResourceLocation.LocalOut _ -> + | ILResourceLocation.Local _ -> output_string os " /* loc nyi */ " | ILResourceLocation.File (mref, off) -> output_string os " .file " diff --git a/src/absil/ilread.fs b/src/absil/ilread.fs index de201333a0d..9b7348c31ea 100644 --- a/src/absil/ilread.fs +++ b/src/absil/ilread.fs @@ -19,6 +19,7 @@ open System.Runtime.InteropServices open System.Text open Internal.Utilities open Internal.Utilities.Collections +open FSharp.NativeInterop open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.Internal.Support open FSharp.Compiler.AbstractIL.Diagnostics @@ -29,6 +30,8 @@ open FSharp.Compiler.ErrorLogger open FSharp.Compiler.Range open System.Reflection +#nowarn "9" + let checking = false let logging = false let _ = if checking then dprintn "warning: ILBinaryReader.checking is on" @@ -104,125 +107,31 @@ let stats = let GetStatistics() = stats -[] -/// An abstraction over how we access the contents of .NET binaries. May be backed by managed or unmanaged memory, -/// memory mapped file or by on-disk resources. These objects should never need explicit disposal - they must either -/// not hold resources of clean up after themselves when collected. -type BinaryView() = - - /// Read a byte from the file - abstract ReadByte: addr: int -> byte - - /// Read a chunk of bytes from the file - abstract ReadBytes: addr: int -> int -> byte[] - - /// Read an Int32 from the file - abstract ReadInt32: addr: int -> int +type private BinaryView = ByteMemory - /// Read a UInt16 from the file - abstract ReadUInt16: addr: int -> uint16 - - /// Read a length of a UTF8 string from the file - abstract CountUtf8String: addr: int -> int - - /// Read a UTF8 string from the file - abstract ReadUTF8String: addr: int -> string - -/// An abstraction over how we access the contents of .NET binaries. May be backed by managed or unmanaged memory, -/// memory mapped file or by on-disk resources. +/// An abstraction over how we access the contents of .NET binaries. type BinaryFile = - /// Return a BinaryView for temporary use which eagerly holds any necessary memory resources for the duration of its lifetime, - /// and is faster to access byte-by-byte. The returned BinaryView should _not_ be captured in a closure that outlives the - /// desired lifetime. abstract GetView: unit -> BinaryView -/// A view over a raw pointer to memory -type RawMemoryView(obj: obj, start: nativeint, len: int) = - inherit BinaryView() - - override m.ReadByte i = - if nativeint i + 1n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - Marshal.ReadByte(start + nativeint i) - - override m.ReadBytes i n = - if nativeint i + nativeint n > nativeint len then failwithf "RawMemoryView overrun, i = %d, n = %d, obj = %A" i n obj - let res = Bytes.zeroCreate n - Marshal.Copy(start + nativeint i, res, 0, n) - res - - override m.ReadInt32 i = - if nativeint i + 4n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - Marshal.ReadInt32(start + nativeint i) - - override m.ReadUInt16 i = - if nativeint i + 2n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - uint16(Marshal.ReadInt16(start + nativeint i)) - - override m.CountUtf8String i = - if nativeint i > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj - let pStart = start + nativeint i - let mutable p = start - while Marshal.ReadByte p <> 0uy do - p <- p + 1n - int (p - pStart) - - override m.ReadUTF8String i = - let n = m.CountUtf8String i - if nativeint i + nativeint n > nativeint len then failwithf "RawMemoryView overrun, i = %d, n = %d, obj = %A" i n obj - System.Runtime.InteropServices.Marshal.PtrToStringAnsi(start + nativeint i, n) - - member __.HoldObj() = obj - - /// Gives views over a raw chunk of memory, for example those returned to us by the memory manager in Roslyn's /// Visual Studio integration. 'obj' must keep the memory alive. The object will capture it and thus also keep the memory alive for /// the lifetime of this object. type RawMemoryFile(fileName: string, obj: obj, addr: nativeint, length: int) = do stats.rawMemoryFileCount <- stats.rawMemoryFileCount + 1 - let view = RawMemoryView(obj, addr, length) + let view = ByteMemory.FromUnsafePointer(addr, length, obj) member __.HoldObj() = obj // make sure we capture 'obj' member __.FileName = fileName interface BinaryFile with - override __.GetView() = view :>_ - -/// Read file from memory blocks -type ByteView(bytes: byte[]) = - inherit BinaryView() - - override __.ReadByte addr = bytes.[addr] - - override __.ReadBytes addr len = Array.sub bytes addr len - - override __.CountUtf8String addr = - let mutable p = addr - while bytes.[p] <> 0uy do - p <- p + 1 - p - addr - - override bfv.ReadUTF8String addr = - let n = bfv.CountUtf8String addr - System.Text.Encoding.UTF8.GetString (bytes, addr, n) - - override bfv.ReadInt32 addr = - let b0 = bfv.ReadByte addr - let b1 = bfv.ReadByte (addr+1) - let b2 = bfv.ReadByte (addr+2) - let b3 = bfv.ReadByte (addr+3) - int b0 ||| (int b1 <<< 8) ||| (int b2 <<< 16) ||| (int b3 <<< 24) - - override bfv.ReadUInt16 addr = - let b0 = bfv.ReadByte addr - let b1 = bfv.ReadByte (addr+1) - uint16 b0 ||| (uint16 b1 <<< 8) + override __.GetView() = view /// A BinaryFile backed by an array of bytes held strongly as managed memory [] type ByteFile(fileName: string, bytes: byte[]) = - let view = ByteView bytes + let view = ByteMemory.FromArray(bytes, 0, bytes.Length) do stats.byteFileCount <- stats.byteFileCount + 1 member __.FileName = fileName interface BinaryFile with - override bf.GetView() = view :> BinaryView + override bf.GetView() = view /// Same as ByteFile but holds the bytes weakly. The bytes will be re-read from the backing file when a view is requested. /// This is the default implementation used by F# Compiler Services when accessing "stable" binaries. It is not used @@ -261,13 +170,13 @@ type WeakByteFile(fileName: string, chunk: (int * int) option) = tg - (ByteView strongBytes :> BinaryView) + ByteMemory.FromArray(strongBytes, 0, strongBytes.Length) -let seekReadByte (mdv: BinaryView) addr = mdv.ReadByte addr -let seekReadBytes (mdv: BinaryView) addr len = mdv.ReadBytes addr len -let seekReadInt32 (mdv: BinaryView) addr = mdv.ReadInt32 addr -let seekReadUInt16 (mdv: BinaryView) addr = mdv.ReadUInt16 addr +let seekReadByte (mdv: BinaryView) addr = mdv.[addr] +let seekReadBytes (mdv: BinaryView) addr len = mdv.GetBytes(addr, len) +let seekReadInt32 (mdv: BinaryView) addr = mdv.GetInt32 addr +let seekReadUInt16 (mdv: BinaryView) addr = mdv.GetUInt16 addr let seekReadByteAsInt32 mdv addr = int32 (seekReadByte mdv addr) @@ -1501,8 +1410,8 @@ let rvaToData (ctxt: ILMetadataReader) (pectxt: PEReader) nm rva = let isSorted (ctxt: ILMetadataReader) (tab: TableName) = ((ctxt.sorted &&& (int64 1 <<< tab.Index)) <> int64 0x0) -// Note, pectxtEager and pevEager must not be captured by the results of this function -let rec seekReadModule (ctxt: ILMetadataReader) (pectxtEager: PEReader) pevEager peinfo ilMetadataVersion idx = +// Note, pectxtEager must not be captured by the results of this function +let rec seekReadModule (ctxt: ILMetadataReader) canReduceMemory (pectxtEager: PEReader) pevEager peinfo ilMetadataVersion idx = let (subsys, subsysversion, useHighEntropyVA, ilOnly, only32, is32bitpreferred, only64, platform, isDll, alignVirt, alignPhys, imageBaseReal) = peinfo let mdv = ctxt.mdfile.GetView() let (_generation, nameIdx, _mvidIdx, _encidIdx, _encbaseidIdx) = seekReadModuleRow ctxt mdv idx @@ -1531,7 +1440,7 @@ let rec seekReadModule (ctxt: ILMetadataReader) (pectxtEager: PEReader) pevEager PhysicalAlignment = alignPhys ImageBase = imageBaseReal MetadataVersion = ilMetadataVersion - Resources = seekReadManifestResources ctxt mdv pectxtEager pevEager } + Resources = seekReadManifestResources ctxt canReduceMemory mdv pectxtEager pevEager } and seekReadAssemblyManifest (ctxt: ILMetadataReader) pectxt idx = let mdview = ctxt.mdfile.GetView() @@ -3101,10 +3010,9 @@ and sigptrGetILNativeType ctxt bytes sigptr = else sigptrGetZInt32 bytes sigptr ILNativeType.Array (Some nt, Some(pnum, Some additive)), sigptr else (ILNativeType.Empty, sigptr) - + // Note, pectxtEager and pevEager must not be captured by the results of this function -// As a result, reading the resource offsets in the physical file is done eagerly to avoid holding on to any resources -and seekReadManifestResources (ctxt: ILMetadataReader) (mdv: BinaryView) (pectxtEager: PEReader) (pevEager: BinaryView) = +and seekReadManifestResources (ctxt: ILMetadataReader) canReduceMemory (mdv: BinaryView) (pectxtEager: PEReader) (pevEager: BinaryView) = mkILResources [ for i = 1 to ctxt.getNumRows TableNames.ManifestResource do let (offset, flags, nameIdx, implIdx) = seekReadManifestResourceRow ctxt mdv i @@ -3117,10 +3025,14 @@ and seekReadManifestResources (ctxt: ILMetadataReader) (mdv: BinaryView) (pectxt let start = pectxtEager.anyV2P ("resource", offset + pectxtEager.resourcesAddr) let resourceLength = seekReadInt32 pevEager start let offsetOfBytesFromStartOfPhysicalPEFile = start + 4 - if pectxtEager.noFileOnDisk then - ILResourceLocation.LocalOut (seekReadBytes pevEager offsetOfBytesFromStartOfPhysicalPEFile resourceLength) - else - ILResourceLocation.LocalIn (ctxt.fileName, offsetOfBytesFromStartOfPhysicalPEFile, resourceLength) + let bytes = + let bytes = pevEager.Slice(offsetOfBytesFromStartOfPhysicalPEFile, resourceLength) + // If we are trying to reduce memory, create a memory mapped file based on the contents. + if canReduceMemory then + ByteMemory.CreateMemoryMappedFile bytes + else + ByteMemory.FromArray(bytes.ToArray(), 0, bytes.Length) + ILResourceLocation.Local bytes | ILScopeRef.Module mref -> ILResourceLocation.File (mref, offset) | ILScopeRef.Assembly aref -> ILResourceLocation.Assembly aref @@ -3207,7 +3119,7 @@ let getPdbReader pdbDirPath fileName = #endif // Note, pectxtEager and pevEager must not be captured by the results of this function -let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, pectxtEager: PEReader, pevEager: BinaryView, pectxtCaptured, reduceMemoryUsage, ilGlobals) = +let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, pectxtEager: PEReader, pevEager, pectxtCaptured, reduceMemoryUsage, ilGlobals) = let mdv = mdfile.GetView() let magic = seekReadUInt16AsInt32 mdv metadataPhysLoc if magic <> 0x5342 then failwith (fileName + ": bad metadata magic number: " + string magic) @@ -3584,7 +3496,7 @@ let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, p tableBigness=tableBigness } ctxtH := Some ctxt - let ilModule = seekReadModule ctxt pectxtEager pevEager peinfo (System.Text.Encoding.UTF8.GetString (ilMetadataVersion, 0, ilMetadataVersion.Length)) 1 + let ilModule = seekReadModule ctxt reduceMemoryUsage pectxtEager pevEager peinfo (System.Text.Encoding.UTF8.GetString (ilMetadataVersion, 0, ilMetadataVersion.Length)) 1 let ilAssemblyRefs = lazy [ for i in 1 .. getNumRows TableNames.AssemblyRef do yield seekReadAssemblyRef ctxt i ] ilModule, ilAssemblyRefs @@ -3797,8 +3709,8 @@ let openPE (fileName, pefile, pdbDirPath, reduceMemoryUsage, ilGlobals, noFileOn let ilModule, ilAssemblyRefs = openMetadataReader (fileName, pefile, metadataPhysLoc, peinfo, pectxt, pev, Some pectxt, reduceMemoryUsage, ilGlobals) ilModule, ilAssemblyRefs, pdb -let openPEMetadataOnly (fileName, peinfo, pectxtEager, pev, mdfile: BinaryFile, reduceMemoryUsage, ilGlobals) = - openMetadataReader (fileName, mdfile, 0, peinfo, pectxtEager, pev, None, reduceMemoryUsage, ilGlobals) +let openPEMetadataOnly (fileName, peinfo, pectxtEager, pevEager, mdfile: BinaryFile, reduceMemoryUsage, ilGlobals) = + openMetadataReader (fileName, mdfile, 0, peinfo, pectxtEager, pevEager, None, reduceMemoryUsage, ilGlobals) let ClosePdbReader pdb = #if FX_NO_PDB_READER @@ -3992,13 +3904,12 @@ let OpenILModuleReader fileName opts = // We do however care about avoiding locks on files that prevent their deletion during a // multi-proc build. So use memory mapping, but only for stable files. Other files // still use an in-memory ByteFile - let _disposer, pefile = + let pefile = if alwaysMemoryMapFSC || stableFileHeuristicApplies fullPath then - createMemoryMapFile fullPath + let _, pefile = createMemoryMapFile fullPath + pefile else - let pefile = createByteFileChunk opts fullPath None - let disposer = { new IDisposable with member __.Dispose() = () } - disposer, pefile + createByteFileChunk opts fullPath None let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbDirPath, reduceMemoryUsage, opts.ilGlobals, false) let ilModuleReader = new ILModuleReaderImpl(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb)) diff --git a/src/absil/ilreflect.fs b/src/absil/ilreflect.fs index 69209100b98..329f95095ec 100644 --- a/src/absil/ilreflect.fs +++ b/src/absil/ilreflect.fs @@ -2070,11 +2070,8 @@ let buildModuleFragment cenv emEnv (asmB: AssemblyBuilder) (modB: ModuleBuilder) m.Resources.AsList |> List.iter (fun r -> let attribs = (match r.Access with ILResourceAccess.Public -> ResourceAttributes.Public | ILResourceAccess.Private -> ResourceAttributes.Private) match r.Location with - | ILResourceLocation.LocalIn (file, start, len) -> - let bytes = FileSystem.ReadAllBytesShim(file).[start .. start + len - 1] - modB.DefineManifestResourceAndLog (r.Name, new MemoryStream(bytes), attribs) - | ILResourceLocation.LocalOut bytes -> - modB.DefineManifestResourceAndLog (r.Name, new MemoryStream(bytes), attribs) + | ILResourceLocation.Local bytes -> + modB.DefineManifestResourceAndLog (r.Name, bytes.AsStream(), attribs) | ILResourceLocation.File (mr, _) -> asmB.AddResourceFileAndLog (r.Name, mr.Name, attribs) | ILResourceLocation.Assembly _ -> diff --git a/src/absil/ilwrite.fs b/src/absil/ilwrite.fs index 409c91d590b..056b495f37d 100644 --- a/src/absil/ilwrite.fs +++ b/src/absil/ilwrite.fs @@ -2702,8 +2702,7 @@ let rec GetResourceAsManifestResourceRow cenv r = Data (alignedOffset, true), (i_File, 0) match r.Location with - | ILResourceLocation.LocalIn _ -> embedManagedResources (r.GetBytes()) - | ILResourceLocation.LocalOut bytes -> embedManagedResources bytes + | ILResourceLocation.Local bytes -> embedManagedResources (bytes.ToArray()) // TODO: | ILResourceLocation.File (mref, offset) -> ULong offset, (i_File, GetModuleRefAsFileIdx cenv mref) | ILResourceLocation.Assembly aref -> ULong 0x0, (i_AssemblyRef, GetAssemblyRefAsIdx cenv aref) diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index ab20bfcf139..ab25947a57e 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -1893,9 +1893,9 @@ type IRawFSharpAssemblyData = /// in the language service abstract TryGetILModuleDef: unit -> ILModuleDef option /// The raw F# signature data in the assembly, if any - abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The raw F# optimization data in the assembly, if any - abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The table of type forwarders in the assembly abstract GetRawTypeForwarders: unit -> ILExportedTypesAndForwarders /// The identity of the module @@ -3589,7 +3589,7 @@ let IsReflectedDefinitionsResource (r: ILResource) = let MakeILResource rName bytes = { Name = rName - Location = ILResourceLocation.LocalOut bytes + Location = ILResourceLocation.Local(ByteMemory.FromArray(bytes, 0, bytes.Length)) Access = ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -3598,7 +3598,7 @@ let PickleToResource inMem file (g: TcGlobals) scope rName p x = let file = PathMap.apply g.pathMap file { Name = rName - Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.LocalOut bytes) + Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.Local(ByteMemory.FromArray(bytes, 0, bytes.Length))) Access = ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -3649,21 +3649,21 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR [ for iresource in resources do if IsSignatureDataResource iresource then let ccuName = GetSignatureDataResourceName iresource - yield (ccuName, fun () -> iresource.GetBytes()) ] + yield (ccuName, fun () -> iresource.GetBytes() |> ReadOnlyByteMemory) ] let sigDataReaders = if sigDataReaders.IsEmpty && List.contains ilShortAssemName externalSigAndOptData then let sigFileName = Path.ChangeExtension(filename, "sigdata") if not (FileSystem.SafeExists sigFileName) then error(Error(FSComp.SR.buildExpectedSigdataFile (FileSystem.GetFullPathShim sigFileName), m)) - [ (ilShortAssemName, fun () -> FileSystem.ReadAllBytesShim sigFileName)] + [ (ilShortAssemName, fun () -> ByteMemory.FromFile(sigFileName, FileAccess.Read, canShadowCopy=true) |> ReadOnlyByteMemory)] else sigDataReaders sigDataReaders member __.GetRawFSharpOptimizationData(m, ilShortAssemName, filename) = let optDataReaders = ilModule.Resources.AsList - |> List.choose (fun r -> if IsOptimizationDataResource r then Some(GetOptimizationDataResourceName r, (fun () -> r.GetBytes())) else None) + |> List.choose (fun r -> if IsOptimizationDataResource r then Some(GetOptimizationDataResourceName r, (fun () -> r.GetBytes() |> ReadOnlyByteMemory)) else None) // Look for optimization data in a file let optDataReaders = @@ -3671,7 +3671,7 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR let optDataFile = Path.ChangeExtension(filename, "optdata") if not (FileSystem.SafeExists optDataFile) then error(Error(FSComp.SR.buildExpectedFileAlongSideFSharpCore(optDataFile, FileSystem.GetFullPathShim optDataFile), m)) - [ (ilShortAssemName, (fun () -> FileSystem.ReadAllBytesShim optDataFile))] + [ (ilShortAssemName, (fun () -> ByteMemory.FromFile(optDataFile, FileAccess.Read, canShadowCopy=true) |> ReadOnlyByteMemory))] else optDataReaders optDataReaders diff --git a/src/fsharp/CompileOps.fsi b/src/fsharp/CompileOps.fsi index 51b10bc0287..230bd508de5 100644 --- a/src/fsharp/CompileOps.fsi +++ b/src/fsharp/CompileOps.fsi @@ -11,6 +11,7 @@ open FSharp.Compiler.AbstractIL open FSharp.Compiler.AbstractIL.IL open FSharp.Compiler.AbstractIL.ILBinaryReader open FSharp.Compiler.AbstractIL.ILPdbWriter +open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.Internal.Library open FSharp.Compiler open FSharp.Compiler.TypeChecker @@ -154,9 +155,9 @@ type IRawFSharpAssemblyData = abstract HasAnyFSharpSignatureDataAttribute: bool abstract HasMatchingFSharpSignatureDataAttribute: ILGlobals -> bool /// The raw F# signature data in the assembly, if any - abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The raw F# optimization data in the assembly, if any - abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list + abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list /// The table of type forwarders in the assembly abstract GetRawTypeForwarders: unit -> ILExportedTypesAndForwarders /// The identity of the module diff --git a/src/fsharp/TastPickle.fs b/src/fsharp/TastPickle.fs index 97a723f6cfd..56717edafe1 100644 --- a/src/fsharp/TastPickle.fs +++ b/src/fsharp/TastPickle.fs @@ -832,7 +832,7 @@ let check (ilscope: ILScopeRef) (inMap : NodeInTable<_, _>) = // an identical copy of the source for the DLL containing the data being unpickled. A message will // then be printed indicating the name of the item. -let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u (phase2bytes: byte[]) = +let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u (phase2bytes: ReadOnlyByteMemory) = let st2 = { is = ByteStream.FromBytes (phase2bytes, 0, phase2bytes.Length) iilscope= ilscope @@ -923,8 +923,8 @@ let p_ILScopeRef x st = let u_ILPublicKey st = let tag = u_byte st match tag with - | 0 -> u_bytes st |> PublicKey - | 1 -> u_bytes st |> PublicKeyToken + | 0 -> (u_bytes st).ToArray() |> PublicKey + | 1 -> (u_bytes st).ToArray() |> PublicKeyToken | _ -> ufailwith st "u_ILPublicKey" let u_ILVersion st = @@ -933,14 +933,14 @@ let u_ILVersion st = let u_ILModuleRef st = let (a, b, c) = u_tup3 u_string u_bool (u_option u_bytes) st - ILModuleRef.Create(a, b, c) + ILModuleRef.Create(a, b, c |> Option.map (fun x -> x.ToArray())) let u_ILAssemblyRef st = let tag = u_byte st match tag with | 0 -> let a, b, c, d, e, f = u_tup6 u_string (u_option u_bytes) (u_option u_ILPublicKey) u_bool (u_option u_ILVersion) (u_option u_string) st - ILAssemblyRef.Create(a, b, c, d, e, f) + ILAssemblyRef.Create(a, b |> Option.map (fun x -> x.ToArray()), c, d, e, f) | _ -> ufailwith st "u_ILAssemblyRef" // IL scope references are rescoped as they are unpickled. This means @@ -2537,7 +2537,7 @@ and u_op st = | 20 -> TOp.While (NoSequencePointAtWhileLoop, NoSpecialWhileLoopMarker) | 21 -> let dir = match u_int st with 0 -> FSharpForLoopUp | 1 -> CSharpForLoopUp | 2 -> FSharpForLoopDown | _ -> failwith "unknown for loop" TOp.For (NoSequencePointAtForLoop, dir) - | 22 -> TOp.Bytes (u_bytes st) + | 22 -> TOp.Bytes ((u_bytes st).ToArray()) | 23 -> TOp.TryCatch (NoSequencePointAtTry, NoSequencePointAtWith) | 24 -> TOp.TryFinally (NoSequencePointAtTry, NoSequencePointAtFinally) | 25 -> let a = u_rfref st diff --git a/src/fsharp/TastPickle.fsi b/src/fsharp/TastPickle.fsi index 2b659b0598d..e3d9379c07d 100644 --- a/src/fsharp/TastPickle.fsi +++ b/src/fsharp/TastPickle.fsi @@ -142,7 +142,7 @@ val internal u_ty : unpickler val internal unpickleCcuInfo : ReaderState -> PickledCcuInfo /// Deserialize an arbitrary object which may have holes referring to other compilation units -val internal unpickleObjWithDanglingCcus : file:string -> viewedScope:ILScopeRef -> ilModule:ILModuleDef option -> ('T unpickler) -> byte[] -> PickledDataWithReferences<'T> +val internal unpickleObjWithDanglingCcus : file:string -> viewedScope:ILScopeRef -> ilModule:ILModuleDef option -> ('T unpickler) -> ReadOnlyByteMemory -> PickledDataWithReferences<'T> diff --git a/src/fsharp/fsc.fs b/src/fsharp/fsc.fs index bb46f75e1db..31f0ce0b3bd 100644 --- a/src/fsharp/fsc.fs +++ b/src/fsharp/fsc.fs @@ -442,7 +442,9 @@ let EncodeInterfaceData(tcConfig: TcConfig, tcGlobals, exportRemapping, generate let useDataFiles = (tcConfig.useOptimizationDataFile || tcGlobals.compilingFslib) && not isIncrementalBuild if useDataFiles then let sigDataFileName = (Filename.chopExtension outfile)+".sigdata" - File.WriteAllBytes(sigDataFileName, resource.GetBytes()) + let bytes = resource.GetBytes() + use fileStream = File.Create(sigDataFileName, bytes.Length) + bytes.AsStream().CopyTo(fileStream) let resources = [ resource ] let sigAttr = mkSignatureDataVersionAttr tcGlobals (IL.parseILVersion Internal.Utilities.FSharpEnvironment.FSharpBinaryMetadataFormatRevision) @@ -892,7 +894,7 @@ module MainModuleBuilder = [ ] let reflectedDefinitionResource = { Name=reflectedDefinitionResourceName - Location = ILResourceLocation.LocalOut reflectedDefinitionBytes + Location = ILResourceLocation.Local(ByteMemory.FromArray(reflectedDefinitionBytes, 0, reflectedDefinitionBytes.Length)) Access= ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -936,7 +938,7 @@ module MainModuleBuilder = let bytes = FileSystem.ReadAllBytesShim file name, bytes, pub yield { Name=name - Location=ILResourceLocation.LocalOut bytes + Location=ILResourceLocation.Local(ByteMemory.FromArray(bytes, 0, bytes.Length)) Access=pub CustomAttrsStored=storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } diff --git a/src/fsharp/service/IncrementalBuild.fs b/src/fsharp/service/IncrementalBuild.fs index 774fd78e108..fba68ac1bdd 100755 --- a/src/fsharp/service/IncrementalBuild.fs +++ b/src/fsharp/service/IncrementalBuild.fs @@ -14,6 +14,7 @@ open FSharp.Compiler.Lib open FSharp.Compiler.AbstractIL open FSharp.Compiler.AbstractIL.IL open FSharp.Compiler.AbstractIL.ILBinaryReader +open FSharp.Compiler.AbstractIL.Internal open FSharp.Compiler.AbstractIL.Internal.Library open FSharp.Compiler.CompileOps open FSharp.Compiler.CompileOptions @@ -1183,7 +1184,7 @@ type RawFSharpAssemblyDataBackedByLanguageService (tcConfig, tcGlobals, tcState: let _sigDataAttributes, sigDataResources = Driver.EncodeInterfaceData(tcConfig, tcGlobals, exportRemapping, generatedCcu, outfile, true) [ for r in sigDataResources do let ccuName = GetSignatureDataResourceName r - yield (ccuName, (fun () -> r.GetBytes())) ] + yield (ccuName, (fun () -> r.GetBytes() |> ReadOnlyByteMemory)) ] let autoOpenAttrs = topAttrs.assemblyAttrs |> List.choose (List.singleton >> TryFindFSharpStringAttribute tcGlobals tcGlobals.attrib_AutoOpenAttribute) diff --git a/src/fsharp/service/service.fs b/src/fsharp/service/service.fs index 254491005a5..a98cb2966b9 100644 --- a/src/fsharp/service/service.fs +++ b/src/fsharp/service/service.fs @@ -225,7 +225,7 @@ module CompileHelpers = // Register the reflected definitions for the dynamically generated assembly for resource in ilxMainModule.Resources.AsList do if IsReflectedDefinitionsResource resource then - Quotations.Expr.RegisterReflectedDefinitions (assemblyBuilder, moduleBuilder.Name, resource.GetBytes()) + Quotations.Expr.RegisterReflectedDefinitions (assemblyBuilder, moduleBuilder.Name, resource.GetBytes().ToArray()) // Save the result assemblyBuilderRef := Some assemblyBuilder From 8ad4ea4063e15b74ded313066ebbad9bd2135a0c Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sat, 14 Dec 2019 23:42:15 -0800 Subject: [PATCH 02/11] minor cleanup --- src/absil/bytes.fs | 28 +++++++++++++++++++++------- src/absil/bytes.fsi | 4 +++- src/absil/ilread.fs | 3 ++- src/fsharp/TastPickle.fs | 17 ++++++++++------- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index 7864f032a23..3df889d6190 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -54,6 +54,8 @@ type ByteMemory () = abstract CopyTo: Stream -> unit + abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit + abstract ToArray: unit -> byte[] abstract AsStream: unit -> Stream @@ -62,10 +64,6 @@ type ByteMemory () = type ByteArrayMemory(bytes: byte[], offset, length) = inherit ByteMemory() - let check i = - if i < 0 || i >= length then - failwith "out of range" - do if offset < 0 then failwith "offset is less than zero" @@ -78,10 +76,8 @@ type ByteArrayMemory(bytes: byte[], offset, length) = override _.Item with get i = - check i bytes.[offset + i] - and set i v = - check i + and set i v = bytes.[offset + i] <- v override _.Length = length @@ -104,6 +100,9 @@ type ByteArrayMemory(bytes: byte[], offset, length) = override _.CopyTo stream = stream.Write(bytes, offset, length) + override _.Copy(srcOffset, dest, destOffset, count) = + Array.blit bytes (offset + srcOffset) dest destOffset count + override _.ToArray() = Array.sub bytes offset length @@ -118,6 +117,10 @@ type RawByteMemory(addr: nativeptr, length: int, hold: obj) = if i < 0 || i >= length then failwith "out of range" + do + if length <= 0 then + failwith "length is less than or equal to zero" + override _.Item with get i = check i @@ -160,6 +163,10 @@ type RawByteMemory(addr: nativeptr, length: int, hold: obj) = use stream2 = x.AsStream() stream2.CopyTo stream + override x.Copy(srcOffset, dest, destOffset, count) = + check srcOffset + Marshal.Copy(NativePtr.toNativeInt addr + nativeint srcOffset, dest, destOffset, count) + override _.ToArray() = let res = Array.zeroCreate length Marshal.Copy(NativePtr.toNativeInt addr, res, 0, res.Length) @@ -360,6 +367,13 @@ type internal ByteBuffer = Bytes.blit i 0 buf.bbArray buf.bbCurrent n buf.bbCurrent <- newSize + member buf.EmitBytes (i:ByteMemory) = + let n = i.Length + let newSize = buf.bbCurrent + n + buf.Ensure newSize + i.Copy(0, buf.bbArray, buf.bbCurrent, n) + buf.bbCurrent <- newSize + member buf.EmitInt32AsUInt16 n = let newSize = buf.bbCurrent + 2 buf.Ensure newSize diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index 190b33ede82..08f87603b8e 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -43,6 +43,8 @@ type internal ByteMemory = abstract CopyTo: Stream -> unit + abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit + abstract ToArray: unit -> byte[] abstract AsStream: unit -> Stream @@ -50,7 +52,7 @@ type internal ByteMemory = /// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents. static member CreateMemoryMappedFile: ByteMemory -> ByteMemory - /// Creates a ByteMemory object that has a backing memory mapped file. + /// Creates a ByteMemory object that has a backing memory mapped file from a file on-disk. static member FromFile: path: string * FileAccess * ?canShadowCopy: bool -> ByteMemory /// Creates a ByteMemory object that is backed by a raw pointer. diff --git a/src/absil/ilread.fs b/src/absil/ilread.fs index 9b7348c31ea..b59c2b6dbc0 100644 --- a/src/absil/ilread.fs +++ b/src/absil/ilread.fs @@ -1410,7 +1410,7 @@ let rvaToData (ctxt: ILMetadataReader) (pectxt: PEReader) nm rva = let isSorted (ctxt: ILMetadataReader) (tab: TableName) = ((ctxt.sorted &&& (int64 1 <<< tab.Index)) <> int64 0x0) -// Note, pectxtEager must not be captured by the results of this function +// Note, pectxtEager and pevEager must not be captured by the results of this function let rec seekReadModule (ctxt: ILMetadataReader) canReduceMemory (pectxtEager: PEReader) pevEager peinfo ilMetadataVersion idx = let (subsys, subsysversion, useHighEntropyVA, ilOnly, only32, is32bitpreferred, only64, platform, isDll, alignVirt, alignPhys, imageBaseReal) = peinfo let mdv = ctxt.mdfile.GetView() @@ -3012,6 +3012,7 @@ and sigptrGetILNativeType ctxt bytes sigptr = else (ILNativeType.Empty, sigptr) // Note, pectxtEager and pevEager must not be captured by the results of this function +// As a result, reading the resource offsets in the physical file is done eagerly to avoid holding on to any resources and seekReadManifestResources (ctxt: ILMetadataReader) canReduceMemory (mdv: BinaryView) (pectxtEager: PEReader) (pevEager: BinaryView) = mkILResources [ for i = 1 to ctxt.getNumRows TableNames.ManifestResource do diff --git a/src/fsharp/TastPickle.fs b/src/fsharp/TastPickle.fs index 56717edafe1..5098b16ecd0 100644 --- a/src/fsharp/TastPickle.fs +++ b/src/fsharp/TastPickle.fs @@ -284,10 +284,13 @@ let u_int32 st = assert(b0 = 0xFF) prim_u_int32 st -let u_bytes st = +let u_byte_memory st = let n = (u_int32 st) st.is.ReadBytes n +let u_bytes st = + (u_byte_memory st).ToArray() + let u_prim_string st = let len = (u_int32 st) st.is.ReadUtf8String len @@ -858,7 +861,7 @@ let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u ( (u_array u_encoded_pubpath) (u_array u_encoded_nleref) (u_array u_encoded_simpletyp) - u_bytes + u_byte_memory st2 let ccuTab = new_itbl "iccus" (Array.map (CcuThunk.CreateDelayed) ccuNameTab) let stringTab = new_itbl "istrings" (Array.map decode_string stringTab) @@ -923,8 +926,8 @@ let p_ILScopeRef x st = let u_ILPublicKey st = let tag = u_byte st match tag with - | 0 -> (u_bytes st).ToArray() |> PublicKey - | 1 -> (u_bytes st).ToArray() |> PublicKeyToken + | 0 -> u_bytes st |> PublicKey + | 1 -> u_bytes st |> PublicKeyToken | _ -> ufailwith st "u_ILPublicKey" let u_ILVersion st = @@ -933,14 +936,14 @@ let u_ILVersion st = let u_ILModuleRef st = let (a, b, c) = u_tup3 u_string u_bool (u_option u_bytes) st - ILModuleRef.Create(a, b, c |> Option.map (fun x -> x.ToArray())) + ILModuleRef.Create(a, b, c) let u_ILAssemblyRef st = let tag = u_byte st match tag with | 0 -> let a, b, c, d, e, f = u_tup6 u_string (u_option u_bytes) (u_option u_ILPublicKey) u_bool (u_option u_ILVersion) (u_option u_string) st - ILAssemblyRef.Create(a, b |> Option.map (fun x -> x.ToArray()), c, d, e, f) + ILAssemblyRef.Create(a, b, c, d, e, f) | _ -> ufailwith st "u_ILAssemblyRef" // IL scope references are rescoped as they are unpickled. This means @@ -2537,7 +2540,7 @@ and u_op st = | 20 -> TOp.While (NoSequencePointAtWhileLoop, NoSpecialWhileLoopMarker) | 21 -> let dir = match u_int st with 0 -> FSharpForLoopUp | 1 -> CSharpForLoopUp | 2 -> FSharpForLoopDown | _ -> failwith "unknown for loop" TOp.For (NoSequencePointAtForLoop, dir) - | 22 -> TOp.Bytes ((u_bytes st).ToArray()) + | 22 -> TOp.Bytes (u_bytes st) | 23 -> TOp.TryCatch (NoSequencePointAtTry, NoSequencePointAtWith) | 24 -> TOp.TryFinally (NoSequencePointAtTry, NoSequencePointAtFinally) | 25 -> let a = u_rfref st From 68109eb8477b51a1fcd790df632d8f9199270704 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sat, 14 Dec 2019 23:51:35 -0800 Subject: [PATCH 03/11] minor cleanup --- src/absil/bytes.fsi | 2 ++ src/fsharp/fsc.fs | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index 08f87603b8e..6534b21cdfb 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -47,6 +47,8 @@ type internal ByteMemory = abstract ToArray: unit -> byte[] + /// Get a stream representation of the backing memory. + /// Disposing this will not free up any of the backing memory. abstract AsStream: unit -> Stream /// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents. diff --git a/src/fsharp/fsc.fs b/src/fsharp/fsc.fs index 31f0ce0b3bd..364e819dbd3 100644 --- a/src/fsharp/fsc.fs +++ b/src/fsharp/fsc.fs @@ -444,7 +444,8 @@ let EncodeInterfaceData(tcConfig: TcConfig, tcGlobals, exportRemapping, generate let sigDataFileName = (Filename.chopExtension outfile)+".sigdata" let bytes = resource.GetBytes() use fileStream = File.Create(sigDataFileName, bytes.Length) - bytes.AsStream().CopyTo(fileStream) + use stream = bytes.AsStream() + stream.CopyTo(fileStream) let resources = [ resource ] let sigAttr = mkSignatureDataVersionAttr tcGlobals (IL.parseILVersion Internal.Utilities.FSharpEnvironment.FSharpBinaryMetadataFormatRevision) From 7287d763b55030e084af8f101e0bd0855184b345 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 00:07:46 -0800 Subject: [PATCH 04/11] rename EmitBytes to EmitByteMemory --- src/absil/bytes.fs | 2 +- src/absil/bytes.fsi | 1 + src/absil/ilwrite.fs | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index 3df889d6190..e1ee9140d49 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -367,7 +367,7 @@ type internal ByteBuffer = Bytes.blit i 0 buf.bbArray buf.bbCurrent n buf.bbCurrent <- newSize - member buf.EmitBytes (i:ByteMemory) = + member buf.EmitMemory (i:ByteMemory) = let n = i.Length let newSize = buf.bbCurrent + n buf.Ensure newSize diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index 6534b21cdfb..d91e5a071d4 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -93,6 +93,7 @@ type internal ByteBuffer = member EmitIntsAsBytes : int[] -> unit member EmitByte : byte -> unit member EmitBytes : byte[] -> unit + member EmitByteMemory : ByteMemory -> unit member EmitInt32 : int32 -> unit member EmitInt64 : int64 -> unit member FixupInt32 : pos: int -> value: int32 -> unit diff --git a/src/absil/ilwrite.fs b/src/absil/ilwrite.fs index 056b495f37d..af64aa1cffa 100644 --- a/src/absil/ilwrite.fs +++ b/src/absil/ilwrite.fs @@ -2689,7 +2689,7 @@ and GenEventPass3 cenv env (md: ILEventDef) = let rec GetResourceAsManifestResourceRow cenv r = let data, impl = - let embedManagedResources (bytes: byte[]) = + let embedManagedResources (bytes: ByteMemory) = // Embedded managed resources must be word-aligned. However resource format is // not specified in ECMA. Some mscorlib resources appear to be non-aligned - it seems it doesn't matter.. let offset = cenv.resources.Position @@ -2698,11 +2698,11 @@ let rec GetResourceAsManifestResourceRow cenv r = let resourceSize = bytes.Length cenv.resources.EmitPadding pad cenv.resources.EmitInt32 resourceSize - cenv.resources.EmitBytes bytes + cenv.resources.EmitByteMemory bytes Data (alignedOffset, true), (i_File, 0) match r.Location with - | ILResourceLocation.Local bytes -> embedManagedResources (bytes.ToArray()) // TODO: + | ILResourceLocation.Local bytes -> embedManagedResources bytes | ILResourceLocation.File (mref, offset) -> ULong offset, (i_File, GetModuleRefAsFileIdx cenv mref) | ILResourceLocation.Assembly aref -> ULong 0x0, (i_AssemblyRef, GetAssemblyRefAsIdx cenv aref) From 355a71d0b71af2dcd1f1e529d7b32c41afccc45c Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 00:21:05 -0800 Subject: [PATCH 05/11] Fixed ByteArrayMemory --- src/absil/bytes.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index e1ee9140d49..1de6933e632 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -86,13 +86,13 @@ type ByteArrayMemory(bytes: byte[], offset, length) = Array.sub bytes (offset + pos) count override _.GetInt32 pos = - BitConverter.ToInt32(bytes, pos) + BitConverter.ToInt32(bytes, offset + pos) override _.GetUInt16 pos = - BitConverter.ToUInt16(bytes, pos) + BitConverter.ToUInt16(bytes, offset + pos) override _.GetUtf8String(pos, count) = - System.Text.Encoding.UTF8.GetString(bytes, pos, count) + System.Text.Encoding.UTF8.GetString(bytes, offset + pos, count) override _.Slice(pos, count) = ByteArrayMemory(bytes, offset + pos, count) :> ByteMemory From e88bb6c853dc90c31d086adf4d01ba2e913ce209 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 00:27:47 -0800 Subject: [PATCH 06/11] fixing build --- src/absil/bytes.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index 1de6933e632..9c8fc867c64 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -367,7 +367,7 @@ type internal ByteBuffer = Bytes.blit i 0 buf.bbArray buf.bbCurrent n buf.bbCurrent <- newSize - member buf.EmitMemory (i:ByteMemory) = + member buf.EmitByteMemory (i:ByteMemory) = let n = i.Length let newSize = buf.bbCurrent + n buf.Ensure newSize From 25324011c17537341bbf75cfad4150134a191fd1 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 00:45:17 -0800 Subject: [PATCH 07/11] Fixing build again --- src/absil/bytes.fs | 39 ++++++++++++++++++++------------------ src/absil/bytes.fsi | 46 ++++++++++++++++++++++++--------------------- src/absil/ilread.fs | 14 +++++++------- 3 files changed, 53 insertions(+), 46 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index 9c8fc867c64..7ac66073c45 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -42,13 +42,13 @@ type ByteMemory () = abstract Length: int - abstract GetBytes: pos: int * count: int -> byte[] + abstract ReadBytes: pos: int * count: int -> byte[] - abstract GetInt32: pos: int -> int + abstract ReadInt32: pos: int -> int - abstract GetUInt16: pos: int -> uint16 + abstract ReadUInt16: pos: int -> uint16 - abstract GetUtf8String: pos: int * count: int -> string + abstract ReadUtf8String: pos: int * count: int -> string abstract Slice: pos: int * count: int -> ByteMemory @@ -82,16 +82,16 @@ type ByteArrayMemory(bytes: byte[], offset, length) = override _.Length = length - override _.GetBytes(pos, count) = + override _.ReadBytes(pos, count) = Array.sub bytes (offset + pos) count - override _.GetInt32 pos = + override _.ReadInt32 pos = BitConverter.ToInt32(bytes, offset + pos) - override _.GetUInt16 pos = + override _.ReadUInt16 pos = BitConverter.ToUInt16(bytes, offset + pos) - override _.GetUtf8String(pos, count) = + override _.ReadUtf8String(pos, count) = System.Text.Encoding.UTF8.GetString(bytes, offset + pos, count) override _.Slice(pos, count) = @@ -132,24 +132,24 @@ type RawByteMemory(addr: nativeptr, length: int, hold: obj) = override _.Length = length - override _.GetUtf8String(pos, count) = + override _.ReadUtf8String(pos, count) = check pos check (pos + count - 1) System.Text.Encoding.UTF8.GetString(NativePtr.add addr pos, count) - override _.GetBytes(pos, count) = + override _.ReadBytes(pos, count) = check pos check (pos + count - 1) let res = Bytes.zeroCreate count Marshal.Copy(NativePtr.toNativeInt addr + nativeint pos, res, 0, count) res - override _.GetInt32 pos = + override _.ReadInt32 pos = check pos check (pos + 3) Marshal.ReadInt32(NativePtr.toNativeInt addr + nativeint pos) - override _.GetUInt16 pos = + override _.ReadUInt16 pos = check pos check (pos + 1) uint16(Marshal.ReadInt16(NativePtr.toNativeInt addr + nativeint pos)) @@ -183,26 +183,29 @@ type ReadOnlyByteMemory(bytes: ByteMemory) = member _.Length with [] get () = bytes.Length [] - member _.GetBytes(pos, count) = bytes.GetBytes(pos, count) + member _.ReadBytes(pos, count) = bytes.ReadBytes(pos, count) [] - member _.GetInt32 pos = bytes.GetInt32 pos + member _.ReadInt32 pos = bytes.ReadInt32 pos [] - member _.GetUInt16 pos = bytes.GetUInt16 pos + member _.ReadUInt16 pos = bytes.ReadUInt16 pos [] - member _.GetUtf8String(pos, count) = bytes.GetUtf8String(pos, count) + member _.ReadUtf8String(pos, count) = bytes.ReadUtf8String(pos, count) [] member _.Slice(pos, count) = bytes.Slice(pos, count) |> ReadOnlyByteMemory + [] + member _.CopyTo stream = bytes.CopyTo stream + [] member _.ToArray() = bytes.ToArray() type ByteMemory with - static member CreateMemoryMappedFile(bytes: ByteMemory) = + static member CreateMemoryMappedFile(bytes: ReadOnlyByteMemory) = let length = int64 bytes.Length let mmf = let mmf = @@ -297,7 +300,7 @@ type internal ByteStream = b.pos <- b.pos + 1 res member b.ReadUtf8String n = - let res = b.bytes.GetUtf8String(b.pos,n) + let res = b.bytes.ReadUtf8String(b.pos,n) b.pos <- b.pos + n; res static member FromBytes (b: ReadOnlyByteMemory,n,len) = diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index d91e5a071d4..f669953a937 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -31,13 +31,13 @@ type internal ByteMemory = abstract Length: int - abstract GetBytes: pos: int * count: int -> byte[] + abstract ReadBytes: pos: int * count: int -> byte[] - abstract GetInt32: pos: int -> int + abstract ReadInt32: pos: int -> int - abstract GetUInt16: pos: int -> uint16 + abstract ReadUInt16: pos: int -> uint16 - abstract GetUtf8String: pos: int * count: int -> string + abstract ReadUtf8String: pos: int * count: int -> string abstract Slice: pos: int * count: int -> ByteMemory @@ -51,19 +51,6 @@ type internal ByteMemory = /// Disposing this will not free up any of the backing memory. abstract AsStream: unit -> Stream - /// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents. - static member CreateMemoryMappedFile: ByteMemory -> ByteMemory - - /// Creates a ByteMemory object that has a backing memory mapped file from a file on-disk. - static member FromFile: path: string * FileAccess * ?canShadowCopy: bool -> ByteMemory - - /// Creates a ByteMemory object that is backed by a raw pointer. - /// Use with care. - static member FromUnsafePointer: addr: nativeint * length: int * hold: obj -> ByteMemory - - /// Creates a ByteMemory object that is backed by a byte array with the specified offset and length. - static member FromArray: bytes: byte[] * offset: int * length: int -> ByteMemory - [] type internal ReadOnlyByteMemory = @@ -73,18 +60,35 @@ type internal ReadOnlyByteMemory = member Length: int - member GetBytes: pos: int * count: int -> byte[] + member ReadBytes: pos: int * count: int -> byte[] - member GetInt32: pos: int -> int + member ReadInt32: pos: int -> int - member GetUInt16: pos: int -> uint16 + member ReadUInt16: pos: int -> uint16 - member GetUtf8String: pos: int * count: int -> string + member ReadUtf8String: pos: int * count: int -> string member Slice: pos: int * count: int -> ReadOnlyByteMemory + member CopyTo: Stream -> unit + member ToArray: unit -> byte[] +type ByteMemory with + + /// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents. + static member CreateMemoryMappedFile: ReadOnlyByteMemory -> ByteMemory + + /// Creates a ByteMemory object that has a backing memory mapped file from a file on-disk. + static member FromFile: path: string * FileAccess * ?canShadowCopy: bool -> ByteMemory + + /// Creates a ByteMemory object that is backed by a raw pointer. + /// Use with care. + static member FromUnsafePointer: addr: nativeint * length: int * hold: obj -> ByteMemory + + /// Creates a ByteMemory object that is backed by a byte array with the specified offset and length. + static member FromArray: bytes: byte[] * offset: int * length: int -> ByteMemory + /// Imperative buffers and streams of byte[] [] type internal ByteBuffer = diff --git a/src/absil/ilread.fs b/src/absil/ilread.fs index b59c2b6dbc0..3cd350a0bdd 100644 --- a/src/absil/ilread.fs +++ b/src/absil/ilread.fs @@ -107,7 +107,7 @@ let stats = let GetStatistics() = stats -type private BinaryView = ByteMemory +type private BinaryView = ReadOnlyByteMemory /// An abstraction over how we access the contents of .NET binaries. type BinaryFile = @@ -118,7 +118,7 @@ type BinaryFile = /// the lifetime of this object. type RawMemoryFile(fileName: string, obj: obj, addr: nativeint, length: int) = do stats.rawMemoryFileCount <- stats.rawMemoryFileCount + 1 - let view = ByteMemory.FromUnsafePointer(addr, length, obj) + let view = ByteMemory.FromUnsafePointer(addr, length, obj) |> ReadOnlyByteMemory member __.HoldObj() = obj // make sure we capture 'obj' member __.FileName = fileName interface BinaryFile with @@ -127,7 +127,7 @@ type RawMemoryFile(fileName: string, obj: obj, addr: nativeint, length: int) = /// A BinaryFile backed by an array of bytes held strongly as managed memory [] type ByteFile(fileName: string, bytes: byte[]) = - let view = ByteMemory.FromArray(bytes, 0, bytes.Length) + let view = ByteMemory.FromArray(bytes, 0, bytes.Length) |> ReadOnlyByteMemory do stats.byteFileCount <- stats.byteFileCount + 1 member __.FileName = fileName interface BinaryFile with @@ -170,13 +170,13 @@ type WeakByteFile(fileName: string, chunk: (int * int) option) = tg - ByteMemory.FromArray(strongBytes, 0, strongBytes.Length) + ByteMemory.FromArray(strongBytes, 0, strongBytes.Length) |> ReadOnlyByteMemory let seekReadByte (mdv: BinaryView) addr = mdv.[addr] -let seekReadBytes (mdv: BinaryView) addr len = mdv.GetBytes(addr, len) -let seekReadInt32 (mdv: BinaryView) addr = mdv.GetInt32 addr -let seekReadUInt16 (mdv: BinaryView) addr = mdv.GetUInt16 addr +let seekReadBytes (mdv: BinaryView) addr len = mdv.ReadBytes(addr, len) +let seekReadInt32 (mdv: BinaryView) addr = mdv.ReadInt32 addr +let seekReadUInt16 (mdv: BinaryView) addr = mdv.ReadUInt16 addr let seekReadByteAsInt32 mdv addr = int32 (seekReadByte mdv addr) From daf253b8fa4d80ea423784a4f87d1755a7df89e8 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 02:23:36 -0800 Subject: [PATCH 08/11] Better ReadInt32/ReadUInt16 --- src/absil/bytes.fs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index 7ac66073c45..f1d8ad255e6 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -86,10 +86,17 @@ type ByteArrayMemory(bytes: byte[], offset, length) = Array.sub bytes (offset + pos) count override _.ReadInt32 pos = - BitConverter.ToInt32(bytes, offset + pos) + let finalOffset = offset + pos + (uint32 bytes.[finalOffset]) ||| + ((uint32 bytes.[finalOffset + 1]) <<< 8) ||| + ((uint32 bytes.[finalOffset + 2]) <<< 16) ||| + ((uint32 bytes.[finalOffset + 3]) <<< 24) + |> int override _.ReadUInt16 pos = - BitConverter.ToUInt16(bytes, offset + pos) + let finalOffset = offset + pos + (uint16 bytes.[finalOffset]) ||| + ((uint16 bytes.[finalOffset + 1]) <<< 8) override _.ReadUtf8String(pos, count) = System.Text.Encoding.UTF8.GetString(bytes, offset + pos, count) From 123053745e5284e122061d4d46152b5fd3e5a07e Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 05:25:53 -0800 Subject: [PATCH 09/11] ILResourceLocation.Local should use a ReadOnlyByteMemory --- src/absil/bytes.fs | 10 +++++++++- src/absil/bytes.fsi | 9 ++++++++- src/absil/il.fs | 2 +- src/absil/il.fsi | 4 ++-- src/absil/ilread.fs | 10 +++++----- src/absil/ilwrite.fs | 2 +- src/fsharp/CompileOps.fs | 12 ++++++------ src/fsharp/fsc.fs | 7 +++---- src/fsharp/service/IncrementalBuild.fs | 2 +- 9 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index f1d8ad255e6..2c59cec22e7 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -207,11 +207,16 @@ type ReadOnlyByteMemory(bytes: ByteMemory) = [] member _.CopyTo stream = bytes.CopyTo stream + [] + member _.Copy(srcOffset, dest, destOffset, count) = bytes.Copy(srcOffset, dest, destOffset, count) + [] member _.ToArray() = bytes.ToArray() type ByteMemory with + member x.AsReadOnly() = ReadOnlyByteMemory x + static member CreateMemoryMappedFile(bytes: ReadOnlyByteMemory) = let length = int64 bytes.Length let mmf = @@ -297,6 +302,9 @@ type ByteMemory with static member FromArray(bytes, offset, length) = ByteArrayMemory(bytes, offset, length) :> ByteMemory + static member FromArray bytes = + ByteArrayMemory.FromArray(bytes, 0, bytes.Length) + type internal ByteStream = { bytes: ReadOnlyByteMemory mutable pos: int @@ -377,7 +385,7 @@ type internal ByteBuffer = Bytes.blit i 0 buf.bbArray buf.bbCurrent n buf.bbCurrent <- newSize - member buf.EmitByteMemory (i:ByteMemory) = + member buf.EmitByteMemory (i:ReadOnlyByteMemory) = let n = i.Length let newSize = buf.bbCurrent + n buf.Ensure newSize diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index f669953a937..c2ce33a2837 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -72,10 +72,14 @@ type internal ReadOnlyByteMemory = member CopyTo: Stream -> unit + member Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit + member ToArray: unit -> byte[] type ByteMemory with + member AsReadOnly: unit -> ReadOnlyByteMemory + /// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents. static member CreateMemoryMappedFile: ReadOnlyByteMemory -> ByteMemory @@ -89,6 +93,9 @@ type ByteMemory with /// Creates a ByteMemory object that is backed by a byte array with the specified offset and length. static member FromArray: bytes: byte[] * offset: int * length: int -> ByteMemory + /// Creates a ByteMemory object that is backed by a byte array. + static member FromArray: bytes: byte[] -> ByteMemory + /// Imperative buffers and streams of byte[] [] type internal ByteBuffer = @@ -97,7 +104,7 @@ type internal ByteBuffer = member EmitIntsAsBytes : int[] -> unit member EmitByte : byte -> unit member EmitBytes : byte[] -> unit - member EmitByteMemory : ByteMemory -> unit + member EmitByteMemory : ReadOnlyByteMemory -> unit member EmitInt32 : int32 -> unit member EmitInt64 : int64 -> unit member FixupInt32 : pos: int -> value: int32 -> unit diff --git a/src/absil/il.fs b/src/absil/il.fs index 83cceb602fc..4736a29b39e 100644 --- a/src/absil/il.fs +++ b/src/absil/il.fs @@ -2208,7 +2208,7 @@ type ILResourceAccess = [] type ILResourceLocation = - | Local of ByteMemory + | Local of ReadOnlyByteMemory | File of ILModuleRef * int32 | Assembly of ILAssemblyRef diff --git a/src/absil/il.fsi b/src/absil/il.fsi index 5a3562899f2..ccafd6334f4 100644 --- a/src/absil/il.fsi +++ b/src/absil/il.fsi @@ -1408,7 +1408,7 @@ type ILResourceAccess = type ILResourceLocation = internal /// Represents a manifest resource that can be read or written to a PE file - | Local of ByteMemory + | Local of ReadOnlyByteMemory /// Represents a manifest resource in an associated file | File of ILModuleRef * int32 @@ -1428,7 +1428,7 @@ type ILResource = MetadataIndex: int32 } /// Read the bytes from a resource local to an assembly. Will fail for non-local resources. - member internal GetBytes : unit -> ByteMemory + member internal GetBytes : unit -> ReadOnlyByteMemory member CustomAttrs: ILAttributes diff --git a/src/absil/ilread.fs b/src/absil/ilread.fs index 3cd350a0bdd..fed858c4d50 100644 --- a/src/absil/ilread.fs +++ b/src/absil/ilread.fs @@ -118,7 +118,7 @@ type BinaryFile = /// the lifetime of this object. type RawMemoryFile(fileName: string, obj: obj, addr: nativeint, length: int) = do stats.rawMemoryFileCount <- stats.rawMemoryFileCount + 1 - let view = ByteMemory.FromUnsafePointer(addr, length, obj) |> ReadOnlyByteMemory + let view = ByteMemory.FromUnsafePointer(addr, length, obj).AsReadOnly() member __.HoldObj() = obj // make sure we capture 'obj' member __.FileName = fileName interface BinaryFile with @@ -127,7 +127,7 @@ type RawMemoryFile(fileName: string, obj: obj, addr: nativeint, length: int) = /// A BinaryFile backed by an array of bytes held strongly as managed memory [] type ByteFile(fileName: string, bytes: byte[]) = - let view = ByteMemory.FromArray(bytes, 0, bytes.Length) |> ReadOnlyByteMemory + let view = ByteMemory.FromArray(bytes).AsReadOnly() do stats.byteFileCount <- stats.byteFileCount + 1 member __.FileName = fileName interface BinaryFile with @@ -170,7 +170,7 @@ type WeakByteFile(fileName: string, chunk: (int * int) option) = tg - ByteMemory.FromArray(strongBytes, 0, strongBytes.Length) |> ReadOnlyByteMemory + ByteMemory.FromArray(strongBytes).AsReadOnly() let seekReadByte (mdv: BinaryView) addr = mdv.[addr] @@ -3032,8 +3032,8 @@ and seekReadManifestResources (ctxt: ILMetadataReader) canReduceMemory (mdv: Bin if canReduceMemory then ByteMemory.CreateMemoryMappedFile bytes else - ByteMemory.FromArray(bytes.ToArray(), 0, bytes.Length) - ILResourceLocation.Local bytes + ByteMemory.FromArray(bytes.ToArray()) + ILResourceLocation.Local(bytes.AsReadOnly()) | ILScopeRef.Module mref -> ILResourceLocation.File (mref, offset) | ILScopeRef.Assembly aref -> ILResourceLocation.Assembly aref diff --git a/src/absil/ilwrite.fs b/src/absil/ilwrite.fs index af64aa1cffa..802ad3d3a04 100644 --- a/src/absil/ilwrite.fs +++ b/src/absil/ilwrite.fs @@ -2689,7 +2689,7 @@ and GenEventPass3 cenv env (md: ILEventDef) = let rec GetResourceAsManifestResourceRow cenv r = let data, impl = - let embedManagedResources (bytes: ByteMemory) = + let embedManagedResources (bytes: ReadOnlyByteMemory) = // Embedded managed resources must be word-aligned. However resource format is // not specified in ECMA. Some mscorlib resources appear to be non-aligned - it seems it doesn't matter.. let offset = cenv.resources.Position diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index ab25947a57e..f82b6551ac0 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -3589,7 +3589,7 @@ let IsReflectedDefinitionsResource (r: ILResource) = let MakeILResource rName bytes = { Name = rName - Location = ILResourceLocation.Local(ByteMemory.FromArray(bytes, 0, bytes.Length)) + Location = ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly()) Access = ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -3598,7 +3598,7 @@ let PickleToResource inMem file (g: TcGlobals) scope rName p x = let file = PathMap.apply g.pathMap file { Name = rName - Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.Local(ByteMemory.FromArray(bytes, 0, bytes.Length))) + Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly())) Access = ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -3649,21 +3649,21 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR [ for iresource in resources do if IsSignatureDataResource iresource then let ccuName = GetSignatureDataResourceName iresource - yield (ccuName, fun () -> iresource.GetBytes() |> ReadOnlyByteMemory) ] + yield (ccuName, fun () -> iresource.GetBytes()) ] let sigDataReaders = if sigDataReaders.IsEmpty && List.contains ilShortAssemName externalSigAndOptData then let sigFileName = Path.ChangeExtension(filename, "sigdata") if not (FileSystem.SafeExists sigFileName) then error(Error(FSComp.SR.buildExpectedSigdataFile (FileSystem.GetFullPathShim sigFileName), m)) - [ (ilShortAssemName, fun () -> ByteMemory.FromFile(sigFileName, FileAccess.Read, canShadowCopy=true) |> ReadOnlyByteMemory)] + [ (ilShortAssemName, fun () -> ByteMemory.FromFile(sigFileName, FileAccess.Read, canShadowCopy=true).AsReadOnly())] else sigDataReaders sigDataReaders member __.GetRawFSharpOptimizationData(m, ilShortAssemName, filename) = let optDataReaders = ilModule.Resources.AsList - |> List.choose (fun r -> if IsOptimizationDataResource r then Some(GetOptimizationDataResourceName r, (fun () -> r.GetBytes() |> ReadOnlyByteMemory)) else None) + |> List.choose (fun r -> if IsOptimizationDataResource r then Some(GetOptimizationDataResourceName r, (fun () -> r.GetBytes())) else None) // Look for optimization data in a file let optDataReaders = @@ -3671,7 +3671,7 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR let optDataFile = Path.ChangeExtension(filename, "optdata") if not (FileSystem.SafeExists optDataFile) then error(Error(FSComp.SR.buildExpectedFileAlongSideFSharpCore(optDataFile, FileSystem.GetFullPathShim optDataFile), m)) - [ (ilShortAssemName, (fun () -> ByteMemory.FromFile(optDataFile, FileAccess.Read, canShadowCopy=true) |> ReadOnlyByteMemory))] + [ (ilShortAssemName, (fun () -> ByteMemory.FromFile(optDataFile, FileAccess.Read, canShadowCopy=true).AsReadOnly()))] else optDataReaders optDataReaders diff --git a/src/fsharp/fsc.fs b/src/fsharp/fsc.fs index 364e819dbd3..61cc6e8547d 100644 --- a/src/fsharp/fsc.fs +++ b/src/fsharp/fsc.fs @@ -444,8 +444,7 @@ let EncodeInterfaceData(tcConfig: TcConfig, tcGlobals, exportRemapping, generate let sigDataFileName = (Filename.chopExtension outfile)+".sigdata" let bytes = resource.GetBytes() use fileStream = File.Create(sigDataFileName, bytes.Length) - use stream = bytes.AsStream() - stream.CopyTo(fileStream) + bytes.CopyTo fileStream let resources = [ resource ] let sigAttr = mkSignatureDataVersionAttr tcGlobals (IL.parseILVersion Internal.Utilities.FSharpEnvironment.FSharpBinaryMetadataFormatRevision) @@ -895,7 +894,7 @@ module MainModuleBuilder = [ ] let reflectedDefinitionResource = { Name=reflectedDefinitionResourceName - Location = ILResourceLocation.Local(ByteMemory.FromArray(reflectedDefinitionBytes, 0, reflectedDefinitionBytes.Length)) + Location = ILResourceLocation.Local(ByteMemory.FromArray(reflectedDefinitionBytes).AsReadOnly()) Access= ILResourceAccess.Public CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } @@ -939,7 +938,7 @@ module MainModuleBuilder = let bytes = FileSystem.ReadAllBytesShim file name, bytes, pub yield { Name=name - Location=ILResourceLocation.Local(ByteMemory.FromArray(bytes, 0, bytes.Length)) + Location=ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly()) Access=pub CustomAttrsStored=storeILCustomAttrs emptyILCustomAttrs MetadataIndex = NoMetadataIdx } diff --git a/src/fsharp/service/IncrementalBuild.fs b/src/fsharp/service/IncrementalBuild.fs index fba68ac1bdd..4cb4c23ea4a 100755 --- a/src/fsharp/service/IncrementalBuild.fs +++ b/src/fsharp/service/IncrementalBuild.fs @@ -1184,7 +1184,7 @@ type RawFSharpAssemblyDataBackedByLanguageService (tcConfig, tcGlobals, tcState: let _sigDataAttributes, sigDataResources = Driver.EncodeInterfaceData(tcConfig, tcGlobals, exportRemapping, generatedCcu, outfile, true) [ for r in sigDataResources do let ccuName = GetSignatureDataResourceName r - yield (ccuName, (fun () -> r.GetBytes() |> ReadOnlyByteMemory)) ] + yield (ccuName, (fun () -> r.GetBytes())) ] let autoOpenAttrs = topAttrs.assemblyAttrs |> List.choose (List.singleton >> TryFindFSharpStringAttribute tcGlobals tcGlobals.attrib_AutoOpenAttribute) From 84550f689efc2f3495abf2cb19ddc2de6c115e27 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 05:39:37 -0800 Subject: [PATCH 10/11] Able to have a ReadOnlyStream --- src/absil/bytes.fs | 10 ++++++++++ src/absil/bytes.fsi | 7 +++++++ src/absil/ilreflect.fs | 3 ++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index 2c59cec22e7..c4fbe500253 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -60,6 +60,8 @@ type ByteMemory () = abstract AsStream: unit -> Stream + abstract AsReadOnlyStream: unit -> Stream + [] type ByteArrayMemory(bytes: byte[], offset, length) = inherit ByteMemory() @@ -116,6 +118,9 @@ type ByteArrayMemory(bytes: byte[], offset, length) = override _.AsStream() = new MemoryStream(bytes, offset, length) :> Stream + override _.AsReadOnlyStream() = + new MemoryStream(bytes, offset, length, false) :> Stream + [] type RawByteMemory(addr: nativeptr, length: int, hold: obj) = inherit ByteMemory () @@ -182,6 +187,9 @@ type RawByteMemory(addr: nativeptr, length: int, hold: obj) = override _.AsStream() = new UnmanagedMemoryStream(addr, int64 length) :> Stream + override _.AsReadOnlyStream() = + new UnmanagedMemoryStream(addr, int64 length, int64 length, FileAccess.Read) :> Stream + [] type ReadOnlyByteMemory(bytes: ByteMemory) = @@ -213,6 +221,8 @@ type ReadOnlyByteMemory(bytes: ByteMemory) = [] member _.ToArray() = bytes.ToArray() + member _.AsStream() = bytes.AsReadOnlyStream() + type ByteMemory with member x.AsReadOnly() = ReadOnlyByteMemory x diff --git a/src/absil/bytes.fsi b/src/absil/bytes.fsi index c2ce33a2837..11e7c9f58cc 100644 --- a/src/absil/bytes.fsi +++ b/src/absil/bytes.fsi @@ -51,6 +51,11 @@ type internal ByteMemory = /// Disposing this will not free up any of the backing memory. abstract AsStream: unit -> Stream + /// Get a stream representation of the backing memory. + /// Disposing this will not free up any of the backing memory. + /// Stream cannot be written to. + abstract AsReadOnlyStream: unit -> Stream + [] type internal ReadOnlyByteMemory = @@ -76,6 +81,8 @@ type internal ReadOnlyByteMemory = member ToArray: unit -> byte[] + member AsStream: unit -> Stream + type ByteMemory with member AsReadOnly: unit -> ReadOnlyByteMemory diff --git a/src/absil/ilreflect.fs b/src/absil/ilreflect.fs index 329f95095ec..c6a79dd5b8a 100644 --- a/src/absil/ilreflect.fs +++ b/src/absil/ilreflect.fs @@ -2071,7 +2071,8 @@ let buildModuleFragment cenv emEnv (asmB: AssemblyBuilder) (modB: ModuleBuilder) let attribs = (match r.Access with ILResourceAccess.Public -> ResourceAttributes.Public | ILResourceAccess.Private -> ResourceAttributes.Private) match r.Location with | ILResourceLocation.Local bytes -> - modB.DefineManifestResourceAndLog (r.Name, bytes.AsStream(), attribs) + use stream = bytes.AsStream() + modB.DefineManifestResourceAndLog (r.Name, stream, attribs) | ILResourceLocation.File (mr, _) -> asmB.AddResourceFileAndLog (r.Name, mr.Name, attribs) | ILResourceLocation.Assembly _ -> From de3741baeaa43e75af7793850c176e9c7f55a8fd Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 16 Dec 2019 12:16:50 -0800 Subject: [PATCH 11/11] Using OutOfRangeException according to feedback --- src/absil/bytes.fs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/absil/bytes.fs b/src/absil/bytes.fs index c4fbe500253..56f3a3c1aca 100644 --- a/src/absil/bytes.fs +++ b/src/absil/bytes.fs @@ -67,20 +67,15 @@ type ByteArrayMemory(bytes: byte[], offset, length) = inherit ByteMemory() do - if offset < 0 then - failwith "offset is less than zero" + if length <= 0 || length > bytes.Length then + raise (ArgumentOutOfRangeException("length")) - if length <= 0 then - failwith "length is less than or equal to zero" - - if (offset + length) > bytes.Length then - failwith "span larger than byte array" + if offset < 0 || (offset + length) > bytes.Length then + raise (ArgumentOutOfRangeException("offset")) override _.Item - with get i = - bytes.[offset + i] - and set i v = - bytes.[offset + i] <- v + with get i = bytes.[offset + i] + and set i v = bytes.[offset + i] <- v override _.Length = length @@ -127,11 +122,11 @@ type RawByteMemory(addr: nativeptr, length: int, hold: obj) = let check i = if i < 0 || i >= length then - failwith "out of range" + raise (ArgumentOutOfRangeException("i")) do if length <= 0 then - failwith "length is less than or equal to zero" + raise (ArgumentOutOfRangeException("length")) override _.Item with get i = @@ -221,6 +216,7 @@ type ReadOnlyByteMemory(bytes: ByteMemory) = [] member _.ToArray() = bytes.ToArray() + [] member _.AsStream() = bytes.AsReadOnlyStream() type ByteMemory with