From dff4ef5d9e0f6130586326069bef25efc64c427f Mon Sep 17 00:00:00 2001 From: Don Syme Date: Thu, 22 Nov 2018 14:52:39 +0000 Subject: [PATCH 1/6] read only metadata section --- src/ProvidedTypes.fs | 113 ++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/src/ProvidedTypes.fs b/src/ProvidedTypes.fs index 659818bd..2dfa9a00 100644 --- a/src/ProvidedTypes.fs +++ b/src/ProvidedTypes.fs @@ -4590,7 +4590,7 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader typ_UIntPtr = ILType.Value (mkILTyspec "System" "UIntPtr") systemRuntimeScopeRef = systemRuntimeScopeRef } - type ILModuleReader(infile: string, is: ByteFile, ilg: ILGlobals, lowMem: bool) = + type PEReader(infile: string, is: ByteFile) = //----------------------------------------------------------------------- // Crack the binary headers, build a reader context and return the lazy @@ -4663,6 +4663,7 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let cliHeaderPhysLoc = anyV2P ("cli header",cliHeaderAddr) let metadataAddr = seekReadInt32 is (cliHeaderPhysLoc + 8) + let metadataSize = seekReadInt32 is (cliHeaderPhysLoc + 12) let cliFlags = seekReadInt32 is (cliHeaderPhysLoc + 16) let ilOnly = (cliFlags &&& 0x01) <> 0x00 let only32 = (cliFlags &&& 0x02) <> 0x00 @@ -4672,6 +4673,13 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let resourcesAddr = seekReadInt32 is (cliHeaderPhysLoc + 24) let metadataPhysLoc = anyV2P ("metadata",metadataAddr) + + member __.MetadataPhysLoc = metadataPhysLoc + member __.MetadataSize = metadataSize + + type ILModuleReader(infile: string, is: ByteFile, ilg: ILGlobals, lowMem: bool) = + + let metadataPhysLoc = 0 let magic = seekReadUInt16AsInt32 is metadataPhysLoc do if magic <> 0x5342 then failwith (infile + ": bad metadata magic number: " + string magic); let magic2 = seekReadUInt16AsInt32 is (metadataPhysLoc + 2) @@ -5207,15 +5215,6 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let implIdx = seekReadImplementationIdx &addr (flags,tok,nameIdx,namespaceIdx,implIdx) - /// Read Table ManifestResource - let seekReadManifestResourceRow idx = - let mutable addr = rowAddr ILTableNames.ManifestResource idx - let offset = seekReadInt32Adv &addr - let flags = seekReadInt32Adv &addr - let nameIdx = seekReadStringIdx &addr - let implIdx = seekReadImplementationIdx &addr - (offset,flags,nameIdx,implIdx) - /// Read Table Nested let seekReadNestedRow idx = let mutable addr = rowAddr ILTableNames.Nested idx @@ -5279,7 +5278,7 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader //let subsysversion = (subsysMajor, subsysMinor) let ilMetadataVersion = Encoding.UTF8.GetString (ilMetadataVersion, 0, ilMetadataVersion.Length) - let rec seekReadModule (subsys, subsysversion, useHighEntropyVA, ilOnly, only32, is32bitpreferred, only64, platform, isDll, alignVirt, alignPhys, imageBaseReal, ilMetadataVersion) idx = + let rec seekReadModule (ilMetadataVersion) idx = let (_generation, nameIdx, _mvidIdx, _encidIdx, _encbaseidIdx) = seekReadModuleRow idx let ilModuleName = readStringHeap nameIdx //let nativeResources = readNativeResources tgt @@ -5291,21 +5290,21 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader Name = ilModuleName; //NativeResources=nativeResources; TypeDefs = ILTypeDefs (lazy (seekReadTopTypeDefs ())); - SubSystemFlags = int32 subsys; - IsILOnly = ilOnly; - SubsystemVersion = subsysversion - UseHighEntropyVA = useHighEntropyVA - Platform = platform; - StackReserveSize = None; - Is32Bit = only32; - Is32BitPreferred = is32bitpreferred; - Is64Bit = only64; - IsDLL=isDll; - VirtualAlignment = alignVirt; - PhysicalAlignment = alignPhys; - ImageBase = imageBaseReal; - MetadataVersion = ilMetadataVersion; - Resources = seekReadManifestResources (); + SubsystemVersion = (4, 0) + UseHighEntropyVA = false + SubSystemFlags=3 + IsDLL=true + IsILOnly=true + Platform=None + StackReserveSize=None + Is32Bit=false + Is32BitPreferred=false + Is64Bit=false + PhysicalAlignment=512 + VirtualAlignment=0x2000 + ImageBase=0x034f0000 + MetadataVersion="" + Resources = ILResources (Lazy<_>.CreateFromValue [| |]) } and seekReadAssemblyManifest idx = @@ -5314,13 +5313,12 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let pubkey = readBlobHeapOption publicKeyIdx { Name= name; AuxModuleHashAlgorithm=hash - //SecurityDecls= seekReadSecurityDecls (TaggedIndex(hds_Assembly,idx)); - PublicKey= pubkey; - Version= USome (Version(int v1,int v2,int v3,int v4)); - Locale= readStringHeapOption localeIdx; - CustomAttrs = seekReadCustomAttrs (TaggedIndex(HasCustomAttributeTag.Assembly,idx)); - ExportedTypes= seekReadTopExportedTypes (); - EntrypointElsewhere=(if fst entryPointToken = ILTableNames.File then Some (seekReadFile (snd entryPointToken)) else None); + PublicKey= pubkey + Version= USome (Version(int v1,int v2,int v3,int v4)) + Locale= readStringHeapOption localeIdx + CustomAttrs = seekReadCustomAttrs (TaggedIndex(HasCustomAttributeTag.Assembly,idx)) + ExportedTypes= seekReadTopExportedTypes () + EntrypointElsewhere=None Retargetable = 0 <> (flags &&& 0x100); DisableJitOptimizations = 0 <> (flags &&& 0x4000); JitTracking = 0 <> (flags &&& 0x8000) @@ -6054,28 +6052,6 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader | x when x = uint16 et_CLASS || x = uint16 et_OBJECT -> null | _ -> null - and seekReadManifestResources () = - ILResources - (lazy - [| for i = 1 to getNumRows ILTableNames.ManifestResource do - let (offset,flags,nameIdx,implIdx) = seekReadManifestResourceRow i - let scoref = seekReadImplAsScopeRef implIdx - let datalab = - match scoref with - | ILScopeRef.Local -> - let start = anyV2P ("resource",offset + resourcesAddr) - let len = seekReadInt32 is start - ILResourceLocation.Local (fun () -> seekReadBytes is (start + 4) len) - | ILScopeRef.Module mref -> ILResourceLocation.File (mref,offset) - | ILScopeRef.Assembly aref -> ILResourceLocation.Assembly aref - - let r = - { Name= readStringHeap nameIdx; - Location = datalab; - Access = (if (flags &&& 0x01) <> 0x0 then ILResourceAccess.Public else ILResourceAccess.Private); - CustomAttrs = seekReadCustomAttrs (TaggedIndex(HasCustomAttributeTag.ManifestResource, i)) } - yield r |]) - and seekReadNestedExportedTypes parentIdx = ILNestedExportedTypesAndForwarders (lazy @@ -6118,7 +6094,7 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader yield entry |]) - let ilModule = seekReadModule (subsys, (subsysMajor, subsysMinor), useHighEntropyVA, ilOnly, only32, is32bitpreferred, only64, platform, isDll, alignVirt, alignPhys, imageBaseReal, ilMetadataVersion) 1 + let ilModule = seekReadModule (ilMetadataVersion) 1 let ilAssemblyRefs = [ for i in 1 .. getNumRows ILTableNames.AssemblyRef do yield seekReadAssemblyRef i ] member __.Bytes = is.Bytes @@ -6653,6 +6629,16 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader // resources when all instantiated at the same time. let readersWeakCache = ConcurrentDictionary<(string * string), WeakReference>() + type File with + static member ReadBinaryChunk (fileName: string, start, len) = + use stream = new FileStream(fileName,FileMode.Open,FileAccess.Read,FileShare.ReadWrite) + stream.Seek(int64 start, SeekOrigin.Begin) |> ignore + let buffer = Array.zeroCreate len + let mutable n = 0 + while n < len do + n <- n + stream.Read(buffer, n, len-n) + buffer + let ILModuleReaderAfterReadingAllBytes (file:string, ilGlobals: ILGlobals) = let bytes = File.ReadAllBytes file let key = (file, ilGlobals.systemRuntimeScopeRef.QualifiedName) @@ -6660,7 +6646,11 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader | true, CacheValue mr2 when bytes = mr2.Bytes -> mr2 // throw away the bytes we just read and recycle the existing ILModuleReader | _ -> - let mr = ILModuleReader(file, ByteFile(bytes), ilGlobals, true) + let is = ByteFile(bytes) + let pe = PEReader(file, is) + let mdchunk = File.ReadBinaryChunk (file, pe.MetadataPhysLoc, pe.MetadataSize) + let mdfile = ByteFile(mdchunk) + let mr = ILModuleReader(file, mdfile, ilGlobals, true) readersWeakCache.[key] <- CacheValue (mr) mr @@ -7952,10 +7942,11 @@ namespace ProviderImplementation.ProvidedTypes override __.ReflectionOnly = true override x.GetManifestResourceStream(resourceName:string) = - let r = getReader().ILModuleDef.Resources.Entries |> Seq.find (fun r -> r.Name = resourceName) - match r.Location with - | ILResourceLocation.Local f -> new MemoryStream(f()) :> Stream - | _ -> notRequired x "reading manifest resource %s from non-embedded location" resourceName + //let r = getReader().ILModuleDef.Resources.Entries |> Seq.find (fun r -> r.Name = resourceName) + //match r.Location with + //| ILResourceLocation.Local f -> new MemoryStream(f()) :> Stream + //| _ -> + notRequired x "reading manifest resource %s" resourceName member __.TxILTypeDef declTyOpt inp = txILTypeDef declTyOpt inp From ce731907187dc732b5e3c9a2d94c1d2bf9ac7e5b Mon Sep 17 00:00:00 2001 From: Don Syme Date: Thu, 22 Nov 2018 14:53:22 +0000 Subject: [PATCH 2/6] read only metadata section --- src/ProvidedTypes.fs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ProvidedTypes.fs b/src/ProvidedTypes.fs index 2dfa9a00..524b3888 100644 --- a/src/ProvidedTypes.fs +++ b/src/ProvidedTypes.fs @@ -6639,18 +6639,18 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader n <- n + stream.Read(buffer, n, len-n) buffer - let ILModuleReaderAfterReadingAllBytes (file:string, ilGlobals: ILGlobals) = - let bytes = File.ReadAllBytes file - let key = (file, ilGlobals.systemRuntimeScopeRef.QualifiedName) + let ILModuleReaderAfterReadingAllBytes (fileName:string, ilGlobals: ILGlobals) = + let bytes = File.ReadAllBytes fileName + let key = (fileName, ilGlobals.systemRuntimeScopeRef.QualifiedName) match readersWeakCache.TryGetValue (key) with | true, CacheValue mr2 when bytes = mr2.Bytes -> mr2 // throw away the bytes we just read and recycle the existing ILModuleReader | _ -> let is = ByteFile(bytes) - let pe = PEReader(file, is) - let mdchunk = File.ReadBinaryChunk (file, pe.MetadataPhysLoc, pe.MetadataSize) + let pe = PEReader(fileName, is) + let mdchunk = File.ReadBinaryChunk (fileName, pe.MetadataPhysLoc, pe.MetadataSize) let mdfile = ByteFile(mdchunk) - let mr = ILModuleReader(file, mdfile, ilGlobals, true) + let mr = ILModuleReader(fileName, mdfile, ilGlobals, true) readersWeakCache.[key] <- CacheValue (mr) mr From ac0355a1f68530804c1667b4c7bf40440f2b1634 Mon Sep 17 00:00:00 2001 From: Don Syme Date: Thu, 22 Nov 2018 23:19:31 +0000 Subject: [PATCH 3/6] read only metadata section --- src/ProvidedTypes.fs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/ProvidedTypes.fs b/src/ProvidedTypes.fs index 524b3888..126dcdfb 100644 --- a/src/ProvidedTypes.fs +++ b/src/ProvidedTypes.fs @@ -6097,7 +6097,7 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let ilModule = seekReadModule (ilMetadataVersion) 1 let ilAssemblyRefs = [ for i in 1 .. getNumRows ILTableNames.AssemblyRef do yield seekReadAssemblyRef i ] - member __.Bytes = is.Bytes + member __.MetadataBytes = is.Bytes member __.ILGlobals = ilg member __.ILModuleDef = ilModule member __.ILAssemblyRefs = ilAssemblyRefs @@ -6627,7 +6627,7 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader // Amortize readers weakly - this is enough that all the type providers in this DLL will at least share // resources when all instantiated at the same time. - let readersWeakCache = ConcurrentDictionary<(string * string), WeakReference>() + let readersWeakCache = ConcurrentDictionary<(string * DateTime * string), WeakReference>() type File with static member ReadBinaryChunk (fileName: string, start, len) = @@ -6640,12 +6640,13 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader buffer let ILModuleReaderAfterReadingAllBytes (fileName:string, ilGlobals: ILGlobals) = - let bytes = File.ReadAllBytes fileName - let key = (fileName, ilGlobals.systemRuntimeScopeRef.QualifiedName) + let timeStamp = File.GetLastWriteTimeUtc(fileName) + let key = (fileName, timeStamp, ilGlobals.systemRuntimeScopeRef.QualifiedName) match readersWeakCache.TryGetValue (key) with - | true, CacheValue mr2 when bytes = mr2.Bytes -> + | true, CacheValue mr2 -> mr2 // throw away the bytes we just read and recycle the existing ILModuleReader | _ -> + let bytes = File.ReadAllBytes fileName let is = ByteFile(bytes) let pe = PEReader(fileName, is) let mdchunk = File.ReadBinaryChunk (fileName, pe.MetadataPhysLoc, pe.MetadataSize) @@ -9194,7 +9195,11 @@ namespace ProviderImplementation.ProvidedTypes member this.ReadRelatedAssembly(bytes:byte[]) = let fileName = "file.dll" let ilg = ilGlobals.Force() - let reader = ILModuleReader(fileName, ByteFile(bytes), ilg, true) + let is = ByteFile(bytes) + let pe = PEReader(fileName, is) + let mdchunk = File.ReadBinaryChunk (fileName, pe.MetadataPhysLoc, pe.MetadataSize) + let mdfile = ByteFile(mdchunk) + let reader = ILModuleReader(fileName, mdfile, ilg, true) TargetAssembly(ilg, this.TryBindILAssemblyRefToTgt, Some reader, fileName) :> Assembly member __.AddSourceAssembly(asm: Assembly) = @@ -14518,7 +14523,7 @@ namespace ProviderImplementation.ProvidedTypes #else File.Delete assemblyFileName #endif - let bytes = reader.Bytes + let bytes = File.ReadAllBytes(assemblyFileName) // Use a real Reflection Load when running in F# Interactive if isHostedExecution then From ae931436736ef06ebfed7b221baa627b9e3f4ecd Mon Sep 17 00:00:00 2001 From: Don Syme Date: Thu, 22 Nov 2018 23:25:16 +0000 Subject: [PATCH 4/6] read only metadata section --- src/ProvidedTypes.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ProvidedTypes.fs b/src/ProvidedTypes.fs index 126dcdfb..cd3c3fde 100644 --- a/src/ProvidedTypes.fs +++ b/src/ProvidedTypes.fs @@ -6621,13 +6621,13 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader with err -> failwithf "FAILED decodeILCustomAttribData, data.Length = %d, data = %A, meth = %A, argtypes = %A, fixedArgs=%A, nnamed = %A, sigptr before named = %A, innerError = %A" bytes.Length bytes ca.Method.EnclosingType ca.Method.FormalArgTypes fixedArgs nnamed sigptr (err.ToString()) - type CacheValue = ILModuleReader + type CacheValue = ILModuleReader * DateTime let (|CacheValue|_|) (wr: WeakReference) = match wr.Target with null -> None | v -> Some (v :?> CacheValue) let CacheValue (reader: CacheValue) = System.WeakReference reader // Amortize readers weakly - this is enough that all the type providers in this DLL will at least share // resources when all instantiated at the same time. - let readersWeakCache = ConcurrentDictionary<(string * DateTime * string), WeakReference>() + let readersWeakCache = ConcurrentDictionary<(string * string), WeakReference>() type File with static member ReadBinaryChunk (fileName: string, start, len) = @@ -6641,9 +6641,9 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let ILModuleReaderAfterReadingAllBytes (fileName:string, ilGlobals: ILGlobals) = let timeStamp = File.GetLastWriteTimeUtc(fileName) - let key = (fileName, timeStamp, ilGlobals.systemRuntimeScopeRef.QualifiedName) + let key = (fileName, ilGlobals.systemRuntimeScopeRef.QualifiedName) match readersWeakCache.TryGetValue (key) with - | true, CacheValue mr2 -> + | true, CacheValue (mr2, timeStamp2) when timeStamp = timeStamp2 -> mr2 // throw away the bytes we just read and recycle the existing ILModuleReader | _ -> let bytes = File.ReadAllBytes fileName @@ -6652,7 +6652,7 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let mdchunk = File.ReadBinaryChunk (fileName, pe.MetadataPhysLoc, pe.MetadataSize) let mdfile = ByteFile(mdchunk) let mr = ILModuleReader(fileName, mdfile, ilGlobals, true) - readersWeakCache.[key] <- CacheValue (mr) + readersWeakCache.[key] <- CacheValue (mr, timeStamp) mr From 4d52582679d2918142afcc2754805669499a08ae Mon Sep 17 00:00:00 2001 From: Don Syme Date: Thu, 22 Nov 2018 23:26:17 +0000 Subject: [PATCH 5/6] read only metadata section --- src/ProvidedTypes.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ProvidedTypes.fs b/src/ProvidedTypes.fs index cd3c3fde..ffd73a2f 100644 --- a/src/ProvidedTypes.fs +++ b/src/ProvidedTypes.fs @@ -4185,9 +4185,10 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader type ByteFile(bytes:byte[]) = - member __.Bytes = bytes member __.ReadByte addr = bytes.[addr] + member __.ReadBytes addr len = Array.sub bytes addr len + member __.CountUtf8String addr = let mutable p = addr while bytes.[p] <> 0uy do @@ -6097,7 +6098,6 @@ namespace ProviderImplementation.ProvidedTypes.AssemblyReader let ilModule = seekReadModule (ilMetadataVersion) 1 let ilAssemblyRefs = [ for i in 1 .. getNumRows ILTableNames.AssemblyRef do yield seekReadAssemblyRef i ] - member __.MetadataBytes = is.Bytes member __.ILGlobals = ilg member __.ILModuleDef = ilModule member __.ILAssemblyRefs = ilAssemblyRefs From 67cd11d0206a75d3bd399537ad7c7c27fb35979d Mon Sep 17 00:00:00 2001 From: Don Syme Date: Thu, 22 Nov 2018 23:38:54 +0000 Subject: [PATCH 6/6] read only metadata section --- src/ProvidedTypes.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ProvidedTypes.fs b/src/ProvidedTypes.fs index ffd73a2f..944c50cb 100644 --- a/src/ProvidedTypes.fs +++ b/src/ProvidedTypes.fs @@ -9197,7 +9197,7 @@ namespace ProviderImplementation.ProvidedTypes let ilg = ilGlobals.Force() let is = ByteFile(bytes) let pe = PEReader(fileName, is) - let mdchunk = File.ReadBinaryChunk (fileName, pe.MetadataPhysLoc, pe.MetadataSize) + let mdchunk = bytes.[pe.MetadataPhysLoc .. pe.MetadataPhysLoc + pe.MetadataSize - 1] let mdfile = ByteFile(mdchunk) let reader = ILModuleReader(fileName, mdfile, ilg, true) TargetAssembly(ilg, this.TryBindILAssemblyRefToTgt, Some reader, fileName) :> Assembly @@ -14518,12 +14518,12 @@ namespace ProviderImplementation.ProvidedTypes assemblyBuilder.Save () //printfn "re-reading generated binary from '%s'" assemblyFileName let reader = ILModuleReaderAfterReadingAllBytes(assemblyFileName, ilg) + let bytes = File.ReadAllBytes(assemblyFileName) #if DEBUG printfn "generated binary is at '%s'" assemblyFileName #else File.Delete assemblyFileName #endif - let bytes = File.ReadAllBytes(assemblyFileName) // Use a real Reflection Load when running in F# Interactive if isHostedExecution then