Skip to content

Further possible fix to reduce memory usage #6390

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 9, 2019
Merged

Further possible fix to reduce memory usage #6390

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a31eab
reduce memory usage
dsyme Mar 28, 2019
c8e7c7c
further possible fix
dsyme Mar 28, 2019
6a971b1
further possible fix
dsyme Mar 28, 2019
8c094bf
Merge branch 'master' of http://github.com/Microsoft/visualfsharp int…
dsyme Apr 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 43 additions & 16 deletions src/absil/ilread.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ module FSharp.Compiler.AbstractIL.ILBinaryReader
#nowarn "42" // This construct is deprecated: it is only for use in the F# library

open System
open System.Collections.Concurrent
open System.Collections.Generic
open System.Diagnostics
open System.IO
Expand Down Expand Up @@ -3601,11 +3602,11 @@ let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, p
let inbase = Filename.fileNameOfPath fileName + ": "

// All the caches. The sizes are guesstimates for the rough sharing-density of the assembly
let cacheAssemblyRef = mkCacheInt32 reduceMemoryUsage inbase "ILAssemblyRef" (getNumRows TableNames.AssemblyRef)
let cacheAssemblyRef = mkCacheInt32 false inbase "ILAssemblyRef" (getNumRows TableNames.AssemblyRef)
let cacheMethodSpecAsMethodData = mkCacheGeneric reduceMemoryUsage inbase "MethodSpecAsMethodData" (getNumRows TableNames.MethodSpec / 20 + 1)
let cacheMemberRefAsMemberData = mkCacheGeneric reduceMemoryUsage inbase "MemberRefAsMemberData" (getNumRows TableNames.MemberRef / 20 + 1)
let cacheCustomAttr = mkCacheGeneric reduceMemoryUsage inbase "CustomAttr" (getNumRows TableNames.CustomAttribute / 50 + 1)
let cacheTypeRef = mkCacheInt32 reduceMemoryUsage inbase "ILTypeRef" (getNumRows TableNames.TypeRef / 20 + 1)
let cacheTypeRef = mkCacheInt32 false inbase "ILTypeRef" (getNumRows TableNames.TypeRef / 20 + 1)
let cacheTypeRefAsType = mkCacheGeneric reduceMemoryUsage inbase "TypeRefAsType" (getNumRows TableNames.TypeRef / 20 + 1)
let cacheBlobHeapAsPropertySig = mkCacheGeneric reduceMemoryUsage inbase "BlobHeapAsPropertySig" (getNumRows TableNames.Property / 20 + 1)
let cacheBlobHeapAsFieldSig = mkCacheGeneric reduceMemoryUsage inbase "BlobHeapAsFieldSig" (getNumRows TableNames.Field / 20 + 1)
Expand Down Expand Up @@ -3964,10 +3965,19 @@ type ILModuleReaderImpl(ilModule: ILModuleDef, ilAssemblyRefs: Lazy<ILAssemblyRe
member x.Dispose() = dispose()

// ++GLOBAL MUTABLE STATE (concurrency safe via locking)
type ILModuleReaderCacheLockToken() = interface LockToken
type ILModuleReaderCacheKey = ILModuleReaderCacheKey of string * DateTime * ILScopeRef * bool * ReduceMemoryFlag * MetadataOnlyFlag
let ilModuleReaderCache = new AgedLookup<ILModuleReaderCacheLockToken, ILModuleReaderCacheKey, ILModuleReader>(stronglyHeldReaderCacheSize, areSimilar=(fun (x, y) -> x = y))
let ilModuleReaderCacheLock = Lock()

// Cache to extend the lifetime of a limited number of readers that are otherwise eligible for GC
type ILModuleReaderCache1LockToken() = interface LockToken
let ilModuleReaderCache1 =
new AgedLookup<ILModuleReaderCache1LockToken, ILModuleReaderCacheKey, ILModuleReader>
(stronglyHeldReaderCacheSize,
keepMax=stronglyHeldReaderCacheSize, // only strong entries
areSimilar=(fun (x, y) -> x = y))
let ilModuleReaderCache1Lock = Lock()

// // Cache to reuse readers that have already been created and are not yet GC'd
let ilModuleReaderCache2 = new ConcurrentDictionary<ILModuleReaderCacheKey, System.WeakReference<ILModuleReader>>(HashIdentity.Structural)

let stableFileHeuristicApplies fileName =
not noStableFileHeuristic && try FileSystem.IsStableFileHeuristic fileName with _ -> false
Expand Down Expand Up @@ -4016,17 +4026,29 @@ let OpenILModuleReader fileName opts =
let fakeKey = ILModuleReaderCacheKey(fileName, System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes)
fakeKey, false

let cacheResult =
if keyOk then
if opts.pdbDirPath.IsSome then None // can't used a cached entry when reading PDBs, since it makes the returned object IDisposable
else ilModuleReaderCacheLock.AcquireLock (fun ltok -> ilModuleReaderCache.TryGet(ltok, key))
let cacheResult1 =
// can't used a cached entry when reading PDBs, since it makes the returned object IDisposable
if keyOk && opts.pdbDirPath.IsNone then
ilModuleReaderCache1Lock.AcquireLock (fun ltok -> ilModuleReaderCache1.TryGet(ltok, key))
else
None

match cacheResult with
match cacheResult1 with
| Some ilModuleReader -> ilModuleReader
| None ->

let cacheResult2 =
// can't used a cached entry when reading PDBs, since it makes the returned object IDisposable
if keyOk && opts.pdbDirPath.IsNone then
ilModuleReaderCache2.TryGetValue(key)
else
false, Unchecked.defaultof<_>

let mutable res = Unchecked.defaultof<_>
match cacheResult2 with
| true, weak when weak.TryGetTarget(&res) -> res
| _ ->

let reduceMemoryUsage = (opts.reduceMemoryUsage = ReduceMemoryFlag.Yes)
let metadataOnly = (opts.metadataOnly = MetadataOnlyFlag.Yes)

Expand Down Expand Up @@ -4065,10 +4087,12 @@ let OpenILModuleReader fileName opts =
let ilModule, ilAssemblyRefs, _pdb = openPE (fullPath, pefile, None, reduceMemoryUsage, opts.ilGlobals, false)
new ILModuleReaderImpl(ilModule, ilAssemblyRefs, ignore)

let ilModuleReader = ilModuleReader :> ILModuleReader
if keyOk then
ilModuleReaderCacheLock.AcquireLock (fun ltok -> ilModuleReaderCache.Put(ltok, key, ilModuleReader))

ilModuleReader :> ILModuleReader
ilModuleReaderCache1Lock.AcquireLock (fun ltok -> ilModuleReaderCache1.Put(ltok, key, ilModuleReader))
ilModuleReaderCache2.[key] <- System.WeakReference<_>(ilModuleReader)
ilModuleReader


else
// This case is primarily used in fsc.exe.
Expand All @@ -4092,11 +4116,14 @@ let OpenILModuleReader fileName opts =
let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbDirPath, reduceMemoryUsage, opts.ilGlobals, false)
let ilModuleReader = new ILModuleReaderImpl(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb))

let ilModuleReader = ilModuleReader :> ILModuleReader

// Readers with PDB reader disposal logic don't go in the cache. Note the PDB reader is only used in static linking.
if keyOk && opts.pdbDirPath.IsNone then
ilModuleReaderCacheLock.AcquireLock (fun ltok -> ilModuleReaderCache.Put(ltok, key, ilModuleReader))
ilModuleReaderCache1Lock.AcquireLock (fun ltok -> ilModuleReaderCache1.Put(ltok, key, ilModuleReader))
ilModuleReaderCache2.[key] <- WeakReference<_>(ilModuleReader)

ilModuleReader :> ILModuleReader
ilModuleReader

[<AutoOpen>]
module Shim =
Expand Down