From 12ba5b4d8736facf3e8d31fde6e22dd27f5256e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Wed, 8 Nov 2023 14:09:39 +0100 Subject: [PATCH 1/8] Implement frozen object heap When allocating a RuntimeType instances, we were creating an object on the pinned object heap, creating a handle to it, and purposefully leaked the handle. The RuntimeTypes live forever. This fragments the pinned object heap. So instead of doing that, port frozen object heap from CoreCLR. This is a line-by-line port. Frozen object heap is a segmented bump memory allocator that interacts with the GC to tell it the boundaries of the segments. --- .../CompilerHelpers/StartupCodeHelpers.cs | 2 +- .../src/System/Runtime/RuntimeImports.cs | 8 - src/coreclr/nativeaot/Runtime/MiscHelpers.cpp | 11 +- src/coreclr/nativeaot/Runtime/gcrhenv.cpp | 20 +- src/coreclr/nativeaot/Runtime/gcrhinterface.h | 3 +- .../Runtime/FrozenObjectHeapManager.Unix.cs | 38 +++ .../FrozenObjectHeapManager.Windows.cs | 23 ++ .../Runtime/FrozenObjectHeapManager.cs | 269 ++++++++++++++++++ .../src/System.Private.CoreLib.csproj | 12 + .../src/System/GC.NativeAot.cs | 6 +- .../src/System/Runtime/RuntimeImports.cs | 13 +- .../src/System/Type.NativeAot.cs | 37 +-- .../src/System/Runtime/RuntimeImports.cs | 12 +- src/native/libs/System.Native/entrypoints.c | 1 + src/native/libs/System.Native/pal_io.c | 13 + src/native/libs/System.Native/pal_io.h | 7 + 16 files changed, 429 insertions(+), 46 deletions(-) create mode 100644 src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs create mode 100644 src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs create mode 100644 src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs diff --git a/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs b/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs index 21c9282c1ecf87..10947ec8831368 100644 --- a/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs +++ b/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs @@ -153,7 +153,7 @@ private static unsafe void InitializeGlobalTablesForModule(TypeManagerHandle typ private static unsafe void InitializeModuleFrozenObjectSegment(IntPtr segmentStart, int length) { - if (RuntimeImports.RhpRegisterFrozenSegment(segmentStart, (IntPtr)length) == IntPtr.Zero) + if (RuntimeImports.RhRegisterFrozenSegment((void*)segmentStart, (nuint)length, (nuint)length, (nuint)length) == IntPtr.Zero) { // This should only happen if we ran out of memory. RuntimeExceptionHelpers.FailFast("Failed to register frozen object segment for the module."); diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeImports.cs index 76835dd8b7ed95..c1663688004012 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeImports.cs @@ -13,14 +13,6 @@ internal static class RuntimeImports { private const string RuntimeLibrary = "*"; - [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhpRegisterFrozenSegment")] - internal static extern IntPtr RhpRegisterFrozenSegment(IntPtr pSegmentStart, IntPtr length); - - [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhpUnregisterFrozenSegment")] - internal static extern void RhpUnregisterFrozenSegment(IntPtr pSegmentHandle); - [RuntimeImport(RuntimeLibrary, "RhpGetModuleSection")] [MethodImplAttribute(MethodImplOptions.InternalCall)] private static extern IntPtr RhGetModuleSection(ref TypeManagerHandle module, ReadyToRunSectionType section, out int length); diff --git a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp index 1dd79b73d9436c..56bb8ffd4ae942 100644 --- a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp @@ -356,12 +356,17 @@ EXTERN_C NATIVEAOT_API int32_t __cdecl RhpGetCurrentThreadStackTrace(void* pOutp return RhpCalculateStackTraceWorker(pOutputBuffer, outputBufferLength, pAddressInCurrentFrame); } -COOP_PINVOKE_HELPER(void*, RhpRegisterFrozenSegment, (void* pSegmentStart, size_t length)) +COOP_PINVOKE_HELPER(void*, RhRegisterFrozenSegment, (void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize)) { - return RedhawkGCInterface::RegisterFrozenSegment(pSegmentStart, length); + return RedhawkGCInterface::RegisterFrozenSegment(pSection, allocSize, commitSize, reservedSize); } -COOP_PINVOKE_HELPER(void, RhpUnregisterFrozenSegment, (void* pSegmentHandle)) +COOP_PINVOKE_HELPER(void, RhUpdateFrozenSegment, (void* pSegmentHandle, uint8_t* allocated, uint8_t* committed)) +{ + RedhawkGCInterface::UpdateFrozenSegment((GcSegmentHandle)pSegmentHandle, allocated, committed); +} + +COOP_PINVOKE_HELPER(void, RhUnregisterFrozenSegment, (void* pSegmentHandle)) { RedhawkGCInterface::UnregisterFrozenSegment((GcSegmentHandle)pSegmentHandle); } diff --git a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp index 09280870510c95..4f7e62ba22e4c9 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp +++ b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp @@ -368,16 +368,20 @@ void RedhawkGCInterface::BulkEnumGcObjRef(PTR_RtuObjectRef pRefs, uint32_t cRefs } // static -GcSegmentHandle RedhawkGCInterface::RegisterFrozenSegment(void * pSection, size_t SizeSection) +GcSegmentHandle RedhawkGCInterface::RegisterFrozenSegment(void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize) { + ASSERT(allocSize <= commitSize); + ASSERT(commitSize <= reservedSize); + ASSERT(allocSize >= MIN_OBJECT_SIZE); + #ifdef FEATURE_BASICFREEZE segment_info seginfo; seginfo.pvMem = pSection; seginfo.ibFirstObject = sizeof(ObjHeader); - seginfo.ibAllocated = SizeSection; - seginfo.ibCommit = seginfo.ibAllocated; - seginfo.ibReserved = seginfo.ibAllocated; + seginfo.ibAllocated = allocSize; + seginfo.ibCommit = commitSize; + seginfo.ibReserved = reservedSize; return (GcSegmentHandle)GCHeapUtilities::GetGCHeap()->RegisterFrozenSegment(&seginfo); #else // FEATURE_BASICFREEZE @@ -385,6 +389,14 @@ GcSegmentHandle RedhawkGCInterface::RegisterFrozenSegment(void * pSection, size_ #endif // FEATURE_BASICFREEZE } +// static +void RedhawkGCInterface::UpdateFrozenSegment(GcSegmentHandle seg, uint8_t* allocated, uint8_t* committed) +{ + ASSERT(allocated <= committed); + + GCHeapUtilities::GetGCHeap()->UpdateFrozenSegment((segment_handle)seg, allocated, committed); +} + // static void RedhawkGCInterface::UnregisterFrozenSegment(GcSegmentHandle segment) { diff --git a/src/coreclr/nativeaot/Runtime/gcrhinterface.h b/src/coreclr/nativeaot/Runtime/gcrhinterface.h index b9279e00902157..b53e9af9e083e8 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhinterface.h +++ b/src/coreclr/nativeaot/Runtime/gcrhinterface.h @@ -129,7 +129,8 @@ class RedhawkGCInterface void * pfnEnumCallback, void * pvCallbackData); - static GcSegmentHandle RegisterFrozenSegment(void * pSection, size_t SizeSection); + static GcSegmentHandle RegisterFrozenSegment(void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize); + static void UpdateFrozenSegment(GcSegmentHandle seg, uint8_t* allocated, uint8_t* committed); static void UnregisterFrozenSegment(GcSegmentHandle segment); #ifdef FEATURE_GC_STRESS diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs new file mode 100644 index 00000000000000..5c1c20f42af67e --- /dev/null +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Win32.SafeHandles; + +namespace Internal.Runtime +{ + internal unsafe partial class FrozenObjectHeapManager + { + static void* ClrVirtualReserve(nuint size) + { + // The shim will return null for failure + return (void*)Interop.Sys.MMap( + 0, + size, + Interop.Sys.MemoryMappedProtections.PROT_NONE, + Interop.Sys.MemoryMappedFlags.MAP_PRIVATE | Interop.Sys.MemoryMappedFlags.MAP_ANONYMOUS, + new SafeFileHandle(-1, false), + 0); + + } + + static void* ClrVirtualCommit(void* pBase, nuint size) + { + int result = Interop.Sys.MProtect( + (nint)pBase, + size, + Interop.Sys.MemoryMappedProtections.PROT_READ | Interop.Sys.MemoryMappedProtections.PROT_WRITE); + + return result == 0 ? pBase : null; + } + + static void ClrVirtualFree(void* pBase, nuint size) + { + Interop.Sys.MUnmap((nint)pBase, size); + } + } +} diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs new file mode 100644 index 00000000000000..7d4f0ffe107af5 --- /dev/null +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Internal.Runtime +{ + internal unsafe partial class FrozenObjectHeapManager + { + static void* ClrVirtualReserve(nuint size) + { + return Interop.Kernel32.VirtualAlloc(null, size, Interop.Kernel32.MemOptions.MEM_RESERVE, Interop.Kernel32.PageOptions.PAGE_READWRITE); + } + + static void* ClrVirtualCommit(void* pBase, nuint size) + { + return Interop.Kernel32.VirtualAlloc(pBase, size, Interop.Kernel32.MemOptions.MEM_COMMIT, Interop.Kernel32.PageOptions.PAGE_READWRITE); + } + + static void ClrVirtualFree(void* pBase, nuint size) + { + Interop.Kernel32.VirtualFree(pBase, size, Interop.Kernel32.MemOptions.MEM_RELEASE); + } + } +} diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs new file mode 100644 index 00000000000000..b87d0e59dc3a70 --- /dev/null +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs @@ -0,0 +1,269 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Runtime; +using System.Runtime.CompilerServices; +using System.Threading; + +using Debug = System.Diagnostics.Debug; + +namespace Internal.Runtime +{ + internal unsafe partial class FrozenObjectHeapManager + { + public static FrozenObjectHeapManager Instance = new FrozenObjectHeapManager(); + + private readonly LowLevelLock m_Crst = new LowLevelLock(); + private readonly LowLevelLock m_SegmentRegistrationCrst = new LowLevelLock(); + private ArrayBuilder m_FrozenSegments; + private FrozenObjectSegment m_CurrentSegment; + + // Default size to reserve for a frozen segment + private const nuint FOH_SEGMENT_DEFAULT_SIZE = 4 * 1024 * 1024; + // Size to commit on demand in that reserved space + private const nuint FOH_COMMIT_SIZE = 64 * 1024; + + public T? TryAllocateObject() where T : class + { + MethodTable* pMT = MethodTable.Of(); + return Unsafe.As(TryAllocateObject(pMT, pMT->BaseSize)); + } + + private object? TryAllocateObject(MethodTable* type, nuint objectSize) + { + HalfBakedObject* obj = null; + FrozenObjectSegment? curSeg = null; + byte* curSegmentCurrent = null; + nuint curSegSizeCommitted = 0; + + m_Crst.Acquire(); + + try + { + Debug.Assert(type != null); + // _ASSERT(FOH_COMMIT_SIZE >= MIN_OBJECT_SIZE); + + // Currently we don't support frozen objects with special alignment requirements + // TODO: We should also give up on arrays of doubles on 32-bit platforms. + // (we currently never allocate them on frozen segments) +#if FEATURE_64BIT_ALIGNMENT + if (type->RequiresAlign8) + { + // Align8 objects are not supported yet + return nullptr; + } +#endif + + // NOTE: objectSize is expected be the full size including header + // _ASSERT(objectSize >= MIN_OBJECT_SIZE); + + if (objectSize > FOH_COMMIT_SIZE) + { + // The current design doesn't allow objects larger than FOH_COMMIT_SIZE and + // since FrozenObjectHeap is just an optimization, let's not fill it with huge objects. + return null; + } + + obj = m_CurrentSegment == null ? null : m_CurrentSegment.TryAllocateObject(type, objectSize); + // obj is nullptr if the current segment is full or hasn't been allocated yet + if (obj == null) + { + nuint newSegmentSize = FOH_SEGMENT_DEFAULT_SIZE; + if (m_CurrentSegment != null) + { + // Double the reserved size to reduce the number of frozen segments in apps with lots of frozen objects + // Use the same size in case if prevSegmentSize*2 operation overflows. + nuint prevSegmentSize = m_CurrentSegment.m_Size; + newSegmentSize = Math.Max(prevSegmentSize, prevSegmentSize * 2); + } + + m_CurrentSegment = new FrozenObjectSegment(newSegmentSize); + m_FrozenSegments.Add(m_CurrentSegment); + + // Try again + obj = m_CurrentSegment.TryAllocateObject(type, objectSize); + + // This time it's not expected to be null + Debug.Assert(obj != null); + } + + curSeg = m_CurrentSegment; + curSegSizeCommitted = curSeg.m_SizeCommitted; + curSegmentCurrent = curSeg.m_pCurrent; + } // end of m_Crst lock + finally + { + m_Crst.Release(); + } + + // Let GC know about the new segment or changes in it. + // We do it under a new lock because the main one (m_Crst) can be used by Profiler in a GC's thread + // and that might cause deadlocks since RegisterFrozenSegment may stuck on GC's lock. + m_SegmentRegistrationCrst.Acquire(); + try + { + curSeg.RegisterOrUpdate(curSegmentCurrent, curSegSizeCommitted); + } + finally + { + m_SegmentRegistrationCrst.Release(); + } + + //PublishFrozenObject(obj); + + IntPtr result = (IntPtr)obj; + + return Unsafe.As(ref result); + } + + private class FrozenObjectSegment + { + // Start of the reserved memory, the first object starts at "m_pStart + sizeof(ObjHeader)" (its pMT) + byte* m_pStart; + + // NOTE: To handle potential race conditions, only m_[x]Registered fields should be accessed + // externally as they guarantee that GC is aware of the current state of the segment. + + // Pointer to the end of the current segment, ready to be used as a pMT for a new object + // meaning that "m_pCurrent - sizeof(ObjHeader)" is the actual start of the new object (header). + // + // m_pCurrent <= m_SizeCommitted + public byte* m_pCurrent; + + // Last known value of m_pCurrent that GC is aware of. + // + // m_pCurrentRegistered <= m_pCurrent + byte* m_pCurrentRegistered; + + // Memory committed in the current segment + // + // m_SizeCommitted <= m_pStart + FOH_SIZE_RESERVED + public nuint m_SizeCommitted; + + // Total memory reserved for the current segment + public nuint m_Size; + + IntPtr m_SegmentHandle; + + public FrozenObjectSegment(nuint sizeHint) + { + m_Size = sizeHint; + + Debug.Assert(m_Size > FOH_COMMIT_SIZE); + Debug.Assert(m_Size % FOH_COMMIT_SIZE == 0); + + void* alloc = ClrVirtualReserve(m_Size); + if (alloc == null) + { + // Try again with the default FOH size + if (m_Size > FOH_SEGMENT_DEFAULT_SIZE) + { + m_Size = FOH_SEGMENT_DEFAULT_SIZE; + Debug.Assert(m_Size > FOH_COMMIT_SIZE); + Debug.Assert(m_Size % FOH_COMMIT_SIZE == 0); + alloc = ClrVirtualReserve(m_Size); + } + + if (alloc == null) + { + throw new OutOfMemoryException(); + } + } + + // Commit a chunk in advance + void* committedAlloc = ClrVirtualCommit(alloc, FOH_COMMIT_SIZE); + if (committedAlloc == null) + { + ClrVirtualFree(alloc, 0); + throw new OutOfMemoryException(); + } + + m_pStart = (byte*)committedAlloc; + m_pCurrent = m_pStart + sizeof(ObjHeader); + m_SizeCommitted = FOH_COMMIT_SIZE; + + // ClrVirtualAlloc is expected to be PageSize-aligned so we can expect + // DATA_ALIGNMENT alignment as well + // _ASSERT(IS_ALIGNED(committedAlloc, DATA_ALIGNMENT)); + } + + public void RegisterOrUpdate(byte* current, nuint sizeCommited) + { + if (m_pCurrentRegistered == null) + { + Debug.Assert(current >= m_pStart); + + // NOTE: RegisterFrozenSegment may take a GC lock inside. + m_SegmentHandle = RuntimeImports.RhRegisterFrozenSegment(m_pStart, (nuint)current - (nuint)m_pStart, sizeCommited, m_Size); + if (m_SegmentHandle == IntPtr.Zero) + { + throw new OutOfMemoryException(); + } + m_pCurrentRegistered = current; + } + else + { + if (current > m_pCurrentRegistered) + { + RuntimeImports.RhUpdateFrozenSegment( + m_SegmentHandle, current, m_pStart + sizeCommited); + m_pCurrentRegistered = current; + } + else + { + // Some other thread already advanced it. + } + } + } + + public HalfBakedObject* TryAllocateObject(MethodTable* type, nuint objectSize) + { + Debug.Assert((m_pStart != null) && (m_Size > 0)); + //_ASSERT(IS_ALIGNED(m_pCurrent, DATA_ALIGNMENT)); + //_ASSERT(IS_ALIGNED(objectSize, DATA_ALIGNMENT)); + Debug.Assert(objectSize <= FOH_COMMIT_SIZE); + Debug.Assert(m_pCurrent >= m_pStart + sizeof(ObjHeader)); + + nuint spaceUsed = (nuint)(m_pCurrent - m_pStart); + nuint spaceLeft = m_Size - spaceUsed; + + Debug.Assert(spaceUsed >= (nuint)sizeof(ObjHeader)); + Debug.Assert(spaceLeft >= (nuint)sizeof(ObjHeader)); + + // Test if we have a room for the given object (including extra sizeof(ObjHeader) for next object) + if (spaceLeft - (nuint)sizeof(ObjHeader) < objectSize) + { + return null; + } + + // Check if we need to commit a new chunk + if (spaceUsed + objectSize + (nuint)sizeof(ObjHeader) > m_SizeCommitted) + { + // Make sure we don't go out of bounds during this commit + Debug.Assert(m_SizeCommitted + FOH_COMMIT_SIZE <= m_Size); + + if (ClrVirtualCommit(m_pStart + m_SizeCommitted, FOH_COMMIT_SIZE) == null) + { + throw new OutOfMemoryException(); + } + m_SizeCommitted += FOH_COMMIT_SIZE; + } + + HalfBakedObject* obj = (HalfBakedObject*)m_pCurrent; + obj->SetMethodTable(type); + + m_pCurrent += objectSize; + + return obj; + } + } + + private struct HalfBakedObject + { + private MethodTable* _methodTable; + public void SetMethodTable(MethodTable* methodTable) => _methodTable = methodTable; + } + } +} diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj index 79321635129465..bd6f9a3127a34e 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj @@ -94,6 +94,7 @@ Common\src\Internal\Runtime\TypeLoader\ExternalReferencesTable.cs + @@ -252,6 +253,7 @@ + @@ -291,6 +293,7 @@ + @@ -302,6 +305,15 @@ Interop\Unix\System.Native\Interop.Exit.cs + + Interop\Unix\System.Native\Interop.MMap.cs + + + Interop\Unix\System.Native\Interop.MProtect.cs + + + Interop\Unix\System.Native\Interop.MUnmap.cs + diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs index 3d263d5de6b63d..8ecade80ffc253 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs @@ -741,14 +741,14 @@ public static long GetTotalMemory(bool forceFullCollection) return size; } - private static IntPtr _RegisterFrozenSegment(IntPtr sectionAddress, IntPtr sectionSize) + private static unsafe IntPtr _RegisterFrozenSegment(IntPtr sectionAddress, IntPtr sectionSize) { - return RuntimeImports.RhpRegisterFrozenSegment(sectionAddress, sectionSize); + return RuntimeImports.RhRegisterFrozenSegment((void*)sectionAddress, (nuint)sectionSize, (nuint)sectionSize, (nuint)sectionSize); } private static void _UnregisterFrozenSegment(IntPtr segmentHandle) { - RuntimeImports.RhpUnregisterFrozenSegment(segmentHandle); + RuntimeImports.RhUnregisterFrozenSegment(segmentHandle); } public static long GetAllocatedBytesForCurrentThread() diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index 46e91a79aa636c..b6871f7d0199ab 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -165,13 +165,14 @@ internal static void RhWaitForPendingFinalizers(bool allowReentrantWait) [RuntimeImport(RuntimeLibrary, "RhGetLastGCDuration")] internal static extern long RhGetLastGCDuration(int generation); - [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhpRegisterFrozenSegment")] - internal static extern IntPtr RhpRegisterFrozenSegment(IntPtr pSegmentStart, IntPtr length); + [LibraryImport(RuntimeLibrary)] + internal static extern unsafe IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); - [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhpUnregisterFrozenSegment")] - internal static extern void RhpUnregisterFrozenSegment(IntPtr pSegmentHandle); + [LibraryImport(RuntimeLibrary)] + internal static unsafe partial void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); + + [LibraryImport(RuntimeLibrary)] + internal static extern void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [MethodImpl(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhRegisterForFullGCNotification")] diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Type.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Type.NativeAot.cs index 0a9cd70dd643e1..d9c30882033000 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Type.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Type.NativeAot.cs @@ -40,31 +40,36 @@ internal static unsafe RuntimeType GetTypeFromMethodTable(MethodTable* pMT) } } + private static class AllocationLockHolder + { + public static LowLevelLock AllocationLock = new LowLevelLock(); + } + [MethodImpl(MethodImplOptions.NoInlining)] private static unsafe RuntimeType GetTypeFromMethodTableSlow(MethodTable* pMT) { - // TODO: instead of fragmenting the frozen object heap, we should have our own allocator - // for objects that live forever outside the GC heap. + // Allocate and set the RuntimeType under a lock - there's no way to free it if there is a race. + AllocationLockHolder.AllocationLock.Acquire(); + try + { + ref RuntimeType? runtimeTypeCache = ref Unsafe.AsRef(pMT->WritableData); + if (runtimeTypeCache != null) + return runtimeTypeCache; - RuntimeType? type = null; - RuntimeImports.RhAllocateNewObject( - (IntPtr)MethodTable.Of(), - (uint)GC_ALLOC_FLAGS.GC_ALLOC_PINNED_OBJECT_HEAP, - Unsafe.AsPointer(ref type)); + RuntimeType? type = FrozenObjectHeapManager.Instance.TryAllocateObject(); + if (type == null) + throw new OutOfMemoryException(); - if (type == null) - throw new OutOfMemoryException(); + type.DangerousSetUnderlyingEEType(pMT); - type.DangerousSetUnderlyingEEType(pMT); + runtimeTypeCache = type; - ref RuntimeType? runtimeTypeCache = ref Unsafe.AsRef(pMT->WritableData); - if (Interlocked.CompareExchange(ref runtimeTypeCache, type, null) == null) + return type; + } + finally { - // Create and leak a GC handle - UnsafeGCHandle.Alloc(type); + AllocationLockHolder.AllocationLock.Release(); } - - return runtimeTypeCache; } // diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs index 5232ab073a6ca8..41eadb5bc13eb4 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -41,12 +41,16 @@ internal static IntPtr RhHandleAlloc(object value, GCHandleType type) } [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhpRegisterFrozenSegment")] - internal static extern IntPtr RhpRegisterFrozenSegment(IntPtr pSegmentStart, IntPtr length); + [RuntimeImport(RuntimeLibrary, "RhRegisterFrozenSegment")] + internal static extern unsafe IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhpUnregisterFrozenSegment")] - internal static extern void RhpUnregisterFrozenSegment(IntPtr pSegmentHandle); + [RuntimeImport(RuntimeLibrary, "RhUpdateFrozenSegment")] + internal static extern unsafe void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); + + [MethodImpl(MethodImplOptions.InternalCall)] + [RuntimeImport(RuntimeLibrary, "RhUnregisterFrozenSegment")] + internal static extern void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [RuntimeImport(RuntimeLibrary, "RhpGetModuleSection")] [MethodImplAttribute(MethodImplOptions.InternalCall)] diff --git a/src/native/libs/System.Native/entrypoints.c b/src/native/libs/System.Native/entrypoints.c index f4fcd6f8d4d86c..4e23c1dd67d45a 100644 --- a/src/native/libs/System.Native/entrypoints.c +++ b/src/native/libs/System.Native/entrypoints.c @@ -93,6 +93,7 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_MksTemps) DllImportEntry(SystemNative_MMap) DllImportEntry(SystemNative_MUnmap) + DllImportEntry(SystemNative_MProtect); DllImportEntry(SystemNative_MAdvise) DllImportEntry(SystemNative_MSync) DllImportEntry(SystemNative_SysConf) diff --git a/src/native/libs/System.Native/pal_io.c b/src/native/libs/System.Native/pal_io.c index d6d4a84e1faeb2..9c65db55c4ee03 100644 --- a/src/native/libs/System.Native/pal_io.c +++ b/src/native/libs/System.Native/pal_io.c @@ -1019,6 +1019,19 @@ int32_t SystemNative_MUnmap(void* address, uint64_t length) return munmap(address, (size_t)length); } +int32_t SystemNative_MProtect(void* address, uint64_t length, int32_t protection) +{ + if (length > SIZE_MAX) + { + errno = ERANGE; + return -1; + } + + protection = ConvertMMapProtection(protection); + + return mprotect(address, (size_t)length, protection); +} + int32_t SystemNative_MAdvise(void* address, uint64_t length, int32_t advice) { if (length > SIZE_MAX) diff --git a/src/native/libs/System.Native/pal_io.h b/src/native/libs/System.Native/pal_io.h index 5ad83e29ed99ee..03fd94cea25417 100644 --- a/src/native/libs/System.Native/pal_io.h +++ b/src/native/libs/System.Native/pal_io.h @@ -600,6 +600,13 @@ PALEXPORT void* SystemNative_MMap(void* address, */ PALEXPORT int32_t SystemNative_MUnmap(void* address, uint64_t length); +/** + * Change the access protections for the specified memory pages. + * + * Returns 0 for success, -1 for failure. Sets errno on failure. + */ +PALEXPORT int32_t SystemNative_MProtect(void* address, uint64_t length, int32_t protection); + /** * Give advice about use of memory. Implemented as shim to madvise(2). * From 5e9c3c508da093a1c3050ad39a768f061ed26972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Wed, 8 Nov 2023 14:13:00 +0100 Subject: [PATCH 2/8] Fixes --- .../src/System/Runtime/RuntimeImports.cs | 4 ++-- .../Test.CoreLib/src/System/Runtime/RuntimeImports.cs | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index b6871f7d0199ab..c29e4190f86ace 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -166,13 +166,13 @@ internal static void RhWaitForPendingFinalizers(bool allowReentrantWait) internal static extern long RhGetLastGCDuration(int generation); [LibraryImport(RuntimeLibrary)] - internal static extern unsafe IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); + internal static unsafe partial IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); [LibraryImport(RuntimeLibrary)] internal static unsafe partial void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); [LibraryImport(RuntimeLibrary)] - internal static extern void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); + internal static partial void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [MethodImpl(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhRegisterForFullGCNotification")] diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs index 41eadb5bc13eb4..195aa4693ae8c2 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -40,16 +40,13 @@ internal static IntPtr RhHandleAlloc(object value, GCHandleType type) return h; } - [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhRegisterFrozenSegment")] + [DllImport(RuntimeLibrary)] internal static extern unsafe IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); - [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhUpdateFrozenSegment")] + [DllImport(RuntimeLibrary)] internal static extern unsafe void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); - [MethodImpl(MethodImplOptions.InternalCall)] - [RuntimeImport(RuntimeLibrary, "RhUnregisterFrozenSegment")] + [DllImport(RuntimeLibrary)] internal static extern void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [RuntimeImport(RuntimeLibrary, "RhpGetModuleSection")] From dd033d6fb2c1f51c1522aaaff4110e91c2f40aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Wed, 8 Nov 2023 14:42:44 +0100 Subject: [PATCH 3/8] Update entrypoints.c --- src/native/libs/System.Native/entrypoints.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/native/libs/System.Native/entrypoints.c b/src/native/libs/System.Native/entrypoints.c index 4e23c1dd67d45a..1a5b7dcae503fd 100644 --- a/src/native/libs/System.Native/entrypoints.c +++ b/src/native/libs/System.Native/entrypoints.c @@ -93,7 +93,7 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_MksTemps) DllImportEntry(SystemNative_MMap) DllImportEntry(SystemNative_MUnmap) - DllImportEntry(SystemNative_MProtect); + DllImportEntry(SystemNative_MProtect) DllImportEntry(SystemNative_MAdvise) DllImportEntry(SystemNative_MSync) DllImportEntry(SystemNative_SysConf) From 9cd6bce230d0d6fe39f30e2f1d4b41f086acb7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Wed, 8 Nov 2023 22:13:08 +0100 Subject: [PATCH 4/8] Forgot a file --- .../Unix/System.Native/Interop.MProtect.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.MProtect.cs diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MProtect.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MProtect.cs new file mode 100644 index 00000000000000..482848da93cf08 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MProtect.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; + +internal static partial class Interop +{ + internal static partial class Sys + { + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_MProtect", SetLastError = true)] + internal static partial int MProtect(IntPtr addr, ulong len, MemoryMappedProtections prot); + } +} From 05dbdafcefd2cfe4f520dabd6afb0c16944d8a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Thu, 9 Nov 2023 08:10:44 +0100 Subject: [PATCH 5/8] FB1 --- src/coreclr/nativeaot/Runtime/MiscHelpers.cpp | 6 +++--- .../Runtime/FrozenObjectHeapManager.Unix.cs | 5 +++-- .../Runtime/FrozenObjectHeapManager.Windows.cs | 10 +++++++++- .../src/Internal/Runtime/FrozenObjectHeapManager.cs | 13 +++++-------- .../src/System/Runtime/RuntimeImports.cs | 3 +++ .../src/System/Runtime/RuntimeImports.cs | 6 +++--- .../src/Interop/Unix/System.Native/Interop.MMap.cs | 7 +++++++ 7 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp index 56bb8ffd4ae942..b30678b38f2bc4 100644 --- a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp @@ -356,17 +356,17 @@ EXTERN_C NATIVEAOT_API int32_t __cdecl RhpGetCurrentThreadStackTrace(void* pOutp return RhpCalculateStackTraceWorker(pOutputBuffer, outputBufferLength, pAddressInCurrentFrame); } -COOP_PINVOKE_HELPER(void*, RhRegisterFrozenSegment, (void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize)) +EXTERN_C NATIVEAOT_API void* __cdecl RhRegisterFrozenSegment(void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize) { return RedhawkGCInterface::RegisterFrozenSegment(pSection, allocSize, commitSize, reservedSize); } -COOP_PINVOKE_HELPER(void, RhUpdateFrozenSegment, (void* pSegmentHandle, uint8_t* allocated, uint8_t* committed)) +EXTERN_C NATIVEAOT_API void __cdecl RhUpdateFrozenSegment(void* pSegmentHandle, uint8_t* allocated, uint8_t* committed) { RedhawkGCInterface::UpdateFrozenSegment((GcSegmentHandle)pSegmentHandle, allocated, committed); } -COOP_PINVOKE_HELPER(void, RhUnregisterFrozenSegment, (void* pSegmentHandle)) +EXTERN_C NATIVEAOT_API void __cdecl RhUnregisterFrozenSegment(void* pSegmentHandle) { RedhawkGCInterface::UnregisterFrozenSegment((GcSegmentHandle)pSegmentHandle); } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs index 5c1c20f42af67e..7ebc8867c5565e 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.Win32.SafeHandles; +using System.Diagnostics; namespace Internal.Runtime { @@ -15,7 +15,7 @@ internal unsafe partial class FrozenObjectHeapManager size, Interop.Sys.MemoryMappedProtections.PROT_NONE, Interop.Sys.MemoryMappedFlags.MAP_PRIVATE | Interop.Sys.MemoryMappedFlags.MAP_ANONYMOUS, - new SafeFileHandle(-1, false), + -1, 0); } @@ -32,6 +32,7 @@ internal unsafe partial class FrozenObjectHeapManager static void ClrVirtualFree(void* pBase, nuint size) { + Debug.Assert(size != 0); Interop.Sys.MUnmap((nint)pBase, size); } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs index 7d4f0ffe107af5..3927b46ae1ff3f 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; + namespace Internal.Runtime { internal unsafe partial class FrozenObjectHeapManager @@ -17,7 +19,13 @@ internal unsafe partial class FrozenObjectHeapManager static void ClrVirtualFree(void* pBase, nuint size) { - Interop.Kernel32.VirtualFree(pBase, size, Interop.Kernel32.MemOptions.MEM_RELEASE); + // We require the size parameter for Unix implementation sake. + // The Win32 API ignores this parameter because we must pass zero. + // If the caller passed zero, this is going to be broken on Unix + // so let's at least assert that. + Debug.Assert(size != 0); + + Interop.Kernel32.VirtualFree(pBase, 0, Interop.Kernel32.MemOptions.MEM_RELEASE); } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs index b87d0e59dc3a70..9a0489989f516b 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; using System.Runtime; using System.Runtime.CompilerServices; using System.Threading; @@ -13,11 +12,10 @@ namespace Internal.Runtime { internal unsafe partial class FrozenObjectHeapManager { - public static FrozenObjectHeapManager Instance = new FrozenObjectHeapManager(); + public static readonly FrozenObjectHeapManager Instance = new FrozenObjectHeapManager(); private readonly LowLevelLock m_Crst = new LowLevelLock(); private readonly LowLevelLock m_SegmentRegistrationCrst = new LowLevelLock(); - private ArrayBuilder m_FrozenSegments; private FrozenObjectSegment m_CurrentSegment; // Default size to reserve for a frozen segment @@ -80,7 +78,6 @@ internal unsafe partial class FrozenObjectHeapManager } m_CurrentSegment = new FrozenObjectSegment(newSegmentSize); - m_FrozenSegments.Add(m_CurrentSegment); // Try again obj = m_CurrentSegment.TryAllocateObject(type, objectSize); @@ -121,7 +118,7 @@ internal unsafe partial class FrozenObjectHeapManager private class FrozenObjectSegment { // Start of the reserved memory, the first object starts at "m_pStart + sizeof(ObjHeader)" (its pMT) - byte* m_pStart; + private byte* m_pStart; // NOTE: To handle potential race conditions, only m_[x]Registered fields should be accessed // externally as they guarantee that GC is aware of the current state of the segment. @@ -135,7 +132,7 @@ private class FrozenObjectSegment // Last known value of m_pCurrent that GC is aware of. // // m_pCurrentRegistered <= m_pCurrent - byte* m_pCurrentRegistered; + private byte* m_pCurrentRegistered; // Memory committed in the current segment // @@ -145,7 +142,7 @@ private class FrozenObjectSegment // Total memory reserved for the current segment public nuint m_Size; - IntPtr m_SegmentHandle; + private IntPtr m_SegmentHandle; public FrozenObjectSegment(nuint sizeHint) { @@ -176,7 +173,7 @@ public FrozenObjectSegment(nuint sizeHint) void* committedAlloc = ClrVirtualCommit(alloc, FOH_COMMIT_SIZE); if (committedAlloc == null) { - ClrVirtualFree(alloc, 0); + ClrVirtualFree(alloc, m_Size); throw new OutOfMemoryException(); } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index c29e4190f86ace..957badf100ed80 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -166,12 +166,15 @@ internal static void RhWaitForPendingFinalizers(bool allowReentrantWait) internal static extern long RhGetLastGCDuration(int generation); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })] internal static unsafe partial IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })] internal static unsafe partial void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })] internal static partial void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [MethodImpl(MethodImplOptions.InternalCall)] diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs index 195aa4693ae8c2..70d82828cd008e 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -40,13 +40,13 @@ internal static IntPtr RhHandleAlloc(object value, GCHandleType type) return h; } - [DllImport(RuntimeLibrary)] + [DllImport(RuntimeLibrary, CallingConvention = CallingConvention.Cdecl)] internal static extern unsafe IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); - [DllImport(RuntimeLibrary)] + [DllImport(RuntimeLibrary, CallingConvention = CallingConvention.Cdecl)] internal static extern unsafe void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); - [DllImport(RuntimeLibrary)] + [DllImport(RuntimeLibrary, CallingConvention = CallingConvention.Cdecl)] internal static extern void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [RuntimeImport(RuntimeLibrary, "RhpGetModuleSection")] diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MMap.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MMap.cs index 8797980a80c1a9..07a7e295fa7acb 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MMap.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MMap.cs @@ -33,5 +33,12 @@ internal static partial IntPtr MMap( IntPtr addr, ulong len, MemoryMappedProtections prot, MemoryMappedFlags flags, SafeFileHandle fd, long offset); + + // NOTE: Shim returns null pointer on failure, not non-null MAP_FAILED sentinel. + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_MMap", SetLastError = true)] + internal static partial IntPtr MMap( + IntPtr addr, ulong len, + MemoryMappedProtections prot, MemoryMappedFlags flags, + IntPtr fd, long offset); } } From 66fa94b898f88a484a5534196af50c91e6674644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Thu, 9 Nov 2023 08:33:15 +0100 Subject: [PATCH 6/8] FB2 --- src/coreclr/nativeaot/Runtime/gcrhenv.cpp | 1 - .../Runtime/FrozenObjectHeapManager.cs | 75 +++---------------- 2 files changed, 10 insertions(+), 66 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp index 4f7e62ba22e4c9..3b457682140aed 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp +++ b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp @@ -372,7 +372,6 @@ GcSegmentHandle RedhawkGCInterface::RegisterFrozenSegment(void * pSection, size_ { ASSERT(allocSize <= commitSize); ASSERT(commitSize <= reservedSize); - ASSERT(allocSize >= MIN_OBJECT_SIZE); #ifdef FEATURE_BASICFREEZE segment_info seginfo; diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs index 9a0489989f516b..7c9d69c89bb3d9 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs @@ -15,7 +15,6 @@ internal unsafe partial class FrozenObjectHeapManager public static readonly FrozenObjectHeapManager Instance = new FrozenObjectHeapManager(); private readonly LowLevelLock m_Crst = new LowLevelLock(); - private readonly LowLevelLock m_SegmentRegistrationCrst = new LowLevelLock(); private FrozenObjectSegment m_CurrentSegment; // Default size to reserve for a frozen segment @@ -32,9 +31,6 @@ internal unsafe partial class FrozenObjectHeapManager private object? TryAllocateObject(MethodTable* type, nuint objectSize) { HalfBakedObject* obj = null; - FrozenObjectSegment? curSeg = null; - byte* curSegmentCurrent = null; - nuint curSegSizeCommitted = 0; m_Crst.Acquire(); @@ -85,31 +81,12 @@ internal unsafe partial class FrozenObjectHeapManager // This time it's not expected to be null Debug.Assert(obj != null); } - - curSeg = m_CurrentSegment; - curSegSizeCommitted = curSeg.m_SizeCommitted; - curSegmentCurrent = curSeg.m_pCurrent; } // end of m_Crst lock finally { m_Crst.Release(); } - // Let GC know about the new segment or changes in it. - // We do it under a new lock because the main one (m_Crst) can be used by Profiler in a GC's thread - // and that might cause deadlocks since RegisterFrozenSegment may stuck on GC's lock. - m_SegmentRegistrationCrst.Acquire(); - try - { - curSeg.RegisterOrUpdate(curSegmentCurrent, curSegSizeCommitted); - } - finally - { - m_SegmentRegistrationCrst.Release(); - } - - //PublishFrozenObject(obj); - IntPtr result = (IntPtr)obj; return Unsafe.As(ref result); @@ -120,20 +97,12 @@ private class FrozenObjectSegment // Start of the reserved memory, the first object starts at "m_pStart + sizeof(ObjHeader)" (its pMT) private byte* m_pStart; - // NOTE: To handle potential race conditions, only m_[x]Registered fields should be accessed - // externally as they guarantee that GC is aware of the current state of the segment. - // Pointer to the end of the current segment, ready to be used as a pMT for a new object // meaning that "m_pCurrent - sizeof(ObjHeader)" is the actual start of the new object (header). // // m_pCurrent <= m_SizeCommitted public byte* m_pCurrent; - // Last known value of m_pCurrent that GC is aware of. - // - // m_pCurrentRegistered <= m_pCurrent - private byte* m_pCurrentRegistered; - // Memory committed in the current segment // // m_SizeCommitted <= m_pStart + FOH_SIZE_RESERVED @@ -170,49 +139,23 @@ public FrozenObjectSegment(nuint sizeHint) } // Commit a chunk in advance - void* committedAlloc = ClrVirtualCommit(alloc, FOH_COMMIT_SIZE); - if (committedAlloc == null) + m_pStart = (byte*)ClrVirtualCommit(alloc, FOH_COMMIT_SIZE); + if (m_pStart == null) { ClrVirtualFree(alloc, m_Size); throw new OutOfMemoryException(); } - m_pStart = (byte*)committedAlloc; m_pCurrent = m_pStart + sizeof(ObjHeader); - m_SizeCommitted = FOH_COMMIT_SIZE; - - // ClrVirtualAlloc is expected to be PageSize-aligned so we can expect - // DATA_ALIGNMENT alignment as well - // _ASSERT(IS_ALIGNED(committedAlloc, DATA_ALIGNMENT)); - } - public void RegisterOrUpdate(byte* current, nuint sizeCommited) - { - if (m_pCurrentRegistered == null) + m_SegmentHandle = RuntimeImports.RhRegisterFrozenSegment(m_pStart, (nuint)m_pCurrent - (nuint)m_pStart, FOH_COMMIT_SIZE, m_Size); + if (m_SegmentHandle == IntPtr.Zero) { - Debug.Assert(current >= m_pStart); - - // NOTE: RegisterFrozenSegment may take a GC lock inside. - m_SegmentHandle = RuntimeImports.RhRegisterFrozenSegment(m_pStart, (nuint)current - (nuint)m_pStart, sizeCommited, m_Size); - if (m_SegmentHandle == IntPtr.Zero) - { - throw new OutOfMemoryException(); - } - m_pCurrentRegistered = current; - } - else - { - if (current > m_pCurrentRegistered) - { - RuntimeImports.RhUpdateFrozenSegment( - m_SegmentHandle, current, m_pStart + sizeCommited); - m_pCurrentRegistered = current; - } - else - { - // Some other thread already advanced it. - } + ClrVirtualFree(alloc, m_Size); + throw new OutOfMemoryException(); } + + m_SizeCommitted = FOH_COMMIT_SIZE; } public HalfBakedObject* TryAllocateObject(MethodTable* type, nuint objectSize) @@ -253,6 +196,8 @@ public void RegisterOrUpdate(byte* current, nuint sizeCommited) m_pCurrent += objectSize; + RuntimeImports.RhUpdateFrozenSegment(m_SegmentHandle, m_pCurrent, m_pStart + m_SizeCommitted); + return obj; } } From 7854e4cef319491a0b7b7d8411c4fdf2eb126b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Fri, 10 Nov 2023 08:40:13 +0100 Subject: [PATCH 7/8] Update src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs Co-authored-by: Jan Kotas --- .../src/Internal/Runtime/FrozenObjectHeapManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs index 7c9d69c89bb3d9..4173098258ab4a 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs @@ -7,6 +7,7 @@ using System.Threading; using Debug = System.Diagnostics.Debug; +// Rewrite of src\coreclr\vm\frozenobjectheap.cpp in C# namespace Internal.Runtime { From f095349d66cb121abef5cfb286c788b426f1842c Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Fri, 10 Nov 2023 02:15:09 -0800 Subject: [PATCH 8/8] Update src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs --- .../src/Internal/Runtime/FrozenObjectHeapManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs index 4173098258ab4a..597341da8a173e 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs @@ -7,6 +7,7 @@ using System.Threading; using Debug = System.Diagnostics.Debug; + // Rewrite of src\coreclr\vm\frozenobjectheap.cpp in C# namespace Internal.Runtime