Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -190,5 +190,13 @@ public static void PulseAll(object obj)

[LibraryImport(RuntimeHelpers.QCall, EntryPoint = "ObjectNative_GetMonitorLockContentionCount")]
private static partial long GetLockContentionCount();

/// <summary>
/// Gets the number of times there was a pause upon using <see cref="Monitor"/>'s wait so far.
/// </summary>
public static long WaitCount => GetWaitCount();

[LibraryImport(RuntimeHelpers.QCall, EntryPoint = "ObjectNative_GetMonitorWaitCount")]
private static partial long GetWaitCount();
}
}
14 changes: 14 additions & 0 deletions src/coreclr/classlibnative/bcltype/objectnative.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -375,3 +375,17 @@ extern "C" INT64 QCALLTYPE ObjectNative_GetMonitorLockContentionCount()
END_QCALL;
return result;
}

extern "C" INT64 QCALLTYPE ObjectNative_GetMonitorWaitCount()
{
QCALL_CONTRACT;

INT64 result = 0;

BEGIN_QCALL;

result = (INT64)Thread::GetTotalMonitorWaitCount();

END_QCALL;
return result;
}
1 change: 1 addition & 0 deletions src/coreclr/classlibnative/bcltype/objectnative.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,5 @@ class ObjectNative
};

extern "C" INT64 QCALLTYPE ObjectNative_GetMonitorLockContentionCount();
extern "C" INT64 QCALLTYPE ObjectNative_GetMonitorWaitCount();
#endif // _OBJECTNATIVE_H_
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ private static Waiter GetWaiterForCurrentThread()
return waiter;
}

private static long _waitCount;
internal static long WaitCount => _waitCount;

private readonly Lock _lock;
private Waiter? _waitersHead;
private Waiter? _waitersTail;
Expand Down Expand Up @@ -111,6 +114,7 @@ public unsafe bool Wait(int millisecondsTimeout)
bool success = false;
try
{
Interlocked.Increment(ref _waitCount);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I want the count to be increment only when the thread yields. Not sure if that's the case here.

success = waiter.ev.WaitOne(millisecondsTimeout);
}
finally
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,11 @@ public static void PulseAll(object obj)
/// </summary>
public static long LockContentionCount => Lock.ContentionCount;

/// <summary>
/// Gets the number of times there was a pause upon using <see cref="Monitor"/>'s wait so far.
/// </summary>
public static long WaitCount => Condition.WaitCount;

#endregion
}
}
1 change: 1 addition & 0 deletions src/coreclr/vm/qcallentrypoints.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ static const Entry s_QCall[] =
DllImportEntry(FileLoadException_GetMessageForHR)
DllImportEntry(Interlocked_MemoryBarrierProcessWide)
DllImportEntry(ObjectNative_GetMonitorLockContentionCount)
DllImportEntry(ObjectNative_GetMonitorWaitCount)
DllImportEntry(ReflectionInvocation_RunClassConstructor)
DllImportEntry(ReflectionInvocation_RunModuleConstructor)
DllImportEntry(ReflectionInvocation_CompileMethod)
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/vm/syncblk.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2851,6 +2851,7 @@ BOOL SyncBlock::Wait(INT32 timeOut)

OBJECTREF obj = m_Monitor.GetOwningObject();

Thread::IncrementMonitorWaitCount(pCurThread);
m_Monitor.IncrementTransientPrecious();

// While we are in this frame the thread is considered blocked on the
Expand Down
4 changes: 4 additions & 0 deletions src/coreclr/vm/threads.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ PTR_ThreadLocalModule ThreadLocalBlock::GetTLMIfExists(MethodTable* pMT)
BOOL Thread::s_fCleanFinalizedThread = FALSE;

UINT64 Thread::s_monitorLockContentionCountOverflow = 0;
UINT64 Thread::s_monitorWaitCountOverflow = 0;

CrstStatic g_DeadlockAwareCrst;

Expand Down Expand Up @@ -5316,6 +5317,9 @@ BOOL ThreadStore::RemoveThread(Thread *target)
InterlockedExchangeAdd64(
(LONGLONG *)&Thread::s_monitorLockContentionCountOverflow,
target->m_monitorLockContentionCount);
InterlockedExchangeAdd64(
(LONGLONG *)&Thread::s_monitorWaitCountOverflow,
target->m_monitorWaitCount);

_ASSERTE(s_pThreadStore->m_ThreadCount >= 0);
_ASSERTE(s_pThreadStore->m_BackgroundThreadCount >= 0);
Expand Down
20 changes: 20 additions & 0 deletions src/coreclr/vm/threads.h
Original file line number Diff line number Diff line change
Expand Up @@ -3426,7 +3426,9 @@ class Thread

private:
UINT32 m_monitorLockContentionCount;
UINT32 m_monitorWaitCount;
static UINT64 s_monitorLockContentionCountOverflow;
static UINT64 s_monitorWaitCountOverflow;

#ifndef DACCESS_COMPILE
private:
Expand Down Expand Up @@ -3495,6 +3497,24 @@ class Thread
WRAPPER_NO_CONTRACT;
return GetTotalCount(offsetof(Thread, m_monitorLockContentionCount), &s_monitorLockContentionCountOverflow);
}

static void IncrementMonitorWaitCount(Thread *pThread)
{
WRAPPER_NO_CONTRACT;
IncrementCount(pThread, offsetof(Thread, m_monitorWaitCount), &s_monitorWaitCountOverflow);
}

static UINT64 GetMonitorWaitCountOverflow()
{
WRAPPER_NO_CONTRACT;
return GetOverflowCount(&s_monitorWaitCountOverflow);
}

static UINT64 GetTotalMonitorWaitCount()
{
WRAPPER_NO_CONTRACT;
return GetTotalCount(offsetof(Thread, m_monitorWaitCount), &s_monitorWaitCountOverflow);
}
#endif // !DACCESS_COMPILE

public:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public static class Keywords
private PollingCounter? _workingSetCounter;
private PollingCounter? _threadPoolThreadCounter;
private IncrementingPollingCounter? _monitorContentionCounter;
private IncrementingPollingCounter? _monitorWaitCounter;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a very niche metric. I do not think it belongs to this default set of runtime perf counters.

Copy link
Contributor Author

@verdie-g verdie-g Nov 15, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it really more niche than the contention metric? In both case it blocks a thread and can cause scalability issue. Note that Monitor.Wait is used by synchronous I/O operations, ManualEventSlim, Task.Wait, ...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The interpretation of this counter depends heavily on how all Monitor.Wait are used by the app. For example - if this counter grows by 100 or 1000 per second, does the app have a scalability problem?

Some Monitor.Waits are fine and some can be scalability problems. I do not think you can tell in general. It is what makes it a bad candidate for on-by-default counter.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The interpretation of this counter depends heavily on how all Monitor.Wait are used by the app

Isn't that true for most runtime metrics? For example if I see an increase of gen 0 collections, I'll have to get more info about the app to determine if that's an issue.

If I see a spike of latency and I'm able to correlate it with a spike of Monitor.Wait that could point out a thread starvation. Of course that doesn't prove anything but it gives a lead, and it could be confirmed by collecting the event proposed in #94737.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Most default runtime metrics have a healthy range/rate that will hold for most apps. I think it will be hard to come up with a general healthy range/rate for this one.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not think it belongs to this default set of runtime perf counters

Btw are you challenging that new property completely or just its addition to the event source?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am primarily challenging adding this counter to the default set.

For exposing the counter as an API, we should understand what makes Monitor.Wait special that warrants adding the counter for it. Let me rephrase the justification for adding this counter: An app started to misbehave that lead to Monitor.Wait being called much more frequently than before. A counter that measures how often Monitor.Wait gets called can alert to the situation, thus we should add a counter for it. The problem with this justification is that it does not scale. There are many other APIs that will lead to thread pool starvation when called on threadpool threads a lot. It is not reasonable to add a counter for each of them.

We have multiple tools for diagnosing threadpool starvation. The most basic one is counter for threadpool threads. If this counter starts growing suddenly, it is a sign of threadpool starvation. There are other more advanced tools, like the ThreadAdjustmentReasonMap.Starvation event emitted by threadpool. If the current tools are not good enough, we should think about ways to make them work for any root cause of the threadpool starvation.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok! kouvel is not convinced either in #94264 (comment) so I'll probably close this PR.

private PollingCounter? _threadPoolQueueCounter;
private IncrementingPollingCounter? _completedItemsCounter;
private IncrementingPollingCounter? _allocRateCounter;
Expand Down Expand Up @@ -106,6 +107,7 @@ protected override void OnEventCommand(EventCommandEventArgs command)
_gen0BudgetCounter ??= new PollingCounter("gen-0-gc-budget", this, () => GC.GetGenerationBudget(0) / 1_000_000) { DisplayName = "Gen 0 GC Budget", DisplayUnits = "MB" };
_threadPoolThreadCounter ??= new PollingCounter("threadpool-thread-count", this, () => ThreadPool.ThreadCount) { DisplayName = "ThreadPool Thread Count" };
_monitorContentionCounter ??= new IncrementingPollingCounter("monitor-lock-contention-count", this, () => Monitor.LockContentionCount) { DisplayName = "Monitor Lock Contention Count", DisplayRateTimeScale = new TimeSpan(0, 0, 1) };
_monitorWaitCounter ??= new IncrementingPollingCounter("monitor-wait-count", this, () => Monitor.WaitCount) { DisplayName = "Monitor Wait Count", DisplayRateTimeScale = new TimeSpan(0, 0, 1) };
_threadPoolQueueCounter ??= new PollingCounter("threadpool-queue-length", this, () => ThreadPool.PendingWorkItemCount) { DisplayName = "ThreadPool Queue Length" };
_completedItemsCounter ??= new IncrementingPollingCounter("threadpool-completed-items-count", this, () => ThreadPool.CompletedWorkItemCount) { DisplayName = "ThreadPool Completed Work Item Count", DisplayRateTimeScale = new TimeSpan(0, 0, 1) };
_allocRateCounter ??= new IncrementingPollingCounter("alloc-rate", this, () => GC.GetTotalAllocatedBytes()) { DisplayName = "Allocation Rate", DisplayUnits = "B", DisplayRateTimeScale = new TimeSpan(0, 0, 1) };
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Threading/ref/System.Threading.cs
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,7 @@ public void Wait(System.Threading.CancellationToken cancellationToken) { }
public static partial class Monitor
{
public static long LockContentionCount { get { throw null; } }
public static long WaitCount { get { throw null; } }
public static void Enter(object obj) { }
public static void Enter(object obj, ref bool lockTaken) { }
public static void Exit(object obj) { }
Expand Down
8 changes: 8 additions & 0 deletions src/libraries/System.Threading/tests/MonitorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,14 @@ public static void Enter_HasToWait_LockContentionCountTest()
Assert.True(Monitor.LockContentionCount - initialLockContentionCount >= 2);
}

[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))]
public static void WaitTest_WaitCountTest()
{
long initialWaitCount = Monitor.WaitCount;
WaitTest();
Assert.True(Monitor.WaitCount - initialWaitCount >= 4);
}

[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))]
public static void ObjectHeaderSyncBlockTransitionTryEnterRaceTest()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,5 +149,10 @@ private static void ReliableEnterTimeout(object obj, int timeout, ref bool lockT

[MethodImplAttribute(MethodImplOptions.InternalCall)]
private static extern long Monitor_get_lock_contention_count();

public static long WaitCount => Monitor_get_wait_count() + Condition.WaitCount;

[MethodImplAttribute(MethodImplOptions.InternalCall)]
private static extern long Monitor_get_wait_count();
}
}
1 change: 1 addition & 0 deletions src/mono/mono/metadata/icall-def.h
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,7 @@ ICALL_TYPE(MONIT, "System.Threading.Monitor", MONIT_0)
HANDLES(MONIT_0, "Enter", ves_icall_System_Threading_Monitor_Monitor_Enter, void, 1, (MonoObject))
HANDLES(MONIT_1, "InternalExit", mono_monitor_exit_icall, void, 1, (MonoObject))
NOHANDLES(ICALL(MONIT_8, "Monitor_get_lock_contention_count", ves_icall_System_Threading_Monitor_Monitor_get_lock_contention_count))
NOHANDLES(ICALL(MONIT_8, "Monitor_get_wait_count", ves_icall_System_Threading_Monitor_Monitor_get_wait_count))
HANDLES(MONIT_2, "Monitor_pulse", ves_icall_System_Threading_Monitor_Monitor_pulse, void, 1, (MonoObject))
HANDLES(MONIT_3, "Monitor_pulse_all", ves_icall_System_Threading_Monitor_Monitor_pulse_all, void, 1, (MonoObject))
HANDLES(MONIT_7, "Monitor_wait", ves_icall_System_Threading_Monitor_Monitor_wait, MonoBoolean, 3, (MonoObject, guint32, MonoBoolean))
Expand Down
9 changes: 9 additions & 0 deletions src/mono/mono/metadata/monitor.c
Original file line number Diff line number Diff line change
Expand Up @@ -787,6 +787,7 @@ signal_monitor (gpointer mon_untyped)
}

static gint64 thread_contentions; /* for Monitor.LockContentionCount */
static gint64 thread_waits; /* for Monitor.WaitCount */

/* If allow_interruption==TRUE, the method will be interrupted if abort or suspend
* is requested. In this case it returns -1.
Expand Down Expand Up @@ -1340,6 +1341,8 @@ mono_monitor_wait (MonoObjectHandle obj_handle, guint32 ms, MonoBoolean allow_in

LOCK_DEBUG (g_message ("%s: (%d) Unlocked %p lock %p", __func__, id, obj, mon));

mono_atomic_inc_i64 (&thread_waits);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I want the count to be increment only when the thread yields. Not sure if that's the case here.


/* There's no race between unlocking mon and waiting for the
* event, because auto reset events are sticky, and this event
* is private to this thread. Therefore even if the event was
Expand Down Expand Up @@ -1438,3 +1441,9 @@ ves_icall_System_Threading_Monitor_Monitor_get_lock_contention_count (void)
{
return thread_contentions;
}

gint64
ves_icall_System_Threading_Monitor_Monitor_get_wait_count (void)
{
return thread_waits;
}
4 changes: 4 additions & 0 deletions src/mono/mono/metadata/monitor.h
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ ICALL_EXPORT
gint64
ves_icall_System_Threading_Monitor_Monitor_get_lock_contention_count (void);

ICALL_EXPORT
gint64
ves_icall_System_Threading_Monitor_Monitor_get_wait_count (void);

#ifdef HOST_WASM
void
mono_set_string_interned_internal (MonoObject* obj);
Expand Down
1 change: 1 addition & 0 deletions src/tests/tracing/eventcounter/runtimecounters.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public RuntimeCounterListener()
{ "gen-2-gc-count", false },
{ "threadpool-thread-count", false },
{ "monitor-lock-contention-count", false },
{ "monitor-wait-count", false },
{ "threadpool-queue-length", false },
{ "threadpool-completed-items-count", false },
{ "alloc-rate", false },
Expand Down