Making crossplat::threadpool::shared_instance() safe to use in cpprest DLL (#611)

* When the C++ REST SDK is built as a DLL on Windows, and unloaded before the process exits, the crossplat::threadpool::shared_instance() is destroyed at DLL_PROCESS_DETACH, at which stage joining threads causes deadlock. Use the workaround provided by asio to terminate the threads in this case.

* Take 2. We can't just use TerminateThread indiscriminately because the thread may be holding e.g. the heap lock, so we need to ensure the thread is in a known state. Rather than reinvent the wheel, use asio's own thread class which already provides the necessary mechanism.

* Export the shared instance to allow cross-platform implementation of functionality (e.g. complete_after) that needs access to the io_service when using the asio-based default scheduler

* Use improved shutdown logic even when building as static lib, because users could include it in a DLL.

Author: garethsb

Release/include/pplx/threadpool.h

@@ -52,7 +52,7 @@ using java_local_ref = std::unique_ptr<typename std::remove_pointer<T>::type, ja
 class threadpool
 {
 public:
-    static threadpool& shared_instance();
+    _ASYNCRTIMP static threadpool& shared_instance();
     _ASYNCRTIMP static std::unique_ptr<threadpool> __cdecl construct(size_t num_threads);
 
     virtual ~threadpool() = default;

Release/src/pplx/threadpool.cpp

@@ -9,16 +9,7 @@
 #if !defined(CPPREST_EXCLUDE_WEBSOCKETS) || !defined(_WIN32)
 #include "pplx/threadpool.h"
 
-#if !defined(_WIN32)
-#define CPPREST_PTHREADS
-#endif
-
-#if defined(CPPREST_PTHREADS)
-#include <pthread.h>
-#else
-#include <thread>
-#endif
-
+#include <boost/asio/detail/thread.hpp>
 #include <vector>
 
 #if defined(__ANDROID__)
@@ -44,27 +35,14 @@ struct threadpool_impl final : crossplat::threadpool
         m_service.stop();
         for (auto iter = m_threads.begin(); iter != m_threads.end(); ++iter)
         {
-#if defined(CPPREST_PTHREADS)
-            pthread_t t = *iter;
-            void* res;
-            pthread_join(t, &res);
-#else
-            iter->join();
-#endif
+            (*iter)->join();
         }
     }
 
 private:
     void add_thread()
     {
-#ifdef CPPREST_PTHREADS
-        pthread_t t;
-        auto result = pthread_create(&t, nullptr, &thread_start, this);
-        if (result == 0)
-            m_threads.push_back(t);
-#else
-        m_threads.push_back(std::thread(&thread_start, this));
-#endif
+        m_threads.push_back(std::unique_ptr<boost::asio::detail::thread>(new boost::asio::detail::thread([&]{ thread_start(this); })));
     }
 
 #if defined(__ANDROID__)
@@ -89,11 +67,7 @@ struct threadpool_impl final : crossplat::threadpool
         return arg;
     }
 
-#if defined(CPPREST_PTHREADS)
-    std::vector<pthread_t> m_threads;
-#else
-    std::vector<std::thread> m_threads;
-#endif
+    std::vector<std::unique_ptr<boost::asio::detail::thread>> m_threads;
     boost::asio::io_service::work m_work;
 };
 }
@@ -133,6 +107,35 @@ threadpool& threadpool::shared_instance()
     return s_shared;
 }
 
+#elif defined(_WIN32)
+
+// if linked into a DLL, the threadpool shared instance will be destroyed at DLL_PROCESS_DETACH,
+// at which stage joining threads causes deadlock, hence this dance
+threadpool& threadpool::shared_instance()
+{
+    static bool terminate_threads = false;
+    static struct restore_terminate_threads
+    {
+        ~restore_terminate_threads()
+        {
+            boost::asio::detail::thread::set_terminate_threads(terminate_threads);
+        }
+    } destroyed_after;
+
+    static threadpool_impl s_shared(40);
+
+    static struct enforce_terminate_threads
+    {
+        ~enforce_terminate_threads()
+        {
+            terminate_threads = boost::asio::detail::thread::terminate_threads();
+            boost::asio::detail::thread::set_terminate_threads(true);
+        }
+    } destroyed_before;
+
+    return s_shared;
+}
+
 #else
 
 // initialize the static shared threadpool
@@ -156,4 +159,4 @@ std::unique_ptr<crossplat::threadpool> crossplat::threadpool::construct(size_t n
 {
     return std::unique_ptr<crossplat::threadpool>(new threadpool_impl(num_threads));
 }
-#endif
+#endif