Notify the user about GPU reset exception

crash-handler-process/main.cpp

@@ -95,13 +95,12 @@ int main(int argc, char **argv)
 	}
 
 #endif
-	ProcessManager *pm = new ProcessManager();
-	pm->runWatcher();
 
-	if (pm->m_applicationCrashed)
-		pm->handleCrash(path);
+	{
+		auto pm = std::make_unique<ProcessManager>(path);
+		pm->run();
+	}
 
-	delete pm;
 	log_info << "=== Terminating CrashHandler ===" << std::endl;
 	logging_end();
 	return 0;

crash-handler-process/message.hpp

@@ -22,7 +22,14 @@
 #include <vector>
 #include <string>
 
-enum class Action : uint8_t { REGISTER = 0, UNREGISTER = 1, REGISTERMEMORYDUMP = 2 };
+enum class Action : uint8_t
+{
+	REGISTER = 0,
+	UNREGISTER = 1,
+	REGISTERMEMORYDUMP = 2,
+	CRASHWITHCODE = 3,
+	HANDLEOUTOFGPU = 4
+};
 
 class Message {
 public:

crash-handler-process/platforms/util-win.cpp

@@ -144,6 +144,29 @@ void Util::runTerminateWindow(bool &shouldRestart)
 	}
 }
 
+void Util::runOutOfGPUWindow(bool &shouldRestart)
+{
+	std::wstring title = from_utf8_to_utf16_wide(boost::locale::translate("Out of GPU memory").str().c_str());
+
+	auto message_translated1 = boost::locale::translate("The GPU is out of memory. Streamlabs Desktop will be closed!");
+	std::wstring message1 = from_utf8_to_utf16_wide(message_translated1.str().c_str());
+	auto message_translated2 = boost::locale::translate("Would you like to relaunch the application?");
+	std::wstring message2 = from_utf8_to_utf16_wide(message_translated2.str().c_str());
+
+	std::wstring message = message1 + L"\n\n" + message2;
+
+	int code = MessageBox(NULL, message.c_str(), title.c_str(), MB_YESNO | MB_SYSTEMMODAL);
+	switch (code) {
+	case IDYES: {
+		shouldRestart = true;
+		break;
+	}
+	case IDNO:
+	default:
+		break;
+	}
+}
+
 static thread_local std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
 
 std::string from_utf16_wide_to_utf8(const wchar_t *from, size_t length = -1)

crash-handler-process/process-manager.cpp

@@ -37,10 +37,9 @@ bool ThreadData::wait_or_stop()
 		return stop_event.wait_for(lck, std::chrono::milliseconds(50), [&flag = this->should_stop] { return flag; });
 }
 
-ProcessManager::ProcessManager()
+ProcessManager::ProcessManager(const std::wstring& appPath)
+	: m_appPath(appPath)
 {
-	m_applicationCrashed = false;
-	m_criticalCrash = false;
 }
 
 ProcessManager::~ProcessManager()
@@ -50,6 +49,12 @@ ProcessManager::~ProcessManager()
 	this->socket.reset();
 }
 
+void ProcessManager::run()
+{
+	runWatcher();
+	finalize();
+}
+
 void ProcessManager::runWatcher()
 {
 	this->watcher = new ThreadData();
@@ -60,6 +65,60 @@ void ProcessManager::runWatcher()
 		this->watcher->worker->join();
 }
 
+void ProcessManager::finalize()
+{
+	log_info << "Finalizing..." << std::endl;
+
+	switch (m_appExitStatus) {
+		case AppExitStatus::Crash: {
+			log_info << "Handling crash - processes state: " << std::endl;
+			for (auto &process : this->processes) {
+				log_info << "----" << std::endl;
+				if (process->isAlive()) {
+					log_info << "process.pid: " << process->getPID() << std::endl;
+				} else {
+					log_info << "process.pid: " << process->getPID() << " (not alive)" << std::endl;
+					if (process->isCritical())
+						m_criticalCrash = true;
+				}
+				log_info << "process.isCritical: " << process->isCritical() << std::endl;
+			}
+			log_info << "----" << std::endl;
+
+			bool shouldRestart = false;
+			if (m_criticalCrash) {
+				terminateAll();
+			} else {
+				log_info << "Terminate non critical processes" << std::endl;
+				terminateNonCritical();
+				// Blocking operation that will return once the user
+				// decides to terminate the application
+				Util::runTerminateWindow(shouldRestart);
+				log_info << "Send exit message" << std::endl;
+				this->sendExitMessage(true);
+			}
+
+			if (shouldRestart)
+				Util::restartApp(m_appPath);
+			break;
+		}
+		case AppExitStatus::OutOfGPU: {
+			log_info << "Out of GPU resources - notifying a user" << std::endl;
+			bool shouldRestart = false;
+			Util::runOutOfGPUWindow(shouldRestart);
+			terminateAll();
+			if (shouldRestart) {
+				Util::restartApp(m_appPath);
+			}
+			break;
+		}
+		default:
+			break;
+	}
+
+	log_info << "Finalized" << std::endl;
+}
+
 void ProcessManager::watcher_fnc()
 {
 	log_info << "Start Watcher" << std::endl;
@@ -97,6 +156,12 @@ void ProcessManager::watcher_fnc()
 				registerProcessMemoryDump(pid, eventName_Start, eventName_Fail, eventName_Success, dumpPath, dumpName);
 				break;
 			}
+			case Action::HANDLEOUTOFGPU: {
+				uint64_t errorCode = msg.readUInt64();
+				std::wstring errorCodeDesc = msg.readWstring();
+				handleOutOfGPU(errorCode, errorCodeDesc);
+				break;
+			}
 			default:
 				break;
 			}
@@ -127,7 +192,7 @@ void ProcessManager::monitor_fnc()
 					log_info << "process.isCritical: " << process->isCritical() << std::endl;
 
 					m_criticalCrash |= process->isCritical();
-					m_applicationCrashed = true;
+					m_appExitStatus = AppExitStatus::Crash;
 				} else if (last_responsive_check == 0) {
 					detectedUnresponsive |= process->isUnResponsive();
 				}
@@ -144,18 +209,18 @@ void ProcessManager::monitor_fnc()
 				unresponsiveMarked = true;
 			}
 		}
-		if (m_applicationCrashed)
+		if (m_appExitStatus != AppExitStatus::Success)
 			break;
 	}
 
-	if (m_applicationCrashed && !m_criticalCrash) {
+	if (m_appExitStatus == AppExitStatus::Crash && !m_criticalCrash) {
 		log_info << "Non critical crash detected. save it to app state file" << std::endl;
 		Util::updateAppState(Util::AppState::NoncriticallyDead);
 	}
 
 	this->watcher->send_stop();
 #ifdef __APPLE__
-	if (m_applicationCrashed) {
+	if (m_appExitStatus == AppExitStatus::Crash) {
 		std::vector<char> buffer;
 		buffer.push_back('-1');
 		this->socket->write(false, buffer);
@@ -233,37 +298,16 @@ void ProcessManager::registerProcessMemoryDump(uint32_t PID, const std::wstring
 	(*it)->startMemoryDumpMonitoring(eventName_Start, eventName_Fail, eventName_Success, dumpPath, dumpName);
 }
 
-void ProcessManager::handleCrash(std::wstring path)
+void ProcessManager::handleOutOfGPU(uint64_t errorCode, const std::wstring& errorCodeDesc)
 {
-	log_info << "Handling crash - processes state: " << std::endl;
-	for (auto &process : this->processes) {
-		log_info << "----" << std::endl;
-		if (process->isAlive()) {
-			log_info << "process.pid: " << process->getPID() << std::endl;
-		} else {
-			log_info << "process.pid: " << process->getPID() << " (not alive)" << std::endl;
-			if (process->isCritical())
-				m_criticalCrash = true;
-		}
-		log_info << "process.isCritical: " << process->isCritical() << std::endl;
-	}
-	log_info << "----" << std::endl;
-
-	bool shouldRestart = false;
-	if (m_criticalCrash) {
-		terminateAll();
-	} else {
-		log_info << "Terminate non critical processes" << std::endl;
-		terminateNonCritical();
-		// Blocking operation that will return once the user
-		// decides to terminate the application
-		Util::runTerminateWindow(shouldRestart);
-		log_info << "Send exit message" << std::endl;
-		this->sendExitMessage(true);
-	}
+	log_info << "Processing 'out of GPU resources'" << std::endl;
+
+	m_appExitStatus = AppExitStatus::OutOfGPU;
+	this->watcher->send_stop();
+	this->stopMonitoring();
+	this->sendExitMessage(false);
 
-	if (shouldRestart)
-		Util::restartApp(path);
+	log_info << "Processed 'out of GPU resources'" << std::endl;
 }
 
 void ProcessManager::sendExitMessage(bool appCrashed)

crash-handler-process/process-manager.hpp

@@ -38,23 +38,33 @@ struct ThreadData {
 
 class ProcessManager {
 public:
-	ProcessManager();
+	ProcessManager(const std::wstring& appPath);
 	~ProcessManager();
 
-	void runWatcher();
-	bool m_applicationCrashed;
-	bool m_criticalCrash;
-
-	void handleCrash(std::wstring path);
-	void sendExitMessage(bool appCrashed);
+	void run();
 
 private:
+	enum class AppExitStatus
+	{
+		Success = 0,
+		Crash,
+		OutOfGPU
+	};
+
+	std::wstring m_appPath;
+
+	AppExitStatus m_appExitStatus = AppExitStatus::Success;
+	bool m_criticalCrash = false;
+
 	ThreadData *watcher = nullptr;
 	ThreadData *monitor = nullptr;
 	std::vector<std::unique_ptr<Process>> processes;
 	std::mutex mtx;
 	std::unique_ptr<Socket> socket;
 
+	void runWatcher();
+	void finalize();
+
 	void watcher_fnc();
 	void monitor_fnc();
 
@@ -65,7 +75,10 @@ class ProcessManager {
 	void unregisterProcess(uint32_t PID);
 	void registerProcessMemoryDump(uint32_t PID, const std::wstring &eventName_Start, const std::wstring &eventName_Fail,
 				       const std::wstring &eventName_Success, const std::wstring &dumpPath, const std::wstring &dumpName);
+	void handleOutOfGPU(uint64_t errorCode, const std::wstring& errorCodeDesc);
 
 	void terminateAll(void);
 	void terminateNonCritical(void);
+
+	void sendExitMessage(bool appCrashed);
 };

crash-handler-process/util.hpp

@@ -24,6 +24,7 @@
 class Util {
 public:
 	static void runTerminateWindow(bool &shouldRestart);
+	static void runOutOfGPUWindow(bool &shouldRestart);
 	static void check_pid_file(std::string &pid_path);
 	static void write_pid_file(std::string &pid_path);
 	static std::string get_temp_directory();