diff --git a/crash-handler-process/main.cpp b/crash-handler-process/main.cpp index 6e15d48..32dbf26 100644 --- a/crash-handler-process/main.cpp +++ b/crash-handler-process/main.cpp @@ -95,13 +95,12 @@ int main(int argc, char **argv) } #endif - ProcessManager *pm = new ProcessManager(); - pm->runWatcher(); - if (pm->m_applicationCrashed) - pm->handleCrash(path); + { + auto pm = std::make_unique(path); + pm->run(); + } - delete pm; log_info << "=== Terminating CrashHandler ===" << std::endl; logging_end(); return 0; diff --git a/crash-handler-process/message.hpp b/crash-handler-process/message.hpp index db3c487..e8b783f 100644 --- a/crash-handler-process/message.hpp +++ b/crash-handler-process/message.hpp @@ -22,7 +22,14 @@ #include #include -enum class Action : uint8_t { REGISTER = 0, UNREGISTER = 1, REGISTERMEMORYDUMP = 2 }; +enum class Action : uint8_t +{ + REGISTER = 0, + UNREGISTER = 1, + REGISTERMEMORYDUMP = 2, + CRASHWITHCODE = 3, + HANDLEOUTOFGPU = 4 +}; class Message { public: diff --git a/crash-handler-process/platforms/util-win.cpp b/crash-handler-process/platforms/util-win.cpp index b65df29..27ff36e 100644 --- a/crash-handler-process/platforms/util-win.cpp +++ b/crash-handler-process/platforms/util-win.cpp @@ -144,6 +144,29 @@ void Util::runTerminateWindow(bool &shouldRestart) } } +void Util::runOutOfGPUWindow(bool &shouldRestart) +{ + std::wstring title = from_utf8_to_utf16_wide(boost::locale::translate("Out of GPU memory").str().c_str()); + + auto message_translated1 = boost::locale::translate("The GPU is out of memory. Streamlabs Desktop will be closed!"); + std::wstring message1 = from_utf8_to_utf16_wide(message_translated1.str().c_str()); + auto message_translated2 = boost::locale::translate("Would you like to relaunch the application?"); + std::wstring message2 = from_utf8_to_utf16_wide(message_translated2.str().c_str()); + + std::wstring message = message1 + L"\n\n" + message2; + + int code = MessageBox(NULL, message.c_str(), title.c_str(), MB_YESNO | MB_SYSTEMMODAL); + switch (code) { + case IDYES: { + shouldRestart = true; + break; + } + case IDNO: + default: + break; + } +} + static thread_local std::wstring_convert> converter; std::string from_utf16_wide_to_utf8(const wchar_t *from, size_t length = -1) diff --git a/crash-handler-process/process-manager.cpp b/crash-handler-process/process-manager.cpp index d406593..cbb17eb 100644 --- a/crash-handler-process/process-manager.cpp +++ b/crash-handler-process/process-manager.cpp @@ -37,10 +37,9 @@ bool ThreadData::wait_or_stop() return stop_event.wait_for(lck, std::chrono::milliseconds(50), [&flag = this->should_stop] { return flag; }); } -ProcessManager::ProcessManager() +ProcessManager::ProcessManager(const std::wstring& appPath) + : m_appPath(appPath) { - m_applicationCrashed = false; - m_criticalCrash = false; } ProcessManager::~ProcessManager() @@ -50,6 +49,12 @@ ProcessManager::~ProcessManager() this->socket.reset(); } +void ProcessManager::run() +{ + runWatcher(); + finalize(); +} + void ProcessManager::runWatcher() { this->watcher = new ThreadData(); @@ -60,6 +65,60 @@ void ProcessManager::runWatcher() this->watcher->worker->join(); } +void ProcessManager::finalize() +{ + log_info << "Finalizing..." << std::endl; + + switch (m_appExitStatus) { + case AppExitStatus::Crash: { + log_info << "Handling crash - processes state: " << std::endl; + for (auto &process : this->processes) { + log_info << "----" << std::endl; + if (process->isAlive()) { + log_info << "process.pid: " << process->getPID() << std::endl; + } else { + log_info << "process.pid: " << process->getPID() << " (not alive)" << std::endl; + if (process->isCritical()) + m_criticalCrash = true; + } + log_info << "process.isCritical: " << process->isCritical() << std::endl; + } + log_info << "----" << std::endl; + + bool shouldRestart = false; + if (m_criticalCrash) { + terminateAll(); + } else { + log_info << "Terminate non critical processes" << std::endl; + terminateNonCritical(); + // Blocking operation that will return once the user + // decides to terminate the application + Util::runTerminateWindow(shouldRestart); + log_info << "Send exit message" << std::endl; + this->sendExitMessage(true); + } + + if (shouldRestart) + Util::restartApp(m_appPath); + break; + } + case AppExitStatus::OutOfGPU: { + log_info << "Out of GPU resources - notifying a user" << std::endl; + bool shouldRestart = false; + Util::runOutOfGPUWindow(shouldRestart); + terminateAll(); + if (shouldRestart) { + Util::restartApp(m_appPath); + } + break; + } + default: + break; + } + + log_info << "Finalized" << std::endl; +} + void ProcessManager::watcher_fnc() { log_info << "Start Watcher" << std::endl; @@ -97,6 +156,12 @@ void ProcessManager::watcher_fnc() registerProcessMemoryDump(pid, eventName_Start, eventName_Fail, eventName_Success, dumpPath, dumpName); break; } + case Action::HANDLEOUTOFGPU: { + uint64_t errorCode = msg.readUInt64(); + std::wstring errorCodeDesc = msg.readWstring(); + handleOutOfGPU(errorCode, errorCodeDesc); + break; + } default: break; } @@ -127,7 +192,7 @@ void ProcessManager::monitor_fnc() log_info << "process.isCritical: " << process->isCritical() << std::endl; m_criticalCrash |= process->isCritical(); - m_applicationCrashed = true; + m_appExitStatus = AppExitStatus::Crash; } else if (last_responsive_check == 0) { detectedUnresponsive |= process->isUnResponsive(); } @@ -144,18 +209,18 @@ void ProcessManager::monitor_fnc() unresponsiveMarked = true; } } - if (m_applicationCrashed) + if (m_appExitStatus != AppExitStatus::Success) break; } - if (m_applicationCrashed && !m_criticalCrash) { + if (m_appExitStatus == AppExitStatus::Crash && !m_criticalCrash) { log_info << "Non critical crash detected. save it to app state file" << std::endl; Util::updateAppState(Util::AppState::NoncriticallyDead); } this->watcher->send_stop(); #ifdef __APPLE__ - if (m_applicationCrashed) { + if (m_appExitStatus == AppExitStatus::Crash) { std::vector buffer; buffer.push_back('-1'); this->socket->write(false, buffer); @@ -233,37 +298,16 @@ void ProcessManager::registerProcessMemoryDump(uint32_t PID, const std::wstring (*it)->startMemoryDumpMonitoring(eventName_Start, eventName_Fail, eventName_Success, dumpPath, dumpName); } -void ProcessManager::handleCrash(std::wstring path) +void ProcessManager::handleOutOfGPU(uint64_t errorCode, const std::wstring& errorCodeDesc) { - log_info << "Handling crash - processes state: " << std::endl; - for (auto &process : this->processes) { - log_info << "----" << std::endl; - if (process->isAlive()) { - log_info << "process.pid: " << process->getPID() << std::endl; - } else { - log_info << "process.pid: " << process->getPID() << " (not alive)" << std::endl; - if (process->isCritical()) - m_criticalCrash = true; - } - log_info << "process.isCritical: " << process->isCritical() << std::endl; - } - log_info << "----" << std::endl; - - bool shouldRestart = false; - if (m_criticalCrash) { - terminateAll(); - } else { - log_info << "Terminate non critical processes" << std::endl; - terminateNonCritical(); - // Blocking operation that will return once the user - // decides to terminate the application - Util::runTerminateWindow(shouldRestart); - log_info << "Send exit message" << std::endl; - this->sendExitMessage(true); - } + log_info << "Processing 'out of GPU resources'" << std::endl; + + m_appExitStatus = AppExitStatus::OutOfGPU; + this->watcher->send_stop(); + this->stopMonitoring(); + this->sendExitMessage(false); - if (shouldRestart) - Util::restartApp(path); + log_info << "Processed 'out of GPU resources'" << std::endl; } void ProcessManager::sendExitMessage(bool appCrashed) diff --git a/crash-handler-process/process-manager.hpp b/crash-handler-process/process-manager.hpp index 78cd072..a6ff8f5 100644 --- a/crash-handler-process/process-manager.hpp +++ b/crash-handler-process/process-manager.hpp @@ -38,23 +38,33 @@ struct ThreadData { class ProcessManager { public: - ProcessManager(); + ProcessManager(const std::wstring& appPath); ~ProcessManager(); - void runWatcher(); - bool m_applicationCrashed; - bool m_criticalCrash; - - void handleCrash(std::wstring path); - void sendExitMessage(bool appCrashed); + void run(); private: + enum class AppExitStatus + { + Success = 0, + Crash, + OutOfGPU + }; + + std::wstring m_appPath; + + AppExitStatus m_appExitStatus = AppExitStatus::Success; + bool m_criticalCrash = false; + ThreadData *watcher = nullptr; ThreadData *monitor = nullptr; std::vector> processes; std::mutex mtx; std::unique_ptr socket; + void runWatcher(); + void finalize(); + void watcher_fnc(); void monitor_fnc(); @@ -65,7 +75,10 @@ class ProcessManager { void unregisterProcess(uint32_t PID); void registerProcessMemoryDump(uint32_t PID, const std::wstring &eventName_Start, const std::wstring &eventName_Fail, const std::wstring &eventName_Success, const std::wstring &dumpPath, const std::wstring &dumpName); + void handleOutOfGPU(uint64_t errorCode, const std::wstring& errorCodeDesc); void terminateAll(void); void terminateNonCritical(void); + + void sendExitMessage(bool appCrashed); }; \ No newline at end of file diff --git a/crash-handler-process/util.hpp b/crash-handler-process/util.hpp index 730dced..a6816c4 100644 --- a/crash-handler-process/util.hpp +++ b/crash-handler-process/util.hpp @@ -24,6 +24,7 @@ class Util { public: static void runTerminateWindow(bool &shouldRestart); + static void runOutOfGPUWindow(bool &shouldRestart); static void check_pid_file(std::string &pid_path); static void write_pid_file(std::string &pid_path); static std::string get_temp_directory();