From c18d07d6f8c76ea8d6b8b76d78473ac1d043783d Mon Sep 17 00:00:00 2001 From: agent Date: Tue, 27 Oct 2026 23:28:11 +0300 Subject: [PATCH 1/2] Diagnostics: add hang detection and reporting HangWatchdog watches UI phase timestamps and writes a report when a non-idle phase exceeds its threshold. It records the phase, handled input count and open scope trail, samples every thread through StackTrace, and classifies the UI as blocked, busy or waiting repeatedly. Reports support follow-ups, recovery, custom contributors and retention. The scope trail is private to Diagnostics. The default report directory is HOME//hangs, overridable with MODULES_HANG_DIR, so the module needs no AppStorage dependency. --- CMakeLists.txt | 1 + Diagnostics/CMakeLists.txt | 16 ++ Diagnostics/private/HangWatchdog.cpp | 347 +++++++++++++++++++++++++++ Diagnostics/private/UiTrail.cpp | 67 ++++++ Diagnostics/private/UiTrail.hpp | 19 ++ Diagnostics/public/HangWatchdog.hpp | 120 +++++++++ 6 files changed, 570 insertions(+) create mode 100644 Diagnostics/CMakeLists.txt create mode 100644 Diagnostics/private/HangWatchdog.cpp create mode 100644 Diagnostics/private/UiTrail.cpp create mode 100644 Diagnostics/private/UiTrail.hpp create mode 100644 Diagnostics/public/HangWatchdog.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index db5b4851..0465385d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ add_subdirectory(Threading) add_subdirectory(Containers) add_subdirectory(Math) add_subdirectory(Callstack) +add_subdirectory(Diagnostics) add_subdirectory(Allocators) add_subdirectory(Strings) add_subdirectory(Language) diff --git a/Diagnostics/CMakeLists.txt b/Diagnostics/CMakeLists.txt new file mode 100644 index 00000000..071faecc --- /dev/null +++ b/Diagnostics/CMakeLists.txt @@ -0,0 +1,16 @@ +project(Diagnostics) + +### ---------------------- Static Library --------------------- ### +file(GLOB SOURCES "./private/*.cpp") +file(GLOB HEADERS "./public/*.hpp" "./private/*.hpp") +add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) +target_include_directories(${PROJECT_NAME} PUBLIC ./public/ PRIVATE ./private/) +target_link_libraries(${PROJECT_NAME} PUBLIC Modules Callstack) +find_package(Threads REQUIRED) +target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads) + +### -------------------------- Tests -------------------------- ### +file(GLOB TEST_SOURCES "./tests/*.cpp") +add_executable(Tests${PROJECT_NAME} ${TEST_SOURCES}) +target_link_libraries(Tests${PROJECT_NAME} ${PROJECT_NAME} UnitTest++) +add_test(NAME Tests${PROJECT_NAME} COMMAND Tests${PROJECT_NAME}) diff --git a/Diagnostics/private/HangWatchdog.cpp b/Diagnostics/private/HangWatchdog.cpp new file mode 100644 index 00000000..1060915b --- /dev/null +++ b/Diagnostics/private/HangWatchdog.cpp @@ -0,0 +1,347 @@ +#include "HangWatchdog.hpp" + +#include "UiTrail.hpp" +#include "StackTrace.hpp" + +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +namespace tp { + + namespace { + int64_t nowNs() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + + std::string timestamp(const char* pattern) { + const std::time_t now = std::time(nullptr); + char text[64]; + std::strftime(text, sizeof(text), pattern, std::localtime(&now)); + return text; + } + + void writeFile(const std::filesystem::path& path, const std::string& text, std::ios::openmode mode) { + std::ofstream file(path, mode); + file << text; + } + + constexpr double kCheckIntervalMs = 100; + } + + HangWatchdog::HangWatchdog(HangSettings settings) : + mSettings(std::move(settings)) { + mSettings.directory = directoryFor(mSettings.directory.empty() ? defaultHangsDirectory() : mSettings.directory); + if (!mSettings.clockNs) mSettings.clockNs = nowNs; + mPhaseStartNs.store(now()); + allowDebuggerAttach(); + StackTrace::prepareThreadSampling(); + mUiTid = StackTrace::callingThreadId(); + UiTrail::trackCallingThread(); + pruneOldReports(); + mThread = std::thread([this]() { watch(); }); + } + + HangWatchdog::~HangWatchdog() { + { + std::lock_guard lock(mStopMutex); + mStopping = true; + } + mStopped.notify_all(); + mThread.join(); + UiTrail::stopTracking(); + } + + std::string HangWatchdog::formatThreads(const std::vector& threads, int uiTid) { + std::ostringstream text; + for (const ThreadSample& thread : threads) { + text << (thread.tid == uiTid ? "UI THREAD " : "thread ") << thread.tid << " \"" << thread.name << "\" state " + << thread.state; + if (!thread.waitChannel.empty() && thread.waitChannel != "0") text << " waiting in " << thread.waitChannel; + text << "\n"; + if (!thread.answered) text << " (did not answer the sample signal)\n"; + for (uintptr_t frame : thread.frames) + text << " " << StackTrace::describe(frame) << "\n"; + text << "\n"; + } + return text.str(); + } + + void HangWatchdog::enterPhase(UiPhase phase) { + mPhaseStartNs.store(now(), std::memory_order_relaxed); + mPhaseInputEvents.store(mInputEvents.load(std::memory_order_relaxed), std::memory_order_relaxed); + mPhase.store(phase, std::memory_order_relaxed); + mBeat.fetch_add(1, std::memory_order_release); + } + + void HangWatchdog::noteInputEvent() { mInputEvents.fetch_add(1, std::memory_order_relaxed); } + + int64_t HangWatchdog::now() const { return mSettings.clockNs(); } + + void HangWatchdog::addContributor(Contributor contributor) { + std::lock_guard lock(mContributorsMutex); + mContributors.push_back(std::move(contributor)); + } + + std::filesystem::path HangWatchdog::defaultHangsDirectory() { + const char* home = std::getenv("HOME"); + const std::filesystem::path root = home && *home ? std::filesystem::path(home) : std::filesystem::current_path(); + return root / processName() / "hangs"; + } + + std::filesystem::path HangWatchdog::directoryFor(const std::filesystem::path& applicationHangsDirectory) { + if (const char* configured = std::getenv("MODULES_HANG_DIR"); configured && *configured) return configured; + return applicationHangsDirectory; + } + + std::string HangWatchdog::processName() { +#if defined(__linux__) + std::ifstream comm("/proc/self/comm"); + std::string name; + if (std::getline(comm, name) && !name.empty()) return name; +#endif + return "app"; + } + + void HangWatchdog::allowDebuggerAttach() { +#if defined(__linux__) && defined(PR_SET_PTRACER) + prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif + } + + const char* HangWatchdog::phaseName(UiPhase phase) { + switch (phase) { + case UiPhase::Idle: return "idle"; + case UiPhase::Input: return "processing input"; + case UiPhase::Draw: return "drawing"; + case UiPhase::Present: return "presenting"; + } + return "unknown"; + } + + void HangWatchdog::watch() { + Stall stall; + int64_t previous = now(); + std::unique_lock lock(mStopMutex); + while (!mStopping) { + mStopped.wait_for(lock, std::chrono::duration(kCheckIntervalMs), [this]() { + return mStopping; + }); + if (mStopping) return; + lock.unlock(); + const int64_t current = now(); + discountProcessPause(current, previous); + inspect(stall, current); + previous = current; + mInspections.fetch_add(1); + lock.lock(); + } + } + + void HangWatchdog::discountProcessPause(int64_t now, int64_t previous) { + const double sinceLastCheckMs = (double) (now - previous) / 1.0e6; + const double pausedMs = sinceLastCheckMs - kCheckIntervalMs; + if (pausedMs < mSettings.processPauseMs) return; + + const uint64_t beat = mBeat.load(std::memory_order_acquire); + if (beat != mPausedBeat) { + mPausedBeat = beat; + mPausedNs = 0; + } + mPausedNs += (int64_t) (pausedMs * 1.0e6); + } + + double HangWatchdog::stalledMs(int64_t now) const { + const bool pausedThisPhase = mPausedBeat == mBeat.load(std::memory_order_acquire); + const int64_t paused = pausedThisPhase ? mPausedNs : 0; + return (double) (now - mPhaseStartNs.load(std::memory_order_relaxed) - paused) / 1.0e6; + } + + uint64_t HangWatchdog::inputEventsInPhase() const { + return mInputEvents.load(std::memory_order_relaxed) - mPhaseInputEvents.load(std::memory_order_relaxed); + } + + void HangWatchdog::inspect(Stall& stall, int64_t now) { + const uint64_t beat = mBeat.load(std::memory_order_acquire); + const UiPhase phase = mPhase.load(std::memory_order_relaxed); + const double stalled = stalledMs(now); + + if (!stall.report.empty() && stall.beat != beat) { + finishRecovered(stall, stalled); + stall = {}; + } + if (phase == UiPhase::Idle || stalled < mSettings.thresholdMs) return; + + if (stall.report.empty()) { + stall.beat = beat; + stall.phase = phase; + beginReport(stall, stalled); + return; + } + if (stall.followUps < mSettings.followUpMs.size() && stalled >= mSettings.followUpMs[stall.followUps]) + appendFollowUp(stall, stalled); + } + + std::string HangWatchdog::uiSnapshot(double stalled, UiPhase phase) const { + std::ostringstream text; + text << "ui thread " << phaseName(phase) << " for " << (int64_t) stalled << " ms\n"; + text << "input events handled during this phase " << inputEventsInPhase() << "\n"; + text << "scope trail (outermost first):\n"; + const std::vector names = UiTrail::snapshot(); + if (names.empty()) text << " (no profiled scope open)\n"; + for (size_t index = 0; index < names.size(); index++) + text << " " << std::string(index * 2, ' ') << names[index] << "\n"; + return text.str(); + } + + std::filesystem::path HangWatchdog::newReportPath() const { + const std::string base = mSettings.application + "-" + timestamp("%Y-%m-%d_%H-%M-%S") + "-build" + mSettings.build; + std::filesystem::path path = mSettings.directory / base; + for (int suffix = 2; std::filesystem::exists(path); suffix++) + path = mSettings.directory / (base + "-" + std::to_string(suffix)); + return path; + } + + std::string HangWatchdog::reportHeader() const { + std::ostringstream text; + text << "application " << mSettings.application << "\n"; + text << "build " << mSettings.build << "\n"; + text << "detected " << timestamp("%Y-%m-%d %H:%M:%S") << "\n"; + text << "threshold " << (int64_t) mSettings.thresholdMs << " ms\n"; + text << "debugger attached " << (StackTrace::isDebuggerAttached() ? "yes" : "no") << "\n\n"; + return text.str(); + } + + void HangWatchdog::beginReport(Stall& stall, double stalled) { + std::error_code error; + std::filesystem::create_directories(mSettings.directory, error); + stall.report = newReportPath(); + std::filesystem::create_directories(stall.report, error); + if (error) return; + + writeFile( + stall.report / "summary.txt", reportHeader() + uiSnapshot(stalled, stall.phase), std::ios::out | std::ios::trunc + ); + writeContributions(stall.report); + mReportsWritten.fetch_add(1); + + const Investigation investigation = investigate(); + writeFile( + stall.report / "summary.txt", "\nverdict: " + investigation.verdict + "\n", std::ios::out | std::ios::app + ); + if (!investigation.threads.empty()) + writeFile(stall.report / "threads.txt", investigation.threads, std::ios::out | std::ios::trunc); + pruneOldReports(); + } + + void HangWatchdog::writeContributions(const std::filesystem::path& report) { + std::vector contributors; + { + std::lock_guard lock(mContributorsMutex); + contributors = mContributors; + } + for (const Contributor& contributor : contributors) + for (const HangReportSection& section : contributor()) + writeFile(report / section.fileName, section.text, std::ios::out | std::ios::trunc); + } + + void HangWatchdog::appendFollowUp(Stall& stall, double stalled) { + stall.followUps++; + const std::string header = "\nstill stuck at " + timestamp("%H:%M:%S") + "\n"; + writeFile(stall.report / "summary.txt", header + uiSnapshot(stalled, stall.phase), std::ios::out | std::ios::app); + + const Investigation investigation = investigate(); + writeFile(stall.report / "summary.txt", "verdict: " + investigation.verdict + "\n", std::ios::out | std::ios::app); + if (!investigation.threads.empty()) + writeFile( + stall.report / "threads.txt", + "\n===== follow-up at " + timestamp("%H:%M:%S") + " =====\n" + investigation.threads, + std::ios::out | std::ios::app + ); + } + + void HangWatchdog::finishRecovered(Stall& stall, double stalled) { + (void) stalled; + writeFile( + stall.report / "summary.txt", "\nrecovered at " + timestamp("%H:%M:%S") + "\n", std::ios::out | std::ios::app + ); + } + + HangWatchdog::Investigation HangWatchdog::investigate() const { + if (StackTrace::isDebuggerAttached()) + return { "not sampled: a debugger is attached, its thread view shows where the ui thread is", "" }; + if (!StackTrace::canSampleThreads()) return { "not sampled: thread stacks are not available on this platform", "" }; + + std::vector> uiSamples; + std::string uiState; + std::string uiWait; + std::string threads = sampleStacks(uiSamples, uiState, uiWait); + std::string verdict = stackVerdict(uiSamples, uiState, uiWait); + if (inputEventsInPhase() > 0) + verdict += + "; " + std::to_string(inputEventsInPhase()) + " input events were handled without leaving the input phase"; + return { verdict, threads }; + } + + std::string HangWatchdog::sampleStacks( + std::vector>& uiSamples, std::string& uiState, std::string& uiWait + ) const { + std::ostringstream text; + for (int sample = 0; sample < std::max(1, mSettings.stackSamples); sample++) { + if (sample > 0) + std::this_thread::sleep_for(std::chrono::duration(mSettings.stackSampleIntervalMs)); + const std::vector threads = StackTrace::sampleAllThreads(); + for (const ThreadSample& thread : threads) + if (thread.tid == mUiTid) { + uiSamples.push_back(thread.frames); + uiState = thread.state; + uiWait = thread.waitChannel; + } + text << "===== sample " << sample + 1 << " =====\n" << formatThreads(threads, mUiTid); + } + return text.str(); + } + + std::string HangWatchdog::stackVerdict( + const std::vector>& uiSamples, const std::string& state, const std::string& waitChannel + ) { + if (uiSamples.empty()) return "ui thread not found"; + const bool sameStack = + std::all_of(uiSamples.begin(), uiSamples.end(), [&](const auto& frames) { return frames == uiSamples.front(); }); + const bool sleeping = !state.empty() && (state[0] == 'S' || state[0] == 'D'); + if (sameStack && sleeping) { + std::string text = "blocked: the ui thread sleeps at the same place in every sample"; + if (!waitChannel.empty() && waitChannel != "0") text += " (kernel wait " + waitChannel + ")"; + return text; + } + if (!sleeping) return "busy: the ui thread is running, check the changing frames in threads.txt"; + return "waiting repeatedly: the ui thread sleeps but at different places, likely polling or lock contention"; + } + + std::vector HangWatchdog::reports() const { + std::vector found; + std::error_code error; + for (const auto& entry : std::filesystem::directory_iterator(mSettings.directory, error)) + if (entry.is_directory()) found.push_back(entry.path()); + std::sort(found.begin(), found.end(), [](const auto& a, const auto& b) { + return std::filesystem::last_write_time(a) < std::filesystem::last_write_time(b); + }); + return found; + } + + void HangWatchdog::pruneOldReports() const { + const std::vector found = reports(); + if (found.size() <= mSettings.keptReports) return; + std::error_code error; + for (size_t index = 0; index + mSettings.keptReports < found.size(); index++) + std::filesystem::remove_all(found[index], error); + } +} diff --git a/Diagnostics/private/UiTrail.cpp b/Diagnostics/private/UiTrail.cpp new file mode 100644 index 00000000..b3eee3df --- /dev/null +++ b/Diagnostics/private/UiTrail.cpp @@ -0,0 +1,67 @@ +#include "UiTrail.hpp" + +#include +#include +#include + +namespace tp { + + namespace { + struct Trail { + std::atomic names[UiTrail::kDepth] = {}; + std::atomic depth{ 0 }; + std::atomic overflow{ 0 }; + }; + + Trail& trail() { + static Trail shared; + return shared; + } + + std::atomic& trackedThread() { + static std::atomic tracked{}; + return tracked; + } + } + + void UiTrail::push(const char* name) { + if (!isTracked()) return; + Trail& shared = trail(); + const int depth = shared.depth.load(std::memory_order_relaxed); + if (depth < kDepth) shared.names[depth].store(name, std::memory_order_relaxed); + else shared.overflow.fetch_add(1, std::memory_order_relaxed); + shared.depth.store(depth + 1, std::memory_order_release); + } + + void UiTrail::pop() { + if (!isTracked()) return; + Trail& shared = trail(); + const int depth = shared.depth.load(std::memory_order_relaxed); + if (depth <= 0) return; + if (depth > kDepth) shared.overflow.fetch_sub(1, std::memory_order_relaxed); + shared.depth.store(depth - 1, std::memory_order_release); + } + + bool UiTrail::isTracked() { return trackedThread().load(std::memory_order_relaxed) == std::this_thread::get_id(); } + + void UiTrail::trackCallingThread() { + trail().depth.store(0); + trail().overflow.store(0); + trackedThread().store(std::this_thread::get_id()); + } + + void UiTrail::stopTracking() { trackedThread().store(std::thread::id{}); } + + std::vector UiTrail::snapshot() { + Trail& shared = trail(); + const int depth = std::min(shared.depth.load(std::memory_order_acquire), kDepth); + std::vector names; + for (int index = 0; index < depth; index++) { + const char* name = shared.names[index].load(std::memory_order_relaxed); + names.emplace_back(name ? name : "?"); + } + const int overflow = shared.overflow.load(std::memory_order_relaxed); + if (overflow > 0) names.push_back("... " + std::to_string(overflow) + " deeper"); + return names; + } +} diff --git a/Diagnostics/private/UiTrail.hpp b/Diagnostics/private/UiTrail.hpp new file mode 100644 index 00000000..adf175f9 --- /dev/null +++ b/Diagnostics/private/UiTrail.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace tp { + + class UiTrail { + public: + static constexpr int kDepth = 32; + + static void push(const char* name); + static void pop(); + [[nodiscard]] static bool isTracked(); + static void trackCallingThread(); + static void stopTracking(); + [[nodiscard]] static std::vector snapshot(); + }; +} diff --git a/Diagnostics/public/HangWatchdog.hpp b/Diagnostics/public/HangWatchdog.hpp new file mode 100644 index 00000000..2be24497 --- /dev/null +++ b/Diagnostics/public/HangWatchdog.hpp @@ -0,0 +1,120 @@ +#pragma once + +#include "StackTrace.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace tp { + + enum class UiPhase : uint8_t { Idle, Input, Draw, Present }; + + struct HangSettings { + double thresholdMs = 2000; + std::vector followUpMs = { 10000, 30000 }; + int stackSamples = 5; + double stackSampleIntervalMs = 200; + double processPauseMs = 1000; + size_t keptReports = 20; + std::string application = "app"; + std::string build = "dev"; + std::filesystem::path directory; + std::function clockNs; + }; + + struct HangReportSection { + std::string fileName; + std::string text; + }; + + class HangWatchdog { + public: + using Contributor = std::function()>; + + explicit HangWatchdog(HangSettings settings); + ~HangWatchdog(); + + HangWatchdog(const HangWatchdog&) = delete; + HangWatchdog& operator=(const HangWatchdog&) = delete; + + void enterPhase(UiPhase phase); + void noteInputEvent(); + void addContributor(Contributor contributor); + + [[nodiscard]] const std::filesystem::path& directory() const { return mSettings.directory; } + [[nodiscard]] std::vector reports() const; + [[nodiscard]] size_t reportsWritten() const { return mReportsWritten.load(); } + [[nodiscard]] uint64_t inspections() const { return mInspections.load(); } + + [[nodiscard]] static std::filesystem::path defaultHangsDirectory(); + [[nodiscard]] static std::filesystem::path directoryFor(const std::filesystem::path& applicationHangsDirectory); + [[nodiscard]] static std::string processName(); + static void allowDebuggerAttach(); + [[nodiscard]] static const char* phaseName(UiPhase phase); + + private: + struct Stall { + uint64_t beat = 0; + UiPhase phase = UiPhase::Idle; + std::filesystem::path report; + size_t followUps = 0; + }; + + struct Investigation { + std::string verdict; + std::string threads; + }; + + void watch(); + void inspect(Stall& stall, int64_t now); + void discountProcessPause(int64_t now, int64_t previous); + [[nodiscard]] double stalledMs(int64_t now) const; + [[nodiscard]] uint64_t inputEventsInPhase() const; + + void beginReport(Stall& stall, double stalled); + void appendFollowUp(Stall& stall, double stalled); + void finishRecovered(Stall& stall, double stalled); + [[nodiscard]] std::string reportHeader() const; + [[nodiscard]] std::string uiSnapshot(double stalled, UiPhase phase) const; + void writeContributions(const std::filesystem::path& report); + [[nodiscard]] Investigation investigate() const; + [[nodiscard]] std::string + sampleStacks(std::vector>& uiSamples, std::string& uiState, std::string& uiWait) const; + [[nodiscard]] static std::string formatThreads(const std::vector& threads, int uiTid); + [[nodiscard]] static std::string stackVerdict( + const std::vector>& uiSamples, const std::string& state, const std::string& waitChannel + ); + + void pruneOldReports() const; + [[nodiscard]] std::filesystem::path newReportPath() const; + [[nodiscard]] int64_t now() const; + + HangSettings mSettings; + std::atomic mBeat{ 0 }; + std::atomic mPhaseStartNs{ 0 }; + std::atomic mPhase{ UiPhase::Idle }; + std::atomic mInputEvents{ 0 }; + std::atomic mPhaseInputEvents{ 0 }; + std::atomic mReportsWritten{ 0 }; + std::atomic mInspections{ 0 }; + int mUiTid = 0; + + uint64_t mPausedBeat = 0; + int64_t mPausedNs = 0; + + std::mutex mContributorsMutex; + std::vector mContributors; + + std::mutex mStopMutex; + std::condition_variable mStopped; + bool mStopping = false; + std::thread mThread; + }; +} -- 2.49.1 From 66c7b7c4a5022add33fa41a0dfa874afb2b7026c Mon Sep 17 00:00:00 2001 From: agent Date: Tue, 27 Oct 2026 23:28:11 +0300 Subject: [PATCH 2/2] Diagnostics: cover hang reporting with unit tests The tests exercise blocked and busy reports, scope names, debugger pauses, summary-before-sampling, input event counts, idle suppression, contributors and report retention. They drive UiTrail directly and run with an injectable clock. --- Diagnostics/tests/HangWatchdogTests.cpp | 244 ++++++++++++++++++++++++ Diagnostics/tests/Main.cpp | 3 + 2 files changed, 247 insertions(+) create mode 100644 Diagnostics/tests/HangWatchdogTests.cpp create mode 100644 Diagnostics/tests/Main.cpp diff --git a/Diagnostics/tests/HangWatchdogTests.cpp b/Diagnostics/tests/HangWatchdogTests.cpp new file mode 100644 index 00000000..9b2488c4 --- /dev/null +++ b/Diagnostics/tests/HangWatchdogTests.cpp @@ -0,0 +1,244 @@ +#include "HangWatchdog.hpp" +#include "UiTrail.hpp" + +#include "UnitTest++/UnitTest++.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace tp; + +namespace { + struct TempDirectory { + std::filesystem::path path; + + explicit TempDirectory(const std::string& name) : + path(std::filesystem::temp_directory_path() / ("hang-watchdog-test-" + name)) { + std::filesystem::remove_all(path); + } + ~TempDirectory() { + if (!std::getenv("KEEP_HANG_REPORTS")) std::filesystem::remove_all(path); + } + }; + + HangSettings fastSettings(const std::filesystem::path& directory) { + HangSettings settings; + settings.thresholdMs = 150; + settings.followUpMs = {}; + settings.stackSamples = 3; + settings.stackSampleIntervalMs = 30; + settings.application = "test"; + settings.build = "0"; + settings.directory = directory; + return settings; + } + + std::string readFile(const std::filesystem::path& path) { + std::ifstream file(path); + std::stringstream text; + text << file.rdbuf(); + return text.str(); + } + + struct TestScope { + explicit TestScope(const char* name) { UiTrail::push(name); } + ~TestScope() { UiTrail::pop(); } + }; + + bool hasVerdict(const std::filesystem::path& report) { + return readFile(report / "summary.txt").find("verdict:") != std::string::npos; + } + + template + bool waitUntil(Done&& done, double limitMs = 5000) { + const auto start = std::chrono::steady_clock::now(); + while (!done()) { + if (std::chrono::duration(std::chrono::steady_clock::now() - start).count() > limitMs) + return false; + std::this_thread::yield(); + } + return true; + } + + class Gate { + public: + void waitOpen() { + std::unique_lock lock(mMutex); + mOpened.wait(lock, [this]() { return mOpen; }); + } + void open() { + { + std::lock_guard lock(mMutex); + mOpen = true; + } + mOpened.notify_all(); + } + + private: + std::mutex mMutex; + std::condition_variable mOpened; + bool mOpen = false; + }; +} + +SUITE(HangWatchdog) { + TEST(StuckDrawWritesAReportNamingTheOpenScopes) { + TempDirectory directory("stuck"); + HangWatchdog watchdog(fastSettings(directory.path)); + Gate gate; + watchdog.enterPhase(UiPhase::Draw); + { + TestScope outer("test.outer"); + TestScope waiting("test.waiting_on_gate"); + std::thread release([&]() { + waitUntil([&]() { return watchdog.reportsWritten() > 0 && hasVerdict(watchdog.reports().front()); }); + gate.open(); + }); + gate.waitOpen(); + release.join(); + } + watchdog.enterPhase(UiPhase::Idle); + REQUIRE CHECK_EQUAL(1u, watchdog.reports().size()); + const std::string summary = readFile(watchdog.reports().front() / "summary.txt"); + CHECK(summary.find("drawing") != std::string::npos); + CHECK(summary.find("test.outer") != std::string::npos); + CHECK(summary.find("test.waiting_on_gate") != std::string::npos); + CHECK(summary.find("verdict: blocked") != std::string::npos); + const std::string threads = readFile(watchdog.reports().front() / "threads.txt"); + CHECK(threads.find("UI THREAD") != std::string::npos); + CHECK( + threads.find("waitOpen") != std::string::npos || threads.find("pthread_cond") != std::string::npos || + threads.find("libc") != std::string::npos + ); + CHECK(waitUntil([&]() { + return readFile(watchdog.reports().front() / "summary.txt").find("recovered") != std::string::npos; + })); + } + + TEST(SpinningUiThreadIsReportedAsBusy) { + TempDirectory directory("busy"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.enterPhase(UiPhase::Input); + { + TestScope spinning("test.spinning"); + volatile uint64_t work = 0; + while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front())) + work = work + 1; + } + watchdog.enterPhase(UiPhase::Idle); + const std::string summary = readFile(watchdog.reports().front() / "summary.txt"); + CHECK(summary.find("test.spinning") != std::string::npos); + CHECK(summary.find("verdict: busy") != std::string::npos); + } + + TEST(AProcessPauseIsNotCountedAsAHang) { + TempDirectory directory("paused"); + std::atomic clock{ 0 }; + HangSettings settings = fastSettings(directory.path); + settings.thresholdMs = 2000; + settings.processPauseMs = 1000; + settings.clockNs = [&clock]() { return clock.load(); }; + HangWatchdog watchdog(settings); + watchdog.enterPhase(UiPhase::Draw); + + const auto step = [&](int64_t milliseconds) { + const uint64_t seen = watchdog.inspections(); + clock.fetch_add(milliseconds * 1000000); + REQUIRE CHECK(waitUntil([&]() { return watchdog.inspections() > seen; })); + }; + step(100); + step(30000); + step(100); + CHECK_EQUAL(0u, watchdog.reportsWritten()); + + for (int tick = 0; tick < 25; tick++) + step(100); + CHECK_EQUAL(1u, watchdog.reportsWritten()); + watchdog.enterPhase(UiPhase::Idle); + } + + TEST(SummaryIsWrittenBeforeThreadsAreSampled) { + TempDirectory directory("summary-first"); + HangSettings settings = fastSettings(directory.path); + std::filesystem::path seenReport; + bool summaryExisted = false; + HangWatchdog watchdog(settings); + watchdog.addContributor([&]() { + seenReport = watchdog.reports().empty() ? std::filesystem::path() : watchdog.reports().front(); + summaryExisted = !seenReport.empty() && std::filesystem::exists(seenReport / "summary.txt"); + return std::vector{}; + }); + watchdog.enterPhase(UiPhase::Present); + REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; })); + watchdog.enterPhase(UiPhase::Idle); + CHECK(summaryExisted); + const std::string summary = readFile(seenReport / "summary.txt"); + CHECK(summary.find("debugger attached no") != std::string::npos); + } + + TEST(InputEventsHandledInOnePhaseAreCounted) { + TempDirectory directory("input-loop"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.enterPhase(UiPhase::Input); + { + TestScope inputLoop("test.input_loop"); + while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front())) + watchdog.noteInputEvent(); + } + watchdog.enterPhase(UiPhase::Idle); + const std::string summary = readFile(watchdog.reports().front() / "summary.txt"); + CHECK(summary.find("input events handled during this phase 0") == std::string::npos); + CHECK(summary.find("input events were handled without leaving the input phase") != std::string::npos); + } + + TEST(IdleNeverProducesAReport) { + TempDirectory directory("idle"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.enterPhase(UiPhase::Idle); + Gate gate; + std::thread release([&]() { + const auto start = std::chrono::steady_clock::now(); + waitUntil([&]() { return std::chrono::steady_clock::now() - start > std::chrono::milliseconds(600); }); + gate.open(); + }); + gate.waitOpen(); + release.join(); + CHECK_EQUAL(0u, watchdog.reportsWritten()); + } + + TEST(ContributorsAddTheirOwnFiles) { + TempDirectory directory("contributors"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.addContributor([]() { return std::vector{ { "engine.txt", "queue 3\n" } }; }); + watchdog.enterPhase(UiPhase::Input); + REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; })); + watchdog.enterPhase(UiPhase::Idle); + CHECK_EQUAL(std::string("queue 3\n"), readFile(watchdog.reports().front() / "engine.txt")); + } + + TEST(OnlyTheNewestReportsAreKept) { + TempDirectory directory("retention"); + std::filesystem::create_directories(directory.path); + for (int index = 0; index < 5; index++) + std::filesystem::create_directories(directory.path / ("old-" + std::to_string(index))); + HangSettings settings = fastSettings(directory.path); + settings.keptReports = 2; + HangWatchdog watchdog(settings); + CHECK_EQUAL(2u, watchdog.reports().size()); + } + + TEST(DirectoryIsTheApplicationsHangFolderUnlessOverridden) { + const std::filesystem::path application = "/home/someone/sketch2d/hangs"; + CHECK(HangWatchdog::directoryFor(application) == application); + setenv("MODULES_HANG_DIR", "/tmp/custom-hangs", 1); + CHECK(HangWatchdog::directoryFor(application) == std::filesystem::path("/tmp/custom-hangs")); + unsetenv("MODULES_HANG_DIR"); + } +} diff --git a/Diagnostics/tests/Main.cpp b/Diagnostics/tests/Main.cpp new file mode 100644 index 00000000..7ae121b1 --- /dev/null +++ b/Diagnostics/tests/Main.cpp @@ -0,0 +1,3 @@ +#include "UnitTest++/UnitTest++.h" + +int main() { return UnitTest::RunAllTests(); } -- 2.49.1