diff --git a/CMakeLists.txt b/CMakeLists.txt index db5b4851..0465385d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ add_subdirectory(Threading) add_subdirectory(Containers) add_subdirectory(Math) add_subdirectory(Callstack) +add_subdirectory(Diagnostics) add_subdirectory(Allocators) add_subdirectory(Strings) add_subdirectory(Language) diff --git a/Diagnostics/CMakeLists.txt b/Diagnostics/CMakeLists.txt new file mode 100644 index 00000000..071faecc --- /dev/null +++ b/Diagnostics/CMakeLists.txt @@ -0,0 +1,16 @@ +project(Diagnostics) + +### ---------------------- Static Library --------------------- ### +file(GLOB SOURCES "./private/*.cpp") +file(GLOB HEADERS "./public/*.hpp" "./private/*.hpp") +add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS}) +target_include_directories(${PROJECT_NAME} PUBLIC ./public/ PRIVATE ./private/) +target_link_libraries(${PROJECT_NAME} PUBLIC Modules Callstack) +find_package(Threads REQUIRED) +target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads) + +### -------------------------- Tests -------------------------- ### +file(GLOB TEST_SOURCES "./tests/*.cpp") +add_executable(Tests${PROJECT_NAME} ${TEST_SOURCES}) +target_link_libraries(Tests${PROJECT_NAME} ${PROJECT_NAME} UnitTest++) +add_test(NAME Tests${PROJECT_NAME} COMMAND Tests${PROJECT_NAME}) diff --git a/Diagnostics/private/HangWatchdog.cpp b/Diagnostics/private/HangWatchdog.cpp new file mode 100644 index 00000000..1060915b --- /dev/null +++ b/Diagnostics/private/HangWatchdog.cpp @@ -0,0 +1,347 @@ +#include "HangWatchdog.hpp" + +#include "UiTrail.hpp" +#include "StackTrace.hpp" + +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +namespace tp { + + namespace { + int64_t nowNs() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count(); + } + + std::string timestamp(const char* pattern) { + const std::time_t now = std::time(nullptr); + char text[64]; + std::strftime(text, sizeof(text), pattern, std::localtime(&now)); + return text; + } + + void writeFile(const std::filesystem::path& path, const std::string& text, std::ios::openmode mode) { + std::ofstream file(path, mode); + file << text; + } + + constexpr double kCheckIntervalMs = 100; + } + + HangWatchdog::HangWatchdog(HangSettings settings) : + mSettings(std::move(settings)) { + mSettings.directory = directoryFor(mSettings.directory.empty() ? defaultHangsDirectory() : mSettings.directory); + if (!mSettings.clockNs) mSettings.clockNs = nowNs; + mPhaseStartNs.store(now()); + allowDebuggerAttach(); + StackTrace::prepareThreadSampling(); + mUiTid = StackTrace::callingThreadId(); + UiTrail::trackCallingThread(); + pruneOldReports(); + mThread = std::thread([this]() { watch(); }); + } + + HangWatchdog::~HangWatchdog() { + { + std::lock_guard lock(mStopMutex); + mStopping = true; + } + mStopped.notify_all(); + mThread.join(); + UiTrail::stopTracking(); + } + + std::string HangWatchdog::formatThreads(const std::vector& threads, int uiTid) { + std::ostringstream text; + for (const ThreadSample& thread : threads) { + text << (thread.tid == uiTid ? "UI THREAD " : "thread ") << thread.tid << " \"" << thread.name << "\" state " + << thread.state; + if (!thread.waitChannel.empty() && thread.waitChannel != "0") text << " waiting in " << thread.waitChannel; + text << "\n"; + if (!thread.answered) text << " (did not answer the sample signal)\n"; + for (uintptr_t frame : thread.frames) + text << " " << StackTrace::describe(frame) << "\n"; + text << "\n"; + } + return text.str(); + } + + void HangWatchdog::enterPhase(UiPhase phase) { + mPhaseStartNs.store(now(), std::memory_order_relaxed); + mPhaseInputEvents.store(mInputEvents.load(std::memory_order_relaxed), std::memory_order_relaxed); + mPhase.store(phase, std::memory_order_relaxed); + mBeat.fetch_add(1, std::memory_order_release); + } + + void HangWatchdog::noteInputEvent() { mInputEvents.fetch_add(1, std::memory_order_relaxed); } + + int64_t HangWatchdog::now() const { return mSettings.clockNs(); } + + void HangWatchdog::addContributor(Contributor contributor) { + std::lock_guard lock(mContributorsMutex); + mContributors.push_back(std::move(contributor)); + } + + std::filesystem::path HangWatchdog::defaultHangsDirectory() { + const char* home = std::getenv("HOME"); + const std::filesystem::path root = home && *home ? std::filesystem::path(home) : std::filesystem::current_path(); + return root / processName() / "hangs"; + } + + std::filesystem::path HangWatchdog::directoryFor(const std::filesystem::path& applicationHangsDirectory) { + if (const char* configured = std::getenv("MODULES_HANG_DIR"); configured && *configured) return configured; + return applicationHangsDirectory; + } + + std::string HangWatchdog::processName() { +#if defined(__linux__) + std::ifstream comm("/proc/self/comm"); + std::string name; + if (std::getline(comm, name) && !name.empty()) return name; +#endif + return "app"; + } + + void HangWatchdog::allowDebuggerAttach() { +#if defined(__linux__) && defined(PR_SET_PTRACER) + prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif + } + + const char* HangWatchdog::phaseName(UiPhase phase) { + switch (phase) { + case UiPhase::Idle: return "idle"; + case UiPhase::Input: return "processing input"; + case UiPhase::Draw: return "drawing"; + case UiPhase::Present: return "presenting"; + } + return "unknown"; + } + + void HangWatchdog::watch() { + Stall stall; + int64_t previous = now(); + std::unique_lock lock(mStopMutex); + while (!mStopping) { + mStopped.wait_for(lock, std::chrono::duration(kCheckIntervalMs), [this]() { + return mStopping; + }); + if (mStopping) return; + lock.unlock(); + const int64_t current = now(); + discountProcessPause(current, previous); + inspect(stall, current); + previous = current; + mInspections.fetch_add(1); + lock.lock(); + } + } + + void HangWatchdog::discountProcessPause(int64_t now, int64_t previous) { + const double sinceLastCheckMs = (double) (now - previous) / 1.0e6; + const double pausedMs = sinceLastCheckMs - kCheckIntervalMs; + if (pausedMs < mSettings.processPauseMs) return; + + const uint64_t beat = mBeat.load(std::memory_order_acquire); + if (beat != mPausedBeat) { + mPausedBeat = beat; + mPausedNs = 0; + } + mPausedNs += (int64_t) (pausedMs * 1.0e6); + } + + double HangWatchdog::stalledMs(int64_t now) const { + const bool pausedThisPhase = mPausedBeat == mBeat.load(std::memory_order_acquire); + const int64_t paused = pausedThisPhase ? mPausedNs : 0; + return (double) (now - mPhaseStartNs.load(std::memory_order_relaxed) - paused) / 1.0e6; + } + + uint64_t HangWatchdog::inputEventsInPhase() const { + return mInputEvents.load(std::memory_order_relaxed) - mPhaseInputEvents.load(std::memory_order_relaxed); + } + + void HangWatchdog::inspect(Stall& stall, int64_t now) { + const uint64_t beat = mBeat.load(std::memory_order_acquire); + const UiPhase phase = mPhase.load(std::memory_order_relaxed); + const double stalled = stalledMs(now); + + if (!stall.report.empty() && stall.beat != beat) { + finishRecovered(stall, stalled); + stall = {}; + } + if (phase == UiPhase::Idle || stalled < mSettings.thresholdMs) return; + + if (stall.report.empty()) { + stall.beat = beat; + stall.phase = phase; + beginReport(stall, stalled); + return; + } + if (stall.followUps < mSettings.followUpMs.size() && stalled >= mSettings.followUpMs[stall.followUps]) + appendFollowUp(stall, stalled); + } + + std::string HangWatchdog::uiSnapshot(double stalled, UiPhase phase) const { + std::ostringstream text; + text << "ui thread " << phaseName(phase) << " for " << (int64_t) stalled << " ms\n"; + text << "input events handled during this phase " << inputEventsInPhase() << "\n"; + text << "scope trail (outermost first):\n"; + const std::vector names = UiTrail::snapshot(); + if (names.empty()) text << " (no profiled scope open)\n"; + for (size_t index = 0; index < names.size(); index++) + text << " " << std::string(index * 2, ' ') << names[index] << "\n"; + return text.str(); + } + + std::filesystem::path HangWatchdog::newReportPath() const { + const std::string base = mSettings.application + "-" + timestamp("%Y-%m-%d_%H-%M-%S") + "-build" + mSettings.build; + std::filesystem::path path = mSettings.directory / base; + for (int suffix = 2; std::filesystem::exists(path); suffix++) + path = mSettings.directory / (base + "-" + std::to_string(suffix)); + return path; + } + + std::string HangWatchdog::reportHeader() const { + std::ostringstream text; + text << "application " << mSettings.application << "\n"; + text << "build " << mSettings.build << "\n"; + text << "detected " << timestamp("%Y-%m-%d %H:%M:%S") << "\n"; + text << "threshold " << (int64_t) mSettings.thresholdMs << " ms\n"; + text << "debugger attached " << (StackTrace::isDebuggerAttached() ? "yes" : "no") << "\n\n"; + return text.str(); + } + + void HangWatchdog::beginReport(Stall& stall, double stalled) { + std::error_code error; + std::filesystem::create_directories(mSettings.directory, error); + stall.report = newReportPath(); + std::filesystem::create_directories(stall.report, error); + if (error) return; + + writeFile( + stall.report / "summary.txt", reportHeader() + uiSnapshot(stalled, stall.phase), std::ios::out | std::ios::trunc + ); + writeContributions(stall.report); + mReportsWritten.fetch_add(1); + + const Investigation investigation = investigate(); + writeFile( + stall.report / "summary.txt", "\nverdict: " + investigation.verdict + "\n", std::ios::out | std::ios::app + ); + if (!investigation.threads.empty()) + writeFile(stall.report / "threads.txt", investigation.threads, std::ios::out | std::ios::trunc); + pruneOldReports(); + } + + void HangWatchdog::writeContributions(const std::filesystem::path& report) { + std::vector contributors; + { + std::lock_guard lock(mContributorsMutex); + contributors = mContributors; + } + for (const Contributor& contributor : contributors) + for (const HangReportSection& section : contributor()) + writeFile(report / section.fileName, section.text, std::ios::out | std::ios::trunc); + } + + void HangWatchdog::appendFollowUp(Stall& stall, double stalled) { + stall.followUps++; + const std::string header = "\nstill stuck at " + timestamp("%H:%M:%S") + "\n"; + writeFile(stall.report / "summary.txt", header + uiSnapshot(stalled, stall.phase), std::ios::out | std::ios::app); + + const Investigation investigation = investigate(); + writeFile(stall.report / "summary.txt", "verdict: " + investigation.verdict + "\n", std::ios::out | std::ios::app); + if (!investigation.threads.empty()) + writeFile( + stall.report / "threads.txt", + "\n===== follow-up at " + timestamp("%H:%M:%S") + " =====\n" + investigation.threads, + std::ios::out | std::ios::app + ); + } + + void HangWatchdog::finishRecovered(Stall& stall, double stalled) { + (void) stalled; + writeFile( + stall.report / "summary.txt", "\nrecovered at " + timestamp("%H:%M:%S") + "\n", std::ios::out | std::ios::app + ); + } + + HangWatchdog::Investigation HangWatchdog::investigate() const { + if (StackTrace::isDebuggerAttached()) + return { "not sampled: a debugger is attached, its thread view shows where the ui thread is", "" }; + if (!StackTrace::canSampleThreads()) return { "not sampled: thread stacks are not available on this platform", "" }; + + std::vector> uiSamples; + std::string uiState; + std::string uiWait; + std::string threads = sampleStacks(uiSamples, uiState, uiWait); + std::string verdict = stackVerdict(uiSamples, uiState, uiWait); + if (inputEventsInPhase() > 0) + verdict += + "; " + std::to_string(inputEventsInPhase()) + " input events were handled without leaving the input phase"; + return { verdict, threads }; + } + + std::string HangWatchdog::sampleStacks( + std::vector>& uiSamples, std::string& uiState, std::string& uiWait + ) const { + std::ostringstream text; + for (int sample = 0; sample < std::max(1, mSettings.stackSamples); sample++) { + if (sample > 0) + std::this_thread::sleep_for(std::chrono::duration(mSettings.stackSampleIntervalMs)); + const std::vector threads = StackTrace::sampleAllThreads(); + for (const ThreadSample& thread : threads) + if (thread.tid == mUiTid) { + uiSamples.push_back(thread.frames); + uiState = thread.state; + uiWait = thread.waitChannel; + } + text << "===== sample " << sample + 1 << " =====\n" << formatThreads(threads, mUiTid); + } + return text.str(); + } + + std::string HangWatchdog::stackVerdict( + const std::vector>& uiSamples, const std::string& state, const std::string& waitChannel + ) { + if (uiSamples.empty()) return "ui thread not found"; + const bool sameStack = + std::all_of(uiSamples.begin(), uiSamples.end(), [&](const auto& frames) { return frames == uiSamples.front(); }); + const bool sleeping = !state.empty() && (state[0] == 'S' || state[0] == 'D'); + if (sameStack && sleeping) { + std::string text = "blocked: the ui thread sleeps at the same place in every sample"; + if (!waitChannel.empty() && waitChannel != "0") text += " (kernel wait " + waitChannel + ")"; + return text; + } + if (!sleeping) return "busy: the ui thread is running, check the changing frames in threads.txt"; + return "waiting repeatedly: the ui thread sleeps but at different places, likely polling or lock contention"; + } + + std::vector HangWatchdog::reports() const { + std::vector found; + std::error_code error; + for (const auto& entry : std::filesystem::directory_iterator(mSettings.directory, error)) + if (entry.is_directory()) found.push_back(entry.path()); + std::sort(found.begin(), found.end(), [](const auto& a, const auto& b) { + return std::filesystem::last_write_time(a) < std::filesystem::last_write_time(b); + }); + return found; + } + + void HangWatchdog::pruneOldReports() const { + const std::vector found = reports(); + if (found.size() <= mSettings.keptReports) return; + std::error_code error; + for (size_t index = 0; index + mSettings.keptReports < found.size(); index++) + std::filesystem::remove_all(found[index], error); + } +} diff --git a/Diagnostics/private/UiTrail.cpp b/Diagnostics/private/UiTrail.cpp new file mode 100644 index 00000000..b3eee3df --- /dev/null +++ b/Diagnostics/private/UiTrail.cpp @@ -0,0 +1,67 @@ +#include "UiTrail.hpp" + +#include +#include +#include + +namespace tp { + + namespace { + struct Trail { + std::atomic names[UiTrail::kDepth] = {}; + std::atomic depth{ 0 }; + std::atomic overflow{ 0 }; + }; + + Trail& trail() { + static Trail shared; + return shared; + } + + std::atomic& trackedThread() { + static std::atomic tracked{}; + return tracked; + } + } + + void UiTrail::push(const char* name) { + if (!isTracked()) return; + Trail& shared = trail(); + const int depth = shared.depth.load(std::memory_order_relaxed); + if (depth < kDepth) shared.names[depth].store(name, std::memory_order_relaxed); + else shared.overflow.fetch_add(1, std::memory_order_relaxed); + shared.depth.store(depth + 1, std::memory_order_release); + } + + void UiTrail::pop() { + if (!isTracked()) return; + Trail& shared = trail(); + const int depth = shared.depth.load(std::memory_order_relaxed); + if (depth <= 0) return; + if (depth > kDepth) shared.overflow.fetch_sub(1, std::memory_order_relaxed); + shared.depth.store(depth - 1, std::memory_order_release); + } + + bool UiTrail::isTracked() { return trackedThread().load(std::memory_order_relaxed) == std::this_thread::get_id(); } + + void UiTrail::trackCallingThread() { + trail().depth.store(0); + trail().overflow.store(0); + trackedThread().store(std::this_thread::get_id()); + } + + void UiTrail::stopTracking() { trackedThread().store(std::thread::id{}); } + + std::vector UiTrail::snapshot() { + Trail& shared = trail(); + const int depth = std::min(shared.depth.load(std::memory_order_acquire), kDepth); + std::vector names; + for (int index = 0; index < depth; index++) { + const char* name = shared.names[index].load(std::memory_order_relaxed); + names.emplace_back(name ? name : "?"); + } + const int overflow = shared.overflow.load(std::memory_order_relaxed); + if (overflow > 0) names.push_back("... " + std::to_string(overflow) + " deeper"); + return names; + } +} diff --git a/Diagnostics/private/UiTrail.hpp b/Diagnostics/private/UiTrail.hpp new file mode 100644 index 00000000..adf175f9 --- /dev/null +++ b/Diagnostics/private/UiTrail.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace tp { + + class UiTrail { + public: + static constexpr int kDepth = 32; + + static void push(const char* name); + static void pop(); + [[nodiscard]] static bool isTracked(); + static void trackCallingThread(); + static void stopTracking(); + [[nodiscard]] static std::vector snapshot(); + }; +} diff --git a/Diagnostics/public/HangWatchdog.hpp b/Diagnostics/public/HangWatchdog.hpp new file mode 100644 index 00000000..2be24497 --- /dev/null +++ b/Diagnostics/public/HangWatchdog.hpp @@ -0,0 +1,120 @@ +#pragma once + +#include "StackTrace.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace tp { + + enum class UiPhase : uint8_t { Idle, Input, Draw, Present }; + + struct HangSettings { + double thresholdMs = 2000; + std::vector followUpMs = { 10000, 30000 }; + int stackSamples = 5; + double stackSampleIntervalMs = 200; + double processPauseMs = 1000; + size_t keptReports = 20; + std::string application = "app"; + std::string build = "dev"; + std::filesystem::path directory; + std::function clockNs; + }; + + struct HangReportSection { + std::string fileName; + std::string text; + }; + + class HangWatchdog { + public: + using Contributor = std::function()>; + + explicit HangWatchdog(HangSettings settings); + ~HangWatchdog(); + + HangWatchdog(const HangWatchdog&) = delete; + HangWatchdog& operator=(const HangWatchdog&) = delete; + + void enterPhase(UiPhase phase); + void noteInputEvent(); + void addContributor(Contributor contributor); + + [[nodiscard]] const std::filesystem::path& directory() const { return mSettings.directory; } + [[nodiscard]] std::vector reports() const; + [[nodiscard]] size_t reportsWritten() const { return mReportsWritten.load(); } + [[nodiscard]] uint64_t inspections() const { return mInspections.load(); } + + [[nodiscard]] static std::filesystem::path defaultHangsDirectory(); + [[nodiscard]] static std::filesystem::path directoryFor(const std::filesystem::path& applicationHangsDirectory); + [[nodiscard]] static std::string processName(); + static void allowDebuggerAttach(); + [[nodiscard]] static const char* phaseName(UiPhase phase); + + private: + struct Stall { + uint64_t beat = 0; + UiPhase phase = UiPhase::Idle; + std::filesystem::path report; + size_t followUps = 0; + }; + + struct Investigation { + std::string verdict; + std::string threads; + }; + + void watch(); + void inspect(Stall& stall, int64_t now); + void discountProcessPause(int64_t now, int64_t previous); + [[nodiscard]] double stalledMs(int64_t now) const; + [[nodiscard]] uint64_t inputEventsInPhase() const; + + void beginReport(Stall& stall, double stalled); + void appendFollowUp(Stall& stall, double stalled); + void finishRecovered(Stall& stall, double stalled); + [[nodiscard]] std::string reportHeader() const; + [[nodiscard]] std::string uiSnapshot(double stalled, UiPhase phase) const; + void writeContributions(const std::filesystem::path& report); + [[nodiscard]] Investigation investigate() const; + [[nodiscard]] std::string + sampleStacks(std::vector>& uiSamples, std::string& uiState, std::string& uiWait) const; + [[nodiscard]] static std::string formatThreads(const std::vector& threads, int uiTid); + [[nodiscard]] static std::string stackVerdict( + const std::vector>& uiSamples, const std::string& state, const std::string& waitChannel + ); + + void pruneOldReports() const; + [[nodiscard]] std::filesystem::path newReportPath() const; + [[nodiscard]] int64_t now() const; + + HangSettings mSettings; + std::atomic mBeat{ 0 }; + std::atomic mPhaseStartNs{ 0 }; + std::atomic mPhase{ UiPhase::Idle }; + std::atomic mInputEvents{ 0 }; + std::atomic mPhaseInputEvents{ 0 }; + std::atomic mReportsWritten{ 0 }; + std::atomic mInspections{ 0 }; + int mUiTid = 0; + + uint64_t mPausedBeat = 0; + int64_t mPausedNs = 0; + + std::mutex mContributorsMutex; + std::vector mContributors; + + std::mutex mStopMutex; + std::condition_variable mStopped; + bool mStopping = false; + std::thread mThread; + }; +} diff --git a/Diagnostics/tests/HangWatchdogTests.cpp b/Diagnostics/tests/HangWatchdogTests.cpp new file mode 100644 index 00000000..9b2488c4 --- /dev/null +++ b/Diagnostics/tests/HangWatchdogTests.cpp @@ -0,0 +1,244 @@ +#include "HangWatchdog.hpp" +#include "UiTrail.hpp" + +#include "UnitTest++/UnitTest++.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace tp; + +namespace { + struct TempDirectory { + std::filesystem::path path; + + explicit TempDirectory(const std::string& name) : + path(std::filesystem::temp_directory_path() / ("hang-watchdog-test-" + name)) { + std::filesystem::remove_all(path); + } + ~TempDirectory() { + if (!std::getenv("KEEP_HANG_REPORTS")) std::filesystem::remove_all(path); + } + }; + + HangSettings fastSettings(const std::filesystem::path& directory) { + HangSettings settings; + settings.thresholdMs = 150; + settings.followUpMs = {}; + settings.stackSamples = 3; + settings.stackSampleIntervalMs = 30; + settings.application = "test"; + settings.build = "0"; + settings.directory = directory; + return settings; + } + + std::string readFile(const std::filesystem::path& path) { + std::ifstream file(path); + std::stringstream text; + text << file.rdbuf(); + return text.str(); + } + + struct TestScope { + explicit TestScope(const char* name) { UiTrail::push(name); } + ~TestScope() { UiTrail::pop(); } + }; + + bool hasVerdict(const std::filesystem::path& report) { + return readFile(report / "summary.txt").find("verdict:") != std::string::npos; + } + + template + bool waitUntil(Done&& done, double limitMs = 5000) { + const auto start = std::chrono::steady_clock::now(); + while (!done()) { + if (std::chrono::duration(std::chrono::steady_clock::now() - start).count() > limitMs) + return false; + std::this_thread::yield(); + } + return true; + } + + class Gate { + public: + void waitOpen() { + std::unique_lock lock(mMutex); + mOpened.wait(lock, [this]() { return mOpen; }); + } + void open() { + { + std::lock_guard lock(mMutex); + mOpen = true; + } + mOpened.notify_all(); + } + + private: + std::mutex mMutex; + std::condition_variable mOpened; + bool mOpen = false; + }; +} + +SUITE(HangWatchdog) { + TEST(StuckDrawWritesAReportNamingTheOpenScopes) { + TempDirectory directory("stuck"); + HangWatchdog watchdog(fastSettings(directory.path)); + Gate gate; + watchdog.enterPhase(UiPhase::Draw); + { + TestScope outer("test.outer"); + TestScope waiting("test.waiting_on_gate"); + std::thread release([&]() { + waitUntil([&]() { return watchdog.reportsWritten() > 0 && hasVerdict(watchdog.reports().front()); }); + gate.open(); + }); + gate.waitOpen(); + release.join(); + } + watchdog.enterPhase(UiPhase::Idle); + REQUIRE CHECK_EQUAL(1u, watchdog.reports().size()); + const std::string summary = readFile(watchdog.reports().front() / "summary.txt"); + CHECK(summary.find("drawing") != std::string::npos); + CHECK(summary.find("test.outer") != std::string::npos); + CHECK(summary.find("test.waiting_on_gate") != std::string::npos); + CHECK(summary.find("verdict: blocked") != std::string::npos); + const std::string threads = readFile(watchdog.reports().front() / "threads.txt"); + CHECK(threads.find("UI THREAD") != std::string::npos); + CHECK( + threads.find("waitOpen") != std::string::npos || threads.find("pthread_cond") != std::string::npos || + threads.find("libc") != std::string::npos + ); + CHECK(waitUntil([&]() { + return readFile(watchdog.reports().front() / "summary.txt").find("recovered") != std::string::npos; + })); + } + + TEST(SpinningUiThreadIsReportedAsBusy) { + TempDirectory directory("busy"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.enterPhase(UiPhase::Input); + { + TestScope spinning("test.spinning"); + volatile uint64_t work = 0; + while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front())) + work = work + 1; + } + watchdog.enterPhase(UiPhase::Idle); + const std::string summary = readFile(watchdog.reports().front() / "summary.txt"); + CHECK(summary.find("test.spinning") != std::string::npos); + CHECK(summary.find("verdict: busy") != std::string::npos); + } + + TEST(AProcessPauseIsNotCountedAsAHang) { + TempDirectory directory("paused"); + std::atomic clock{ 0 }; + HangSettings settings = fastSettings(directory.path); + settings.thresholdMs = 2000; + settings.processPauseMs = 1000; + settings.clockNs = [&clock]() { return clock.load(); }; + HangWatchdog watchdog(settings); + watchdog.enterPhase(UiPhase::Draw); + + const auto step = [&](int64_t milliseconds) { + const uint64_t seen = watchdog.inspections(); + clock.fetch_add(milliseconds * 1000000); + REQUIRE CHECK(waitUntil([&]() { return watchdog.inspections() > seen; })); + }; + step(100); + step(30000); + step(100); + CHECK_EQUAL(0u, watchdog.reportsWritten()); + + for (int tick = 0; tick < 25; tick++) + step(100); + CHECK_EQUAL(1u, watchdog.reportsWritten()); + watchdog.enterPhase(UiPhase::Idle); + } + + TEST(SummaryIsWrittenBeforeThreadsAreSampled) { + TempDirectory directory("summary-first"); + HangSettings settings = fastSettings(directory.path); + std::filesystem::path seenReport; + bool summaryExisted = false; + HangWatchdog watchdog(settings); + watchdog.addContributor([&]() { + seenReport = watchdog.reports().empty() ? std::filesystem::path() : watchdog.reports().front(); + summaryExisted = !seenReport.empty() && std::filesystem::exists(seenReport / "summary.txt"); + return std::vector{}; + }); + watchdog.enterPhase(UiPhase::Present); + REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; })); + watchdog.enterPhase(UiPhase::Idle); + CHECK(summaryExisted); + const std::string summary = readFile(seenReport / "summary.txt"); + CHECK(summary.find("debugger attached no") != std::string::npos); + } + + TEST(InputEventsHandledInOnePhaseAreCounted) { + TempDirectory directory("input-loop"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.enterPhase(UiPhase::Input); + { + TestScope inputLoop("test.input_loop"); + while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front())) + watchdog.noteInputEvent(); + } + watchdog.enterPhase(UiPhase::Idle); + const std::string summary = readFile(watchdog.reports().front() / "summary.txt"); + CHECK(summary.find("input events handled during this phase 0") == std::string::npos); + CHECK(summary.find("input events were handled without leaving the input phase") != std::string::npos); + } + + TEST(IdleNeverProducesAReport) { + TempDirectory directory("idle"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.enterPhase(UiPhase::Idle); + Gate gate; + std::thread release([&]() { + const auto start = std::chrono::steady_clock::now(); + waitUntil([&]() { return std::chrono::steady_clock::now() - start > std::chrono::milliseconds(600); }); + gate.open(); + }); + gate.waitOpen(); + release.join(); + CHECK_EQUAL(0u, watchdog.reportsWritten()); + } + + TEST(ContributorsAddTheirOwnFiles) { + TempDirectory directory("contributors"); + HangWatchdog watchdog(fastSettings(directory.path)); + watchdog.addContributor([]() { return std::vector{ { "engine.txt", "queue 3\n" } }; }); + watchdog.enterPhase(UiPhase::Input); + REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; })); + watchdog.enterPhase(UiPhase::Idle); + CHECK_EQUAL(std::string("queue 3\n"), readFile(watchdog.reports().front() / "engine.txt")); + } + + TEST(OnlyTheNewestReportsAreKept) { + TempDirectory directory("retention"); + std::filesystem::create_directories(directory.path); + for (int index = 0; index < 5; index++) + std::filesystem::create_directories(directory.path / ("old-" + std::to_string(index))); + HangSettings settings = fastSettings(directory.path); + settings.keptReports = 2; + HangWatchdog watchdog(settings); + CHECK_EQUAL(2u, watchdog.reports().size()); + } + + TEST(DirectoryIsTheApplicationsHangFolderUnlessOverridden) { + const std::filesystem::path application = "/home/someone/sketch2d/hangs"; + CHECK(HangWatchdog::directoryFor(application) == application); + setenv("MODULES_HANG_DIR", "/tmp/custom-hangs", 1); + CHECK(HangWatchdog::directoryFor(application) == std::filesystem::path("/tmp/custom-hangs")); + unsetenv("MODULES_HANG_DIR"); + } +} diff --git a/Diagnostics/tests/Main.cpp b/Diagnostics/tests/Main.cpp new file mode 100644 index 00000000..7ae121b1 --- /dev/null +++ b/Diagnostics/tests/Main.cpp @@ -0,0 +1,3 @@ +#include "UnitTest++/UnitTest++.h" + +int main() { return UnitTest::RunAllTests(); }