ilusha/Modules
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 30

Diagnostics: hang watchdog with scoped reports and thread stacks #6

Closed
agent wants to merge 2 commits from Diagnostics into master
pull from: Diagnostics
merge into: ilusha:master
Conversation 0 Commits 2 Files changed 8 +817
8 changed files with 817 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
CMakeLists.txt
View file

@ -23,6 +23,7 @@ add_subdirectory(Threading)
add_subdirectory(Containers)
add_subdirectory(Math)
add_subdirectory(Callstack)
add_subdirectory(Diagnostics)
add_subdirectory(Allocators)
add_subdirectory(Strings)
add_subdirectory(Language)

16
Diagnostics/CMakeLists.txt Normal file
View file

@ -0,0 +1,16 @@
project(Diagnostics)
### ---------------------- Static Library --------------------- ###
file(GLOB SOURCES "./private/*.cpp")
file(GLOB HEADERS "./public/*.hpp" "./private/*.hpp")
add_library(${PROJECT_NAME} STATIC ${SOURCES} ${HEADERS})
target_include_directories(${PROJECT_NAME} PUBLIC ./public/ PRIVATE ./private/)
target_link_libraries(${PROJECT_NAME} PUBLIC Modules Callstack)
find_package(Threads REQUIRED)
target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads)
### -------------------------- Tests -------------------------- ###
file(GLOB TEST_SOURCES "./tests/*.cpp")
add_executable(Tests${PROJECT_NAME} ${TEST_SOURCES})
target_link_libraries(Tests${PROJECT_NAME} ${PROJECT_NAME} UnitTest++)
add_test(NAME Tests${PROJECT_NAME} COMMAND Tests${PROJECT_NAME})

347
Diagnostics/private/HangWatchdog.cpp Normal file
View file

@ -0,0 +1,347 @@
#include "HangWatchdog.hpp"
#include "UiTrail.hpp"
#include "StackTrace.hpp"
#include <algorithm>
#include <chrono>
#include <cstdlib>
#include <ctime>
#include <fstream>
#include <sstream>
#if defined(__linux__)
#include <sys/prctl.h>
#endif
namespace tp {
namespace {
int64_t nowNs() {
return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch())
.count();
}
std::string timestamp(const char* pattern) {
const std::time_t now = std::time(nullptr);
char text[64];
std::strftime(text, sizeof(text), pattern, std::localtime(&now));
return text;
}
void writeFile(const std::filesystem::path& path, const std::string& text, std::ios::openmode mode) {
std::ofstream file(path, mode);
file << text;
}
constexpr double kCheckIntervalMs = 100;
}
HangWatchdog::HangWatchdog(HangSettings settings) :
mSettings(std::move(settings)) {
mSettings.directory = directoryFor(mSettings.directory.empty() ? defaultHangsDirectory() : mSettings.directory);
if (!mSettings.clockNs) mSettings.clockNs = nowNs;
mPhaseStartNs.store(now());
allowDebuggerAttach();
StackTrace::prepareThreadSampling();
mUiTid = StackTrace::callingThreadId();
UiTrail::trackCallingThread();
pruneOldReports();
mThread = std::thread([this]() { watch(); });
}
HangWatchdog::~HangWatchdog() {
{
std::lock_guard lock(mStopMutex);
mStopping = true;
}
mStopped.notify_all();
mThread.join();
UiTrail::stopTracking();
}
std::string HangWatchdog::formatThreads(const std::vector<ThreadSample>& threads, int uiTid) {
std::ostringstream text;
for (const ThreadSample& thread : threads) {
text << (thread.tid == uiTid ? "UI THREAD " : "thread ") << thread.tid << " \"" << thread.name << "\" state "
<< thread.state;
if (!thread.waitChannel.empty() && thread.waitChannel != "0") text << " waiting in " << thread.waitChannel;
text << "\n";
if (!thread.answered) text << " (did not answer the sample signal)\n";
for (uintptr_t frame : thread.frames)
text << " " << StackTrace::describe(frame) << "\n";
text << "\n";
}
return text.str();
}
void HangWatchdog::enterPhase(UiPhase phase) {
mPhaseStartNs.store(now(), std::memory_order_relaxed);
mPhaseInputEvents.store(mInputEvents.load(std::memory_order_relaxed), std::memory_order_relaxed);
mPhase.store(phase, std::memory_order_relaxed);
mBeat.fetch_add(1, std::memory_order_release);
}
void HangWatchdog::noteInputEvent() { mInputEvents.fetch_add(1, std::memory_order_relaxed); }
int64_t HangWatchdog::now() const { return mSettings.clockNs(); }
void HangWatchdog::addContributor(Contributor contributor) {
std::lock_guard lock(mContributorsMutex);
mContributors.push_back(std::move(contributor));
}
std::filesystem::path HangWatchdog::defaultHangsDirectory() {
const char* home = std::getenv("HOME");
const std::filesystem::path root = home && *home ? std::filesystem::path(home) : std::filesystem::current_path();
return root / processName() / "hangs";
}
std::filesystem::path HangWatchdog::directoryFor(const std::filesystem::path& applicationHangsDirectory) {
if (const char* configured = std::getenv("MODULES_HANG_DIR"); configured && *configured) return configured;
return applicationHangsDirectory;
}
std::string HangWatchdog::processName() {
#if defined(__linux__)
std::ifstream comm("/proc/self/comm");
std::string name;
if (std::getline(comm, name) && !name.empty()) return name;
#endif
return "app";
}
void HangWatchdog::allowDebuggerAttach() {
#if defined(__linux__) && defined(PR_SET_PTRACER)
prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
#endif
}
const char* HangWatchdog::phaseName(UiPhase phase) {
switch (phase) {
case UiPhase::Idle: return "idle";
case UiPhase::Input: return "processing input";
case UiPhase::Draw: return "drawing";
case UiPhase::Present: return "presenting";
}
return "unknown";
}
void HangWatchdog::watch() {
Stall stall;
int64_t previous = now();
std::unique_lock lock(mStopMutex);
while (!mStopping) {
mStopped.wait_for(lock, std::chrono::duration<double, std::milli>(kCheckIntervalMs), [this]() {
return mStopping;
});
if (mStopping) return;
lock.unlock();
const int64_t current = now();
discountProcessPause(current, previous);
inspect(stall, current);
previous = current;
mInspections.fetch_add(1);
lock.lock();
}
}
void HangWatchdog::discountProcessPause(int64_t now, int64_t previous) {
const double sinceLastCheckMs = (double) (now - previous) / 1.0e6;
const double pausedMs = sinceLastCheckMs - kCheckIntervalMs;
if (pausedMs < mSettings.processPauseMs) return;
const uint64_t beat = mBeat.load(std::memory_order_acquire);
if (beat != mPausedBeat) {
mPausedBeat = beat;
mPausedNs = 0;
}
mPausedNs += (int64_t) (pausedMs * 1.0e6);
}
double HangWatchdog::stalledMs(int64_t now) const {
const bool pausedThisPhase = mPausedBeat == mBeat.load(std::memory_order_acquire);
const int64_t paused = pausedThisPhase ? mPausedNs : 0;
return (double) (now - mPhaseStartNs.load(std::memory_order_relaxed) - paused) / 1.0e6;
}
uint64_t HangWatchdog::inputEventsInPhase() const {
return mInputEvents.load(std::memory_order_relaxed) - mPhaseInputEvents.load(std::memory_order_relaxed);
}
void HangWatchdog::inspect(Stall& stall, int64_t now) {
const uint64_t beat = mBeat.load(std::memory_order_acquire);
const UiPhase phase = mPhase.load(std::memory_order_relaxed);
const double stalled = stalledMs(now);
if (!stall.report.empty() && stall.beat != beat) {
finishRecovered(stall, stalled);
stall = {};
}
if (phase == UiPhase::Idle || stalled < mSettings.thresholdMs) return;
if (stall.report.empty()) {
stall.beat = beat;
stall.phase = phase;
beginReport(stall, stalled);
return;
}
if (stall.followUps < mSettings.followUpMs.size() && stalled >= mSettings.followUpMs[stall.followUps])
appendFollowUp(stall, stalled);
}
std::string HangWatchdog::uiSnapshot(double stalled, UiPhase phase) const {
std::ostringstream text;
text << "ui thread " << phaseName(phase) << " for " << (int64_t) stalled << " ms\n";
text << "input events handled during this phase " << inputEventsInPhase() << "\n";
text << "scope trail (outermost first):\n";
const std::vector<std::string> names = UiTrail::snapshot();
if (names.empty()) text << " (no profiled scope open)\n";
for (size_t index = 0; index < names.size(); index++)
text << " " << std::string(index * 2, ' ') << names[index] << "\n";
return text.str();
}
std::filesystem::path HangWatchdog::newReportPath() const {
const std::string base = mSettings.application + "-" + timestamp("%Y-%m-%d_%H-%M-%S") + "-build" + mSettings.build;
std::filesystem::path path = mSettings.directory / base;
for (int suffix = 2; std::filesystem::exists(path); suffix++)
path = mSettings.directory / (base + "-" + std::to_string(suffix));
return path;
}
std::string HangWatchdog::reportHeader() const {
std::ostringstream text;
text << "application " << mSettings.application << "\n";
text << "build " << mSettings.build << "\n";
text << "detected " << timestamp("%Y-%m-%d %H:%M:%S") << "\n";
text << "threshold " << (int64_t) mSettings.thresholdMs << " ms\n";
text << "debugger attached " << (StackTrace::isDebuggerAttached() ? "yes" : "no") << "\n\n";
return text.str();
}
void HangWatchdog::beginReport(Stall& stall, double stalled) {
std::error_code error;
std::filesystem::create_directories(mSettings.directory, error);
stall.report = newReportPath();
std::filesystem::create_directories(stall.report, error);
if (error) return;
writeFile(
stall.report / "summary.txt", reportHeader() + uiSnapshot(stalled, stall.phase), std::ios::out | std::ios::trunc
);
writeContributions(stall.report);
mReportsWritten.fetch_add(1);
const Investigation investigation = investigate();
writeFile(
stall.report / "summary.txt", "\nverdict: " + investigation.verdict + "\n", std::ios::out | std::ios::app
);
if (!investigation.threads.empty())
writeFile(stall.report / "threads.txt", investigation.threads, std::ios::out | std::ios::trunc);
pruneOldReports();
}
void HangWatchdog::writeContributions(const std::filesystem::path& report) {
std::vector<Contributor> contributors;
{
std::lock_guard lock(mContributorsMutex);
contributors = mContributors;
}
for (const Contributor& contributor : contributors)
for (const HangReportSection& section : contributor())
writeFile(report / section.fileName, section.text, std::ios::out | std::ios::trunc);
}
void HangWatchdog::appendFollowUp(Stall& stall, double stalled) {
stall.followUps++;
const std::string header = "\nstill stuck at " + timestamp("%H:%M:%S") + "\n";
writeFile(stall.report / "summary.txt", header + uiSnapshot(stalled, stall.phase), std::ios::out | std::ios::app);
const Investigation investigation = investigate();
writeFile(stall.report / "summary.txt", "verdict: " + investigation.verdict + "\n", std::ios::out | std::ios::app);
if (!investigation.threads.empty())
writeFile(
stall.report / "threads.txt",
"\n===== follow-up at " + timestamp("%H:%M:%S") + " =====\n" + investigation.threads,
std::ios::out | std::ios::app
);
}
void HangWatchdog::finishRecovered(Stall& stall, double stalled) {
(void) stalled;
writeFile(
stall.report / "summary.txt", "\nrecovered at " + timestamp("%H:%M:%S") + "\n", std::ios::out | std::ios::app
);
}
HangWatchdog::Investigation HangWatchdog::investigate() const {
if (StackTrace::isDebuggerAttached())
return { "not sampled: a debugger is attached, its thread view shows where the ui thread is", "" };
if (!StackTrace::canSampleThreads()) return { "not sampled: thread stacks are not available on this platform", "" };
std::vector<std::vector<uintptr_t>> uiSamples;
std::string uiState;
std::string uiWait;
std::string threads = sampleStacks(uiSamples, uiState, uiWait);
std::string verdict = stackVerdict(uiSamples, uiState, uiWait);
if (inputEventsInPhase() > 0)
verdict +=
"; " + std::to_string(inputEventsInPhase()) + " input events were handled without leaving the input phase";
return { verdict, threads };
}
std::string HangWatchdog::sampleStacks(
std::vector<std::vector<uintptr_t>>& uiSamples, std::string& uiState, std::string& uiWait
) const {
std::ostringstream text;
for (int sample = 0; sample < std::max(1, mSettings.stackSamples); sample++) {
if (sample > 0)
std::this_thread::sleep_for(std::chrono::duration<double, std::milli>(mSettings.stackSampleIntervalMs));
const std::vector<ThreadSample> threads = StackTrace::sampleAllThreads();
for (const ThreadSample& thread : threads)
if (thread.tid == mUiTid) {
uiSamples.push_back(thread.frames);
uiState = thread.state;
uiWait = thread.waitChannel;
}
text << "===== sample " << sample + 1 << " =====\n" << formatThreads(threads, mUiTid);
}
return text.str();
}
std::string HangWatchdog::stackVerdict(
const std::vector<std::vector<uintptr_t>>& uiSamples, const std::string& state, const std::string& waitChannel
) {
if (uiSamples.empty()) return "ui thread not found";
const bool sameStack =
std::all_of(uiSamples.begin(), uiSamples.end(), [&](const auto& frames) { return frames == uiSamples.front(); });
const bool sleeping = !state.empty() && (state[0] == 'S' || state[0] == 'D');
if (sameStack && sleeping) {
std::string text = "blocked: the ui thread sleeps at the same place in every sample";
if (!waitChannel.empty() && waitChannel != "0") text += " (kernel wait " + waitChannel + ")";
return text;
}
if (!sleeping) return "busy: the ui thread is running, check the changing frames in threads.txt";
return "waiting repeatedly: the ui thread sleeps but at different places, likely polling or lock contention";
}
std::vector<std::filesystem::path> HangWatchdog::reports() const {
std::vector<std::filesystem::path> found;
std::error_code error;
for (const auto& entry : std::filesystem::directory_iterator(mSettings.directory, error))
if (entry.is_directory()) found.push_back(entry.path());
std::sort(found.begin(), found.end(), [](const auto& a, const auto& b) {
return std::filesystem::last_write_time(a) < std::filesystem::last_write_time(b);
});
return found;
}
void HangWatchdog::pruneOldReports() const {
const std::vector<std::filesystem::path> found = reports();
if (found.size() <= mSettings.keptReports) return;
std::error_code error;
for (size_t index = 0; index + mSettings.keptReports < found.size(); index++)
std::filesystem::remove_all(found[index], error);
}
}

67
Diagnostics/private/UiTrail.cpp Normal file
View file

@ -0,0 +1,67 @@
#include "UiTrail.hpp"
#include <algorithm>
#include <atomic>
#include <thread>
namespace tp {
namespace {
struct Trail {
std::atomic<const char*> names[UiTrail::kDepth] = {};
std::atomic<int> depth{ 0 };
std::atomic<int> overflow{ 0 };
};
Trail& trail() {
static Trail shared;
return shared;
}
std::atomic<std::thread::id>& trackedThread() {
static std::atomic<std::thread::id> tracked{};
return tracked;
}
}
void UiTrail::push(const char* name) {
if (!isTracked()) return;
Trail& shared = trail();
const int depth = shared.depth.load(std::memory_order_relaxed);
if (depth < kDepth) shared.names[depth].store(name, std::memory_order_relaxed);
else shared.overflow.fetch_add(1, std::memory_order_relaxed);
shared.depth.store(depth + 1, std::memory_order_release);
}
void UiTrail::pop() {
if (!isTracked()) return;
Trail& shared = trail();
const int depth = shared.depth.load(std::memory_order_relaxed);
if (depth <= 0) return;
if (depth > kDepth) shared.overflow.fetch_sub(1, std::memory_order_relaxed);
shared.depth.store(depth - 1, std::memory_order_release);
}
bool UiTrail::isTracked() { return trackedThread().load(std::memory_order_relaxed) == std::this_thread::get_id(); }
void UiTrail::trackCallingThread() {
trail().depth.store(0);
trail().overflow.store(0);
trackedThread().store(std::this_thread::get_id());
}
void UiTrail::stopTracking() { trackedThread().store(std::thread::id{}); }
std::vector<std::string> UiTrail::snapshot() {
Trail& shared = trail();
const int depth = std::min(shared.depth.load(std::memory_order_acquire), kDepth);
std::vector<std::string> names;
for (int index = 0; index < depth; index++) {
const char* name = shared.names[index].load(std::memory_order_relaxed);
names.emplace_back(name ? name : "?");
}
const int overflow = shared.overflow.load(std::memory_order_relaxed);
if (overflow > 0) names.push_back("... " + std::to_string(overflow) + " deeper");
return names;
}
}

19
Diagnostics/private/UiTrail.hpp Normal file
View file

@ -0,0 +1,19 @@
#pragma once
#include <string>
#include <vector>
namespace tp {
class UiTrail {
public:
static constexpr int kDepth = 32;
static void push(const char* name);
static void pop();
[[nodiscard]] static bool isTracked();
static void trackCallingThread();
static void stopTracking();
[[nodiscard]] static std::vector<std::string> snapshot();
};
}

120
Diagnostics/public/HangWatchdog.hpp Normal file
View file

@ -0,0 +1,120 @@
#pragma once
#include "StackTrace.hpp"
#include <atomic>
#include <condition_variable>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
namespace tp {
enum class UiPhase : uint8_t { Idle, Input, Draw, Present };
struct HangSettings {
double thresholdMs = 2000;
std::vector<double> followUpMs = { 10000, 30000 };
int stackSamples = 5;
double stackSampleIntervalMs = 200;
double processPauseMs = 1000;
size_t keptReports = 20;
std::string application = "app";
std::string build = "dev";
std::filesystem::path directory;
std::function<int64_t()> clockNs;
};
struct HangReportSection {
std::string fileName;
std::string text;
};
class HangWatchdog {
public:
using Contributor = std::function<std::vector<HangReportSection>()>;
explicit HangWatchdog(HangSettings settings);
~HangWatchdog();
HangWatchdog(const HangWatchdog&) = delete;
HangWatchdog& operator=(const HangWatchdog&) = delete;
void enterPhase(UiPhase phase);
void noteInputEvent();
void addContributor(Contributor contributor);
[[nodiscard]] const std::filesystem::path& directory() const { return mSettings.directory; }
[[nodiscard]] std::vector<std::filesystem::path> reports() const;
[[nodiscard]] size_t reportsWritten() const { return mReportsWritten.load(); }
[[nodiscard]] uint64_t inspections() const { return mInspections.load(); }
[[nodiscard]] static std::filesystem::path defaultHangsDirectory();
[[nodiscard]] static std::filesystem::path directoryFor(const std::filesystem::path& applicationHangsDirectory);
[[nodiscard]] static std::string processName();
static void allowDebuggerAttach();
[[nodiscard]] static const char* phaseName(UiPhase phase);
private:
struct Stall {
uint64_t beat = 0;
UiPhase phase = UiPhase::Idle;
std::filesystem::path report;
size_t followUps = 0;
};
struct Investigation {
std::string verdict;
std::string threads;
};
void watch();
void inspect(Stall& stall, int64_t now);
void discountProcessPause(int64_t now, int64_t previous);
[[nodiscard]] double stalledMs(int64_t now) const;
[[nodiscard]] uint64_t inputEventsInPhase() const;
void beginReport(Stall& stall, double stalled);
void appendFollowUp(Stall& stall, double stalled);
void finishRecovered(Stall& stall, double stalled);
[[nodiscard]] std::string reportHeader() const;
[[nodiscard]] std::string uiSnapshot(double stalled, UiPhase phase) const;
void writeContributions(const std::filesystem::path& report);
[[nodiscard]] Investigation investigate() const;
[[nodiscard]] std::string
sampleStacks(std::vector<std::vector<uintptr_t>>& uiSamples, std::string& uiState, std::string& uiWait) const;
[[nodiscard]] static std::string formatThreads(const std::vector<ThreadSample>& threads, int uiTid);
[[nodiscard]] static std::string stackVerdict(
const std::vector<std::vector<uintptr_t>>& uiSamples, const std::string& state, const std::string& waitChannel
);
void pruneOldReports() const;
[[nodiscard]] std::filesystem::path newReportPath() const;
[[nodiscard]] int64_t now() const;
HangSettings mSettings;
std::atomic<uint64_t> mBeat{ 0 };
std::atomic<int64_t> mPhaseStartNs{ 0 };
std::atomic<UiPhase> mPhase{ UiPhase::Idle };
std::atomic<uint64_t> mInputEvents{ 0 };
std::atomic<uint64_t> mPhaseInputEvents{ 0 };
std::atomic<size_t> mReportsWritten{ 0 };
std::atomic<uint64_t> mInspections{ 0 };
int mUiTid = 0;
uint64_t mPausedBeat = 0;
int64_t mPausedNs = 0;
std::mutex mContributorsMutex;
std::vector<Contributor> mContributors;
std::mutex mStopMutex;
std::condition_variable mStopped;
bool mStopping = false;
std::thread mThread;
};
}

244
Diagnostics/tests/HangWatchdogTests.cpp Normal file
View file

@ -0,0 +1,244 @@
#include "HangWatchdog.hpp"
#include "UiTrail.hpp"
#include "UnitTest++/UnitTest++.h"
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <condition_variable>
#include <filesystem>
#include <fstream>
#include <mutex>
#include <sstream>
#include <thread>
using namespace tp;
namespace {
struct TempDirectory {
std::filesystem::path path;
explicit TempDirectory(const std::string& name) :
path(std::filesystem::temp_directory_path() / ("hang-watchdog-test-" + name)) {
std::filesystem::remove_all(path);
}
~TempDirectory() {
if (!std::getenv("KEEP_HANG_REPORTS")) std::filesystem::remove_all(path);
}
};
HangSettings fastSettings(const std::filesystem::path& directory) {
HangSettings settings;
settings.thresholdMs = 150;
settings.followUpMs = {};
settings.stackSamples = 3;
settings.stackSampleIntervalMs = 30;
settings.application = "test";
settings.build = "0";
settings.directory = directory;
return settings;
}
std::string readFile(const std::filesystem::path& path) {
std::ifstream file(path);
std::stringstream text;
text << file.rdbuf();
return text.str();
}
struct TestScope {
explicit TestScope(const char* name) { UiTrail::push(name); }
~TestScope() { UiTrail::pop(); }
};
bool hasVerdict(const std::filesystem::path& report) {
return readFile(report / "summary.txt").find("verdict:") != std::string::npos;
}
template <typename Done>
bool waitUntil(Done&& done, double limitMs = 5000) {
const auto start = std::chrono::steady_clock::now();
while (!done()) {
if (std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count() > limitMs)
return false;
std::this_thread::yield();
}
return true;
}
class Gate {
public:
void waitOpen() {
std::unique_lock lock(mMutex);
mOpened.wait(lock, [this]() { return mOpen; });
}
void open() {
{
std::lock_guard lock(mMutex);
mOpen = true;
}
mOpened.notify_all();
}
private:
std::mutex mMutex;
std::condition_variable mOpened;
bool mOpen = false;
};
}
SUITE(HangWatchdog) {
TEST(StuckDrawWritesAReportNamingTheOpenScopes) {
TempDirectory directory("stuck");
HangWatchdog watchdog(fastSettings(directory.path));
Gate gate;
watchdog.enterPhase(UiPhase::Draw);
{
TestScope outer("test.outer");
TestScope waiting("test.waiting_on_gate");
std::thread release([&]() {
waitUntil([&]() { return watchdog.reportsWritten() > 0 && hasVerdict(watchdog.reports().front()); });
gate.open();
});
gate.waitOpen();
release.join();
}
watchdog.enterPhase(UiPhase::Idle);
REQUIRE CHECK_EQUAL(1u, watchdog.reports().size());
const std::string summary = readFile(watchdog.reports().front() / "summary.txt");
CHECK(summary.find("drawing") != std::string::npos);
CHECK(summary.find("test.outer") != std::string::npos);
CHECK(summary.find("test.waiting_on_gate") != std::string::npos);
CHECK(summary.find("verdict: blocked") != std::string::npos);
const std::string threads = readFile(watchdog.reports().front() / "threads.txt");
CHECK(threads.find("UI THREAD") != std::string::npos);
CHECK(
threads.find("waitOpen") != std::string::npos || threads.find("pthread_cond") != std::string::npos ||
threads.find("libc") != std::string::npos
);
CHECK(waitUntil([&]() {
return readFile(watchdog.reports().front() / "summary.txt").find("recovered") != std::string::npos;
}));
}
TEST(SpinningUiThreadIsReportedAsBusy) {
TempDirectory directory("busy");
HangWatchdog watchdog(fastSettings(directory.path));
watchdog.enterPhase(UiPhase::Input);
{
TestScope spinning("test.spinning");
volatile uint64_t work = 0;
while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front()))
work = work + 1;
}
watchdog.enterPhase(UiPhase::Idle);
const std::string summary = readFile(watchdog.reports().front() / "summary.txt");
CHECK(summary.find("test.spinning") != std::string::npos);
CHECK(summary.find("verdict: busy") != std::string::npos);
}
TEST(AProcessPauseIsNotCountedAsAHang) {
TempDirectory directory("paused");
std::atomic<int64_t> clock{ 0 };
HangSettings settings = fastSettings(directory.path);
settings.thresholdMs = 2000;
settings.processPauseMs = 1000;
settings.clockNs = [&clock]() { return clock.load(); };
HangWatchdog watchdog(settings);
watchdog.enterPhase(UiPhase::Draw);
const auto step = [&](int64_t milliseconds) {
const uint64_t seen = watchdog.inspections();
clock.fetch_add(milliseconds * 1000000);
REQUIRE CHECK(waitUntil([&]() { return watchdog.inspections() > seen; }));
};
step(100);
step(30000);
step(100);
CHECK_EQUAL(0u, watchdog.reportsWritten());
for (int tick = 0; tick < 25; tick++)
step(100);
CHECK_EQUAL(1u, watchdog.reportsWritten());
watchdog.enterPhase(UiPhase::Idle);
}
TEST(SummaryIsWrittenBeforeThreadsAreSampled) {
TempDirectory directory("summary-first");
HangSettings settings = fastSettings(directory.path);
std::filesystem::path seenReport;
bool summaryExisted = false;
HangWatchdog watchdog(settings);
watchdog.addContributor([&]() {
seenReport = watchdog.reports().empty() ? std::filesystem::path() : watchdog.reports().front();
summaryExisted = !seenReport.empty() && std::filesystem::exists(seenReport / "summary.txt");
return std::vector<HangReportSection>{};
});
watchdog.enterPhase(UiPhase::Present);
REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; }));
watchdog.enterPhase(UiPhase::Idle);
CHECK(summaryExisted);
const std::string summary = readFile(seenReport / "summary.txt");
CHECK(summary.find("debugger attached no") != std::string::npos);
}
TEST(InputEventsHandledInOnePhaseAreCounted) {
TempDirectory directory("input-loop");
HangWatchdog watchdog(fastSettings(directory.path));
watchdog.enterPhase(UiPhase::Input);
{
TestScope inputLoop("test.input_loop");
while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front()))
watchdog.noteInputEvent();
}
watchdog.enterPhase(UiPhase::Idle);
const std::string summary = readFile(watchdog.reports().front() / "summary.txt");
CHECK(summary.find("input events handled during this phase 0") == std::string::npos);
CHECK(summary.find("input events were handled without leaving the input phase") != std::string::npos);
}
TEST(IdleNeverProducesAReport) {
TempDirectory directory("idle");
HangWatchdog watchdog(fastSettings(directory.path));
watchdog.enterPhase(UiPhase::Idle);
Gate gate;
std::thread release([&]() {
const auto start = std::chrono::steady_clock::now();
waitUntil([&]() { return std::chrono::steady_clock::now() - start > std::chrono::milliseconds(600); });
gate.open();
});
gate.waitOpen();
release.join();
CHECK_EQUAL(0u, watchdog.reportsWritten());
}
TEST(ContributorsAddTheirOwnFiles) {
TempDirectory directory("contributors");
HangWatchdog watchdog(fastSettings(directory.path));
watchdog.addContributor([]() { return std::vector<HangReportSection>{ { "engine.txt", "queue 3\n" } }; });
watchdog.enterPhase(UiPhase::Input);
REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; }));
watchdog.enterPhase(UiPhase::Idle);
CHECK_EQUAL(std::string("queue 3\n"), readFile(watchdog.reports().front() / "engine.txt"));
}
TEST(OnlyTheNewestReportsAreKept) {
TempDirectory directory("retention");
std::filesystem::create_directories(directory.path);
for (int index = 0; index < 5; index++)
std::filesystem::create_directories(directory.path / ("old-" + std::to_string(index)));
HangSettings settings = fastSettings(directory.path);
settings.keptReports = 2;
HangWatchdog watchdog(settings);
CHECK_EQUAL(2u, watchdog.reports().size());
}
TEST(DirectoryIsTheApplicationsHangFolderUnlessOverridden) {
const std::filesystem::path application = "/home/someone/sketch2d/hangs";
CHECK(HangWatchdog::directoryFor(application) == application);
setenv("MODULES_HANG_DIR", "/tmp/custom-hangs", 1);
CHECK(HangWatchdog::directoryFor(application) == std::filesystem::path("/tmp/custom-hangs"));
unsetenv("MODULES_HANG_DIR");
}
}

3
Diagnostics/tests/Main.cpp Normal file
View file

@ -0,0 +1,3 @@
#include "UnitTest++/UnitTest++.h"
int main() { return UnitTest::RunAllTests(); }