diagnostics/ holds what the application loop uses to observe itself: - Profiler samples section durations and counters per frame (wall and CPU time, mean/median/p95/p99). It is an explicit instance with no global; ProfileScope takes the profiler it reports to. GPU time is reported as a section like any other measurement. - UiTrail is the stack of profile scopes open on the frame-loop thread. The profiler owns it and ProfileScope pushes onto it, so a hang report names the scope that stalled. - HangWatchdog writes a report when a non-idle phase stalls: open scope trail, input count, thread stacks sampled through Callstack, and a blocked/busy verdict. Debugger pauses are not counted. Applications add their own state to reports through report file sources.
255 lines
8.2 KiB
C++
255 lines
8.2 KiB
C++
#include "HangWatchdog.hpp"
|
|
|
|
#include "UnitTest++/UnitTest++.h"
|
|
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cstdlib>
|
|
#include <condition_variable>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <mutex>
|
|
#include <sstream>
|
|
#include <thread>
|
|
|
|
using namespace tp;
|
|
|
|
namespace {
|
|
struct TempDirectory {
|
|
std::filesystem::path path;
|
|
|
|
explicit TempDirectory(const std::string& name) :
|
|
path(std::filesystem::temp_directory_path() / ("hang-watchdog-test-" + name)) {
|
|
std::filesystem::remove_all(path);
|
|
}
|
|
~TempDirectory() {
|
|
if (!std::getenv("KEEP_HANG_REPORTS")) {
|
|
std::filesystem::remove_all(path);
|
|
}
|
|
}
|
|
};
|
|
|
|
HangSettings fastSettings(const std::filesystem::path& directory) {
|
|
HangSettings settings;
|
|
settings.thresholdMs = 150;
|
|
settings.followUpMs = {};
|
|
settings.stackSamples = 3;
|
|
settings.stackSampleIntervalMs = 30;
|
|
settings.application = "test";
|
|
settings.build = "0";
|
|
settings.directory = directory;
|
|
return settings;
|
|
}
|
|
|
|
std::string readFile(const std::filesystem::path& path) {
|
|
std::ifstream file(path);
|
|
std::stringstream text;
|
|
text << file.rdbuf();
|
|
return text.str();
|
|
}
|
|
|
|
UiTrail& testTrail() {
|
|
static UiTrail trail;
|
|
return trail;
|
|
}
|
|
|
|
struct TestScope {
|
|
explicit TestScope(const char* name) { testTrail().push(name); }
|
|
~TestScope() { testTrail().pop(); }
|
|
};
|
|
|
|
bool hasVerdict(const std::filesystem::path& report) {
|
|
return readFile(report / "summary.txt").find("verdict:") != std::string::npos;
|
|
}
|
|
|
|
template <typename Done>
|
|
bool waitUntil(Done&& done, double limitMs = 5000) {
|
|
const auto start = std::chrono::steady_clock::now();
|
|
while (!done()) {
|
|
if (std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count() > limitMs) {
|
|
return false;
|
|
}
|
|
std::this_thread::yield();
|
|
}
|
|
return true;
|
|
}
|
|
|
|
class Gate {
|
|
public:
|
|
void waitOpen() {
|
|
std::unique_lock lock(mMutex);
|
|
mOpened.wait(lock, [this]() { return mOpen; });
|
|
}
|
|
void open() {
|
|
{
|
|
std::lock_guard lock(mMutex);
|
|
mOpen = true;
|
|
}
|
|
mOpened.notify_all();
|
|
}
|
|
|
|
private:
|
|
std::mutex mMutex;
|
|
std::condition_variable mOpened;
|
|
bool mOpen = false;
|
|
};
|
|
}
|
|
|
|
SUITE(HangWatchdog) {
|
|
TEST(StuckDrawWritesAReportNamingTheOpenScopes) {
|
|
TempDirectory directory("stuck");
|
|
HangWatchdog watchdog(fastSettings(directory.path), testTrail());
|
|
Gate gate;
|
|
watchdog.enterPhase(UiPhase::Draw);
|
|
{
|
|
TestScope outer("test.outer");
|
|
TestScope waiting("test.waiting_on_gate");
|
|
std::thread release([&]() {
|
|
waitUntil([&]() { return watchdog.reportsWritten() > 0 && hasVerdict(watchdog.reports().front()); });
|
|
gate.open();
|
|
});
|
|
gate.waitOpen();
|
|
release.join();
|
|
}
|
|
watchdog.enterPhase(UiPhase::Idle);
|
|
REQUIRE CHECK_EQUAL(1u, watchdog.reports().size());
|
|
const std::string summary = readFile(watchdog.reports().front() / "summary.txt");
|
|
CHECK(summary.find("drawing") != std::string::npos);
|
|
CHECK(summary.find("test.outer") != std::string::npos);
|
|
CHECK(summary.find("test.waiting_on_gate") != std::string::npos);
|
|
CHECK(summary.find("verdict: blocked") != std::string::npos);
|
|
const std::string threads = readFile(watchdog.reports().front() / "threads.txt");
|
|
CHECK(threads.find("UI THREAD") != std::string::npos);
|
|
CHECK(
|
|
threads.find("waitOpen") != std::string::npos || threads.find("pthread_cond") != std::string::npos ||
|
|
threads.find("libc") != std::string::npos
|
|
);
|
|
CHECK(waitUntil([&]() {
|
|
return readFile(watchdog.reports().front() / "summary.txt").find("recovered") != std::string::npos;
|
|
}));
|
|
}
|
|
|
|
TEST(SpinningUiThreadIsReportedAsBusy) {
|
|
TempDirectory directory("busy");
|
|
HangWatchdog watchdog(fastSettings(directory.path), testTrail());
|
|
watchdog.enterPhase(UiPhase::Input);
|
|
{
|
|
TestScope spinning("test.spinning");
|
|
volatile uint64_t work = 0;
|
|
while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front())) {
|
|
work = work + 1;
|
|
}
|
|
}
|
|
watchdog.enterPhase(UiPhase::Idle);
|
|
const std::string summary = readFile(watchdog.reports().front() / "summary.txt");
|
|
CHECK(summary.find("test.spinning") != std::string::npos);
|
|
CHECK(summary.find("verdict: busy") != std::string::npos);
|
|
}
|
|
|
|
TEST(AProcessPauseIsNotCountedAsAHang) {
|
|
TempDirectory directory("paused");
|
|
std::atomic<int64_t> clock{ 0 };
|
|
HangSettings settings = fastSettings(directory.path);
|
|
settings.thresholdMs = 2000;
|
|
settings.processPauseMs = 1000;
|
|
settings.clockNs = [&clock]() { return clock.load(); };
|
|
HangWatchdog watchdog(settings, testTrail());
|
|
watchdog.enterPhase(UiPhase::Draw);
|
|
|
|
const auto step = [&](int64_t milliseconds) {
|
|
const uint64_t seen = watchdog.inspections();
|
|
clock.fetch_add(milliseconds * 1000000);
|
|
REQUIRE CHECK(waitUntil([&]() { return watchdog.inspections() > seen; }));
|
|
};
|
|
step(100);
|
|
step(30000);
|
|
step(100);
|
|
CHECK_EQUAL(0u, watchdog.reportsWritten());
|
|
|
|
for (int tick = 0; tick < 25; tick++) {
|
|
step(100);
|
|
}
|
|
CHECK_EQUAL(1u, watchdog.reportsWritten());
|
|
watchdog.enterPhase(UiPhase::Idle);
|
|
}
|
|
|
|
TEST(SummaryIsWrittenBeforeThreadsAreSampled) {
|
|
TempDirectory directory("summary-first");
|
|
HangSettings settings = fastSettings(directory.path);
|
|
std::filesystem::path seenReport;
|
|
bool summaryExisted = false;
|
|
HangWatchdog watchdog(settings, testTrail());
|
|
watchdog.addReportFileSource([&]() {
|
|
seenReport = watchdog.reports().empty() ? std::filesystem::path() : watchdog.reports().front();
|
|
summaryExisted = !seenReport.empty() && std::filesystem::exists(seenReport / "summary.txt");
|
|
return std::vector<HangReportFile>{};
|
|
});
|
|
watchdog.enterPhase(UiPhase::Present);
|
|
REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; }));
|
|
watchdog.enterPhase(UiPhase::Idle);
|
|
CHECK(summaryExisted);
|
|
const std::string summary = readFile(seenReport / "summary.txt");
|
|
CHECK(summary.find("debugger attached no") != std::string::npos);
|
|
}
|
|
|
|
TEST(InputEventsHandledInOnePhaseAreCounted) {
|
|
TempDirectory directory("input-loop");
|
|
HangWatchdog watchdog(fastSettings(directory.path), testTrail());
|
|
watchdog.enterPhase(UiPhase::Input);
|
|
{
|
|
TestScope inputLoop("test.input_loop");
|
|
while (watchdog.reportsWritten() == 0 || !hasVerdict(watchdog.reports().front())) {
|
|
watchdog.noteInputEvents();
|
|
}
|
|
}
|
|
watchdog.enterPhase(UiPhase::Idle);
|
|
const std::string summary = readFile(watchdog.reports().front() / "summary.txt");
|
|
CHECK(summary.find("input events handled during this phase 0") == std::string::npos);
|
|
CHECK(summary.find("input events were handled without leaving the input phase") != std::string::npos);
|
|
}
|
|
|
|
TEST(IdleNeverProducesAReport) {
|
|
TempDirectory directory("idle");
|
|
HangWatchdog watchdog(fastSettings(directory.path), testTrail());
|
|
watchdog.enterPhase(UiPhase::Idle);
|
|
Gate gate;
|
|
std::thread release([&]() {
|
|
const auto start = std::chrono::steady_clock::now();
|
|
waitUntil([&]() { return std::chrono::steady_clock::now() - start > std::chrono::milliseconds(600); });
|
|
gate.open();
|
|
});
|
|
gate.waitOpen();
|
|
release.join();
|
|
CHECK_EQUAL(0u, watchdog.reportsWritten());
|
|
}
|
|
|
|
TEST(ReportFileSourcesAddTheirOwnFiles) {
|
|
TempDirectory directory("reportFileSources");
|
|
HangWatchdog watchdog(fastSettings(directory.path), testTrail());
|
|
watchdog.addReportFileSource([]() { return std::vector<HangReportFile>{ { "engine.txt", "queue 3\n" } }; });
|
|
watchdog.enterPhase(UiPhase::Input);
|
|
REQUIRE CHECK(waitUntil([&]() { return watchdog.reportsWritten() > 0; }));
|
|
watchdog.enterPhase(UiPhase::Idle);
|
|
CHECK_EQUAL(std::string("queue 3\n"), readFile(watchdog.reports().front() / "engine.txt"));
|
|
}
|
|
|
|
TEST(OnlyTheNewestReportsAreKept) {
|
|
TempDirectory directory("retention");
|
|
std::filesystem::create_directories(directory.path);
|
|
for (int index = 0; index < 5; index++) {
|
|
std::filesystem::create_directories(directory.path / ("old-" + std::to_string(index)));
|
|
}
|
|
HangSettings settings = fastSettings(directory.path);
|
|
settings.keptReports = 2;
|
|
HangWatchdog watchdog(settings, testTrail());
|
|
CHECK_EQUAL(2u, watchdog.reports().size());
|
|
}
|
|
|
|
TEST(DirectoryIsTheApplicationsHangFolderUnlessOverridden) {
|
|
const std::filesystem::path application = "/home/someone/sketch2d/hangs";
|
|
CHECK(HangWatchdog::directoryFor(application) == application);
|
|
setenv("MODULES_HANG_DIR", "/tmp/custom-hangs", 1);
|
|
CHECK(HangWatchdog::directoryFor(application) == std::filesystem::path("/tmp/custom-hangs"));
|
|
unsetenv("MODULES_HANG_DIR");
|
|
}
|
|
}
|