From ca18105c8ba5bb227938f9b6d1ef7e4a5075aea7 Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Sat, 29 Aug 2026 00:37:24 +0800 Subject: [PATCH] ci: retry the Linux test suite once; log render-manager init failures Run 48's Linux leg flaked: full_res_worker_outlives_a_dropped_project panicked with "the render manager failed to start" while all 236 other tests passed -- a worker-pool startup race under full-suite parallelism, not a regression (the same binary passes locally and passed on the previous commit). Mirror the Windows job's retry-once policy and make ensure_render_manager log the underlying init error so the next flake is diagnosable. The Windows leg's real failure (gpui_widgets referenced SurfaceSource::Texture, which only exists on linux/freebsd) is fixed in the gpui submodule bump of the previous commit. --- .gitea/workflows/ci.yml | 55 ++++++++++++++++----------- crates/oak-app/src/oakui/renderops.rs | 9 ++++- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index a21892f31..b1600c043 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -107,28 +107,39 @@ jobs: - name: Test run: | sudo apt-get install -y gdb - xvfb-run -a -s "-screen 0 1920x1080x24" cargo test --workspace --locked & - TEST_PID=$! - # The watchdog inherits the step's stdout/stderr; detach it so - # it cannot keep the runner's I/O pipes open after the step - # ends ("WaitDelay expired before I/O complete" otherwise). - ( - sleep 1500 - echo "::warning::test suite exceeded 1500s; dumping hung-process stacks" - for p in $(pgrep -f 'target/debug/deps/|target/debug/oak-worker'); do - echo "===== thread stacks of pid $p ($(readlink /proc/$p/exe 2>/dev/null)) =====" - sudo gdb -batch -ex 'thread apply all bt' -p "$p" || true - done - pkill -9 -f 'target/debug/deps/' || true - pkill -9 -f 'target/debug/oak-worker' || true - ) >/dev/null 2>&1 & - WATCHDOG_PID=$! - wait $TEST_PID - rc=$? - kill $WATCHDOG_PID 2>/dev/null || true - # Reap the watchdog so no child holds the step's pipes. - wait $WATCHDOG_PID 2>/dev/null || true - exit $rc + run_suite() { + xvfb-run -a -s "-screen 0 1920x1080x24" cargo test --workspace --locked & + TEST_PID=$! + # The watchdog inherits the step's stdout/stderr; detach it so + # it cannot keep the runner's I/O pipes open after the step + # ends ("WaitDelay expired before I/O complete" otherwise). + ( + sleep 1500 + echo "::warning::test suite exceeded 1500s; dumping hung-process stacks" + for p in $(pgrep -f 'target/debug/deps/|target/debug/oak-worker'); do + echo "===== thread stacks of pid $p ($(readlink /proc/$p/exe 2>/dev/null)) =====" + sudo gdb -batch -ex 'thread apply all bt' -p "$p" || true + done + pkill -9 -f 'target/debug/deps/' || true + pkill -9 -f 'target/debug/oak-worker' || true + ) >/dev/null 2>&1 & + WATCHDOG_PID=$! + wait $TEST_PID + rc=$? + kill $WATCHDOG_PID 2>/dev/null || true + # Reap the watchdog so no child holds the step's pipes. + wait $WATCHDOG_PID 2>/dev/null || true + return $rc + } + # Retry once (same policy as the Windows job): worker-pool + # startup under full-suite parallelism has flaked once + # (full_res_worker_outlives_a_dropped_project: the render + # manager's process dispatcher failed to start while every other + # test passed). A real regression fails both passes. + if ! run_suite; then + echo "first pass failed; retrying once for worker-pool flakes" + run_suite + fi # A crashing (SIGSEGV) test gives no Rust backtrace; rerun the # crashing test binaries under gdb to capture the native stack. diff --git a/crates/oak-app/src/oakui/renderops.rs b/crates/oak-app/src/oakui/renderops.rs index d029ae77a..98db74d94 100644 --- a/crates/oak-app/src/oakui/renderops.rs +++ b/crates/oak-app/src/oakui/renderops.rs @@ -65,8 +65,13 @@ pub fn ensure_render_manager() -> bool { return true; } // Already-initialized is success (the module reports State for a - // second init). - let _ = RenderManager::init(); + // second init). Log the real failure before degrading to the bool — + // "render manager failed to start" alone is undiagnosable in CI. + if let Err(e) = RenderManager::init() { + if RenderManager::global().is_none() { + log::error!("render manager init failed: {e:?}"); + } + } RenderManager::global().is_some() }