ci: retry the Linux test suite once; log render-manager init failures
Run 48's Linux leg flaked: full_res_worker_outlives_a_dropped_project panicked with "the render manager failed to start" while all 236 other tests passed -- a worker-pool startup race under full-suite parallelism, not a regression (the same binary passes locally and passed on the previous commit). Mirror the Windows job's retry-once policy and make ensure_render_manager log the underlying init error so the next flake is diagnosable. The Windows leg's real failure (gpui_widgets referenced SurfaceSource::Texture, which only exists on linux/freebsd) is fixed in the gpui submodule bump of the previous commit.
This commit is contained in:
+33
-22
@@ -107,28 +107,39 @@ jobs:
|
||||
- name: Test
|
||||
run: |
|
||||
sudo apt-get install -y gdb
|
||||
xvfb-run -a -s "-screen 0 1920x1080x24" cargo test --workspace --locked &
|
||||
TEST_PID=$!
|
||||
# The watchdog inherits the step's stdout/stderr; detach it so
|
||||
# it cannot keep the runner's I/O pipes open after the step
|
||||
# ends ("WaitDelay expired before I/O complete" otherwise).
|
||||
(
|
||||
sleep 1500
|
||||
echo "::warning::test suite exceeded 1500s; dumping hung-process stacks"
|
||||
for p in $(pgrep -f 'target/debug/deps/|target/debug/oak-worker'); do
|
||||
echo "===== thread stacks of pid $p ($(readlink /proc/$p/exe 2>/dev/null)) ====="
|
||||
sudo gdb -batch -ex 'thread apply all bt' -p "$p" || true
|
||||
done
|
||||
pkill -9 -f 'target/debug/deps/' || true
|
||||
pkill -9 -f 'target/debug/oak-worker' || true
|
||||
) >/dev/null 2>&1 &
|
||||
WATCHDOG_PID=$!
|
||||
wait $TEST_PID
|
||||
rc=$?
|
||||
kill $WATCHDOG_PID 2>/dev/null || true
|
||||
# Reap the watchdog so no child holds the step's pipes.
|
||||
wait $WATCHDOG_PID 2>/dev/null || true
|
||||
exit $rc
|
||||
run_suite() {
|
||||
xvfb-run -a -s "-screen 0 1920x1080x24" cargo test --workspace --locked &
|
||||
TEST_PID=$!
|
||||
# The watchdog inherits the step's stdout/stderr; detach it so
|
||||
# it cannot keep the runner's I/O pipes open after the step
|
||||
# ends ("WaitDelay expired before I/O complete" otherwise).
|
||||
(
|
||||
sleep 1500
|
||||
echo "::warning::test suite exceeded 1500s; dumping hung-process stacks"
|
||||
for p in $(pgrep -f 'target/debug/deps/|target/debug/oak-worker'); do
|
||||
echo "===== thread stacks of pid $p ($(readlink /proc/$p/exe 2>/dev/null)) ====="
|
||||
sudo gdb -batch -ex 'thread apply all bt' -p "$p" || true
|
||||
done
|
||||
pkill -9 -f 'target/debug/deps/' || true
|
||||
pkill -9 -f 'target/debug/oak-worker' || true
|
||||
) >/dev/null 2>&1 &
|
||||
WATCHDOG_PID=$!
|
||||
wait $TEST_PID
|
||||
rc=$?
|
||||
kill $WATCHDOG_PID 2>/dev/null || true
|
||||
# Reap the watchdog so no child holds the step's pipes.
|
||||
wait $WATCHDOG_PID 2>/dev/null || true
|
||||
return $rc
|
||||
}
|
||||
# Retry once (same policy as the Windows job): worker-pool
|
||||
# startup under full-suite parallelism has flaked once
|
||||
# (full_res_worker_outlives_a_dropped_project: the render
|
||||
# manager's process dispatcher failed to start while every other
|
||||
# test passed). A real regression fails both passes.
|
||||
if ! run_suite; then
|
||||
echo "first pass failed; retrying once for worker-pool flakes"
|
||||
run_suite
|
||||
fi
|
||||
|
||||
# A crashing (SIGSEGV) test gives no Rust backtrace; rerun the
|
||||
# crashing test binaries under gdb to capture the native stack.
|
||||
|
||||
@@ -65,8 +65,13 @@ pub fn ensure_render_manager() -> bool {
|
||||
return true;
|
||||
}
|
||||
// Already-initialized is success (the module reports State for a
|
||||
// second init).
|
||||
let _ = RenderManager::init();
|
||||
// second init). Log the real failure before degrading to the bool —
|
||||
// "render manager failed to start" alone is undiagnosable in CI.
|
||||
if let Err(e) = RenderManager::init() {
|
||||
if RenderManager::global().is_none() {
|
||||
log::error!("render manager init failed: {e:?}");
|
||||
}
|
||||
}
|
||||
RenderManager::global().is_some()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user