multicam: wizard + AFV + node-graph preview + performance
CI / Build & test (Linux) (push) Failing after 24s
CI / Build & test (Windows) (push) Failing after 3m38s

- wizard: angle multi-select, sync modes, auto-align, create sequence;
  keeps the host sequence current (host clip = the multicam clip)
- build_multicam_sequence: per-angle clip -> SOURCES_INPUT[element],
  array slot growth, audio angle tracks for AFV
- AFV: host linked audio clip follows the switched source (one undo),
  muted host audio track disables it
- node-graph preview: inline producer renders through the traverser
  (viewer/project-matched graph frames); sequence viewer uses the graph
- multicam node value(): element-tagged row keys (sources_in[i]),
  reads the current source; build_row keys array inputs by element
- angle grid: viewer=0 (single-track montage, not whole-graph)
- project explorer: rename (dialog) + delete (undoable) real items
- performance: decoded-frame LRU, per-process NVDEC quota (1 session,
  evict before open), GPU composite fail-once fallback, snapshot
  upload debounced on the engine tick, worker vram budget headroom
- timeline clip: multicam overlay via ClipDecorator
- wizard menu item moved to Sequence menu
This commit is contained in:
2026-09-01 20:27:59 +08:00
parent 8de9af705e
commit 1b0a15192e
25 changed files with 3528 additions and 65 deletions
+135 -14
View File
@@ -801,11 +801,48 @@ static DECODERS: std::sync::OnceLock<
/// "4K 切换后大量 CUDA_ERROR_OUT_OF_MEMORY 报错" log flood). Software
/// sessions (system RAM only) are evicted only when nothing else is
/// available.
const MAX_CACHED_DECODERS: usize = 16;
const MAX_CACHED_DECODERS: usize = 6;
/// LRU tick source for [`DECODERS`].
static DECODER_TICK: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
/// Cap on decoded FRAMES cached per process (release builds of the graph
/// renderer re-decode every frame the pre-render window pulls — each is a
/// seek+decode on the shared decoder session, which serializes the graph
/// path behind the montage baseline. A small frame LRU lets playback pull
/// the forward window from cache instead of thrashing the decoder: the
/// window is sequential, so a short FIFO of recently decoded frames hits
/// on every pre-render restart and on repeat plays).
const MAX_CACHED_FRAMES: usize = 24;
/// Decoded-frame LRU: `(filename, stream, time, w, h)` -> F32 CPU frame.
/// Frame data is the expensive part (a 1080p frame ≈ 31 MB); the decoder
/// session cache alone still re-decodes every `render_footage_frame`.
/// The size is part of the key: the same media at a different target
/// resolution is a different frame (an interleaved source-monitor/proxy
/// request must not reuse a wrongly-sized pixel buffer).
static DECODED_FRAMES: std::sync::OnceLock<
std::sync::Mutex<
std::collections::HashMap<(String, i32, (i64, i64), i32, i32), (Frame, u64)>,
>,
> = std::sync::OnceLock::new();
fn decoded_frames(
) -> std::sync::MutexGuard<
'static,
std::collections::HashMap<(String, i32, (i64, i64), i32, i32), (Frame, u64)>,
> {
DECODED_FRAMES
.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
.lock()
.unwrap_or_else(|e| e.into_inner())
}
/// Time-key rational (the frame-LRU's deterministic key half).
fn time_key(t: &Rational) -> (i64, i64) {
(t.numerator(), t.denominator())
}
fn decoders(
) -> std::sync::MutexGuard<
'static,
@@ -853,6 +890,33 @@ fn open_decoder(filename: &str, stream_index: i32) -> Result<Arc<dyn oak_codec::
}
let decoder: Arc<dyn oak_codec::decoder::Decoder> = Arc::new(FFmpegDecoder::new());
let stream = CodecStream::with_block(filename.to_string(), stream_index, None);
// NVDEC surface-pool budget BEFORE opening: each HARDWARE session
// pins big GPU memory chunks (a 1080p-4K HEVC pool ≈ 300-1000 MB).
// Several worker processes × sessions each exceed even a 16 GB card —
// the `CUDA_ERROR_OUT_OF_MEMORY` log flood. Per process O N L Y one
// hardware session may coexist (the video being played is shared, so
// one NVDEC context serves every montage/graph frame); when one is
// already open, evict it now so the new open has the VRAM budget.
// Eviction before open matters: post-open eviction is too late — the
// failing open was already trying to allocate.
{
let mut cache = decoders();
const MAX_HARDWARE_SESSIONS: usize = 1;
let hw_count = cache
.iter()
.filter(|(_, (d, _))| d.hardware_decoding())
.count();
if hw_count >= MAX_HARDWARE_SESSIONS {
if let Some(victim) = cache
.iter()
.filter(|(_, (d, _))| d.hardware_decoding())
.min_by_key(|(_, (_, t))| *t)
.map(|(k, _)| k.clone())
{
cache.remove(&victim);
}
}
}
decoder
.open(&stream)
.map_err(|e| Error::Failed(format!("footage decode open: {e:?}")))?;
@@ -874,15 +938,32 @@ fn open_decoder(filename: &str, stream_index: i32) -> Result<Arc<dyn oak_codec::
/// Decode the footage frame at `time` and copy/scale it into an
/// oakrender F32 frame of `(w, h)`.
pub fn render_footage_frame(
filename: &str,
stream_index: i32,
time: Rational,
size: (i32, i32),
format: PixelFormat,
) -> Result<Texture> {
let decoder = open_decoder(filename, stream_index)?;
let (w, h) = size;
pub fn render_footage_frame(
filename: &str,
stream_index: i32,
time: Rational,
size: (i32, i32),
format: PixelFormat,
) -> Result<Texture> {
// Decoded frame LRU: playback asks for the same (or nearby)
// (file, stream, time) repeatedly (pre-render restarts, repeated
// scale-up at the same time, graph + montage interleaving); the
// decode itself is the expensive part and must not re-run per
// request. The target size is part of the key (see the cache
// type's doc).
let (w, h) = size;
let cache_size = if w > 0 && h > 0 { (w, h) } else { (-1, -1) };
let cache_key = (filename.to_string(), stream_index, time_key(&time), cache_size.0, cache_size.1);
{
let mut cache = decoded_frames();
let tick = DECODER_TICK.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let frame = cache.get(&cache_key).map(|(f, _)| f.clone());
if let Some(frame) = frame {
cache.insert(cache_key.clone(), (frame.clone(), tick));
return Ok(Texture::wrap_frame(frame));
}
}
let decoder = open_decoder(filename, stream_index)?;
let params = RetrieveVideoParams {
stream: CodecStream::with_block(filename.to_string(), stream_index, None),
time,
@@ -943,6 +1024,24 @@ pub fn render_footage_frame(
// Input node: source colorspace → the pipeline working space (ACEScg
// by default; the legacy sRGB working space keeps the pass-through).
convert_decoded_to_working(&mut dst, &decoded);
// Memoize the finished working-space frame (LRU-capped).
{
let mut cache = decoded_frames();
let tick = DECODER_TICK.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if !cache.contains_key(&cache_key) {
if cache.len() >= MAX_CACHED_FRAMES {
if let Some(victim) = cache
.iter()
.filter(|(_, (_, t))| *t > 0)
.min_by_key(|(_, (_, t))| *t)
.map(|(k, _)| k.clone())
{
cache.remove(&victim);
}
}
cache.insert(cache_key.clone(), (dst.clone(), tick));
}
}
Ok(Texture::wrap_frame(dst))
}
@@ -1056,15 +1155,37 @@ fn composite_tracks_gpu(
/// CPU composite of `frames` into one `size` frame — the fallback when no
/// GPU device is available (or the pass fails): bottom (last) to top
/// (first) via [`composite_over`].
///
/// GPU failures are remembered: a device whose wgpu pipeline fails
/// validation (e.g. an adapter that advertises ComputePipeline yet lacks
/// the required features) fails EVERY frame otherwise — each attempt
/// recompiles the shader, surfaces a validation error and stalls the
/// playback tick (the "picture barely updates on NVIDIA" report). After
/// one failure the composite stays on the CPU path for the process.
static GPU_COMPOSITE_FAILED: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(false);
fn composite_tracks(frames: Vec<Frame>, size: (i32, i32)) -> Frame {
let (w, h) = size;
if w <= 0 || h <= 0 {
return Frame::dummy();
}
if let Some(ctx) = crate::backend::GpuContext::shared() {
match composite_tracks_gpu(&ctx, &frames, (w, h)) {
Ok(frame) => return frame,
Err(err) => eprintln!("GPU track composite failed, using CPU: {err:#}"),
if !GPU_COMPOSITE_FAILED.load(std::sync::atomic::Ordering::Relaxed) {
// Single-frame composites (the common preview case) stay on the
// CPU path: the GPU route costs an upload+download round trip per
// frame for no benefit until two or more tracks overlap.
if frames.len() > 1 {
if let Some(ctx) = crate::backend::GpuContext::shared() {
match composite_tracks_gpu(&ctx, &frames, (w, h)) {
Ok(frame) => return frame,
Err(err) => {
eprintln!(
"GPU track composite failed, using CPU (and staying there): {err:#}"
);
GPU_COMPOSITE_FAILED.store(true, std::sync::atomic::Ordering::Relaxed);
}
}
}
}
}
let Ok(mut acc) = generate_frame(Rational::new(0, 1), (w, h), PixelFormat::F32) else {
+72 -1
View File
@@ -83,7 +83,12 @@ pub struct RenderManager {
/// for (M16 S1: dedup key — revisions alone collide across projects,
/// since every fresh project shares small revision numbers).
current_key: Mutex<Option<(String, u64)>>,
/// Teardown in progress (M16 S1): set first thing in
/// The live project the current snapshot was written from — the
/// INLINE (test) backend's graph-mode render source. The process
/// backend loads the snapshot file in its workers instead; this slot
/// only exists so `render_produced_frame` can run the same
/// [`oak_node::traverser`] graph evaluation without forking a child.
inline_graph: Mutex<Option<(String, std::sync::Arc<std::sync::Mutex<oak_node::project::Project>>)>>, /// Teardown in progress (M16 S1): set first thing in
/// [`RenderManager::shutdown`]; `set_graph_snapshot` /
/// `clear_graph_snapshot` become no-ops afterwards so a stale push
/// from a dying test/app cannot re-arm the worker pool mid-shutdown.
@@ -112,6 +117,35 @@ impl RenderManager {
}
let backend = BackendKind::from_user_config();
let producer: crate::ticket::Producer = Arc::new(|time, params| {
// M16 S1 graph mode on the INLINE (test) backend: when the
// ticket names a viewer of the snapshotted project, evaluate
// the node graph (the same path the oak-worker pool runs in
// the process backend) instead of the montage shortcut — the
// preview must show the node graph, not the decoded-only
// montage.
if params.viewer != 0 && !params.project.is_empty() {
if let Some(m) = crate::manager::RenderManager::global() {
if let Some((uuid, project)) = m.inline_graph() {
if uuid == params.project {
if let Some(viewer_id) =
oak_node::id::NodeId::from_identity(params.viewer)
{
if let Ok(texture) = eval::render_graph_frame(
&project,
viewer_id,
time,
params.render_size(),
oak_core::PixelFormat::F32,
) {
return Ok(crate::ticket::TicketPayload::Video(
texture,
));
}
}
}
}
}
}
eval::render_produced_frame(time, params)
.map(crate::ticket::TicketPayload::Video)
});
@@ -163,12 +197,22 @@ impl RenderManager {
snapshots: GraphSnapshotStore::new(),
current_snapshot: Mutex::new(None),
current_key: Mutex::new(None),
inline_graph: Mutex::new(None),
stopping: AtomicBool::new(false),
process_pool,
}));
Ok(())
}
/// The inline backend's graph-mode render source: the live project the
/// current snapshot was written for, with its uuid. `None` when there
/// is no snapshot (or the process pool answers frames instead).
pub fn inline_graph(
&self,
) -> Option<(String, std::sync::Arc<std::sync::Mutex<oak_node::project::Project>>)> {
lock(&self.inline_graph).clone()
}
/// Global access; `None` before init.
pub fn global() -> Option<Arc<RenderManager>> {
lock(&MANAGER).clone()
@@ -226,6 +270,12 @@ impl RenderManager {
return Ok(()); // teardown: no re-arm after the drain
}
let uuid = lock(project).uuid.clone();
if std::env::var_os("OAK_DEBUG_SNAPSHOT").is_some() {
eprintln!(
"[snapshot] set_graph_snapshot uuid {uuid} revision {revision} key {:?}",
*lock(&self.current_key)
);
}
if *lock(&self.current_key) == Some((uuid.clone(), revision)) {
return Ok(()); // unchanged state: no rewrite, no re-send
}
@@ -234,10 +284,30 @@ impl RenderManager {
self.snapshots.release(&old);
}
self.dispatch.set_graph_snapshot(Some(path));
// Keep a live graph handle for the inline backend's graph mode
// (the same snapshot content, in-process, no file required).
// `project` arrives by reference (the snapshot store's lifetime);
// oak-app installs the Arc via [`RenderManager::set_inline_project`]
// — the inline graph slot is only armed there, never from a
// borrowed reference.
*lock(&self.current_key) = Some((uuid, revision));
Ok(())
}
/// Installs the live project the inline backend's graph mode renders
/// from (oak-app calls this on project adoption; it also arms the uuid
/// the snapshot dedups against). The process backend ignores it.
pub fn set_inline_project(
&self,
project: std::sync::Arc<std::sync::Mutex<oak_node::project::Project>>,
) {
let uuid = match project.lock() {
Ok(g) => g.uuid.clone(),
Err(p) => p.into_inner().uuid.clone(),
};
*lock(&self.inline_graph) = Some((uuid, project));
}
/// M16 S1 graph mode: force the workers to re-load the current project
/// snapshot even when the undo-stack revision is unchanged. The
/// color-settings dialog writes project settings directly (no undo
@@ -277,6 +347,7 @@ impl RenderManager {
self.snapshots.release(&old);
}
*lock(&self.current_key) = None;
*lock(&self.inline_graph) = None;
self.dispatch.set_graph_snapshot(None);
}
+10 -1
View File
@@ -502,7 +502,16 @@ fn per_worker_gpu_budget(frame_size: (i32, i32), fps: u32) -> u64 {
let pixels = (frame_size.0.max(0) as f64) * (frame_size.1.max(0) as f64);
let pixel_ratio = pixels / (1920.0 * 1080.0);
let fps_factor = (fps.max(1) as f64 / 24.0).sqrt().max(1.0);
let peak = 1u64 << 30;
// Peak-vs-quiet spread: a worker sustains a NVDEC session (surface
// pool + an in-flight frame upload) plus the wgpu pipeline and a
// couple of montage uploads. The 1 GiB 1080p / 4 GiB 4K estimates
// have been shown to still exhaust a 16 GB card when the pool pushes
// them simultaneously on several sources (the CUDA_ERROR_OUT_OF_MEMORY
// mid-playback flood), so the budget includes an extra headroom
// factor for undetected spikes (shared decoder surface growth during
// long GOP scans at a keyframe miss).
let headroom = 2;
let peak = (1u64 << 30) * headroom;
let idle = 256u64 << 20;
((peak as f64 * pixel_ratio * fps_factor + idle as f64) as u64).max(1)
}