feat(oakrender): render-process isolation S2 - process backend by default, zero-copy onscreen

- WorkerPool thread pool deleted; RenderManager defaults to the
  Processes backend (oak-worker children), Threads kept as a test-only
  inline dispatcher; audio tickets stay in-process until S3.
- Onscreen path reads worker shm slots directly: BGRA8 slot format,
  RenderedFrame::Shm wrapped into the display buffer (single disclosed
  GPU-staging memcpy), scopes analyze BGRA8; the long-lived full-res /
  thumbnail paths take the counted slot_to_vec copy and release.
- Playback pre-render window: forward 120 frames (configurable) fed to
  the PreviewScheduler at Playback priority, interleaved across
  workers, cached in shm slots until the playhead consumes them;
  generation-based invalidation cancels and releases on edits.
- oaktask export and oak-cli run on private ProcessDispatchers (fixed
  a pump-while-locked self-deadlock in the export loop); facade
  get_frame handles ShmFrame payloads.
- Acceptance: preview path main_heap_frame_copies == 0 with spawned
  workers, CLI transcode/render verified end to end.
This commit is contained in:
2026-08-18 20:45:24 +08:00
parent 74b080f88a
commit cad1d93544
22 changed files with 1644 additions and 671 deletions
+22
View File
@@ -113,6 +113,28 @@ pub(crate) fn f32_rgba_to_bgra_image(width: u32, height: u32, samples: &[f32]) -
RenderImage::new(smallvec::SmallVec::from_elem(image::Frame::new(buffer), 1))
}
/// Wraps a BGRA8 pixel block into the viewers' display image (M15 S2
/// zero-copy onscreen path). The bytes come straight from a worker's
/// shared-memory slot (already in display order) — this is the
/// GPU-upload staging buffer, the single permitted main-process copy on
/// the preview path (design §3.5). Returns `None` when `bytes` is
/// shorter than `width * height * 4`.
pub(crate) fn bgra_bytes_to_render_image(
width: u32,
height: u32,
bytes: &[u8],
) -> Option<RenderImage> {
let need = (width * height * 4) as usize;
if bytes.len() < need {
return None;
}
let buffer = image::RgbaImage::from_raw(width, height, bytes[..need].to_vec())?;
Some(RenderImage::new(smallvec::SmallVec::from_elem(
image::Frame::new(buffer),
1,
)))
}
#[cfg(test)]
mod tests {
use super::*;
+520 -69
View File
@@ -59,7 +59,7 @@
//! worker holds the project's `Arc`, so a project drop mid-render is a
//! non-event (the drained frame is discarded by the generation check).
use std::collections::{BTreeSet, HashMap, HashSet};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::mpsc;
use std::sync::{Arc, Mutex};
@@ -80,15 +80,17 @@ use gpui_widgets::viewer::PlaybackClock;
use oaknode::id::NodeId;
use oaknode::track::TrackType;
use oakrender::manager::RenderManager;
use oakrender::procpool::{bgra8_to_rgba8, ShmFrameRef};
use oaktimeline::handle::CHandle;
use super::engine::{
AppEngine, EngineGateway, ExportSession, LibraryProject, Monitor, Project, ScopeData, Sequence,
VideoFormat,
};
use super::frames::{f32_rgba_to_bgra_image, synthetic_frame_samples};
use super::frames::{bgra_bytes_to_render_image, f32_rgba_to_bgra_image, synthetic_frame_samples};
use super::graphops::{self, ProjectRef};
use super::scopes::analyze_f32_rgba;
use super::scopes::{analyze_bgra8, analyze_f32_rgba};
use super::transport::TransportState;
/// The project name of a blank project before it is saved.
@@ -235,6 +237,42 @@ enum FullResTarget {
Footage(NodeId),
}
// ---------------------------------------------------------------------------
// Playback pre-render window (M15 S2)
// ---------------------------------------------------------------------------
//
// During playback the engine feeds the forward window (default 120 frames,
// config `PlaybackPreRenderFrames`) to the `PreviewScheduler` through the
// process dispatcher: the scheduler interleaves the frames across the
// workers, so by the time the playhead reaches a frame its pixels are
// already sitting in a shm slot. `cpu_frame` hits this slot cache first
// (zero copy — build the display image from the slot bytes, then release
// the slot); the synchronous render path is the miss fallback. Frames
// that fall out of the window (or whose params were invalidated) release
// their slots back to the workers.
/// The default forward pre-render window (frames).
const DEFAULT_PREVIEW_WINDOW_FORWARD: i64 = 120;
/// Config key for the forward pre-render window size (frames).
const CONFIG_KEY_PREVIEW_WINDOW: &str = "PlaybackPreRenderFrames";
/// One monitor's playback pre-render state (UI-thread-owned; the completions
/// delivered by the dispatcher's poll run on the UI thread too).
#[derive(Default)]
struct PreviewWindow {
/// The node identity the window renders (sequence / footage).
sequence: u64,
/// The render-params generation the window was built for (bumped on
/// invalidation; a mismatch cancels the old requests and rebuilds).
generation: u64,
/// Frames already submitted to the scheduler (pending or in flight).
submitted: BTreeSet<i64>,
/// Rendered frames held in shm slots, keyed by frame number. A frame
/// is consumed by `cpu_frame` (slot released after the display image
/// is built) or released when it falls out of the window.
slots: BTreeMap<i64, ShmFrameRef>,
}
/// One background full-resolution render request (built on the UI thread
/// at schedule time; the worker thread owns it from there).
struct FullResRequest {
@@ -323,28 +361,70 @@ fn thumbnail_path(filename: &str) -> PathBuf {
// Frame conversion
// ---------------------------------------------------------------------------
/// Repack one F32 RGBA rendered frame (rows padded to linesize) into
/// tightly packed samples. Returns `(width, height, samples)` when the
/// frame is well-formed (positive geometry, the pipeline's F32 format).
/// Repack one in-process F32 RGBA rendered frame (rows padded to
/// linesize) into tightly packed samples. Returns `(width, height,
/// samples)` when the frame is the in-process F32 variant (the shm
/// variant is BGRA8 and is read as bytes instead).
fn read_f32_frame(frame: &super::renderops::RenderedFrame) -> Option<(u32, u32, Vec<f32>)> {
let (width, height, linesize) = (frame.width, frame.height, frame.linesize);
if width <= 0 || height <= 0 || frame.format != super::renderops::PIXEL_FORMAT_F32 {
let super::renderops::RenderedFrame::CpuF32 {
width,
height,
linesize,
data,
} = frame
else {
return None;
};
if *width <= 0 || *height <= 0 {
return None;
}
let row_bytes = (width * 4 * 4) as usize;
let linesize = (linesize as usize).max(row_bytes);
if frame.data.len() < linesize * height as usize {
let row_bytes = (*width * 4 * 4) as usize;
let linesize = (*linesize as usize).max(row_bytes);
if data.len() < linesize * *height as usize {
return None;
}
let mut samples = vec![0.0f32; (width * height * 4) as usize];
for y in 0..height as usize {
let row = &frame.data[y * linesize..y * linesize + row_bytes];
let mut samples = vec![0.0f32; (*width * *height * 4) as usize];
for y in 0..*height as usize {
let row = &data[y * linesize..y * linesize + row_bytes];
for (i, px) in row.chunks_exact(4).enumerate() {
let v = f32::from_ne_bytes([px[0], px[1], px[2], px[3]]);
samples[y * (width as usize) * 4 + i] = v;
samples[y * (*width as usize) * 4 + i] = v;
}
}
Some((*width as u32, *height as u32, samples))
}
/// Release the shm slot a rendered frame holds, if any (M15 S2: the
/// zero-copy onscreen path builds the display image from the slot bytes,
/// then returns the slot to its worker — slot release = cache eviction).
/// No-op for in-process frames and when the manager is down.
fn release_rendered_frame(rendered: &super::renderops::RenderedFrame) {
if let super::renderops::RenderedFrame::Shm(frame) = rendered {
if let Some(m) = RenderManager::global() {
m.release_frame(frame);
}
}
}
/// Build an owned viewer image from a rendered frame for a long-lived
/// cache (full-res fills, thumbnails). The shm variant copies the slot
/// bytes out once (`slot_to_vec` — the counted copy path, necessary
/// because the image must outlive the slot), then the caller releases the
/// slot; the in-process variant converts F32→BGRA8.
fn rendered_to_owned_image(rendered: &super::renderops::RenderedFrame) -> Option<Arc<RenderImage>> {
match rendered {
super::renderops::RenderedFrame::Shm(f) => {
let meta = &f.meta;
let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32);
let pixels = f.shm.slot_to_vec(f.slot);
let data = pixels.get(..meta.data_size.max(0) as usize)?;
bgra_bytes_to_render_image(w, h, data).map(Arc::new)
}
super::renderops::RenderedFrame::CpuF32 { .. } => {
let (w, h, samples) = read_f32_frame(rendered)?;
Some(Arc::new(f32_rgba_to_bgra_image(w, h, &samples)))
}
}
Some((width as u32, height as u32, samples))
}
// ---------------------------------------------------------------------------
@@ -675,6 +755,15 @@ pub struct RealEngine {
/// project drop); completions tagged with a stale generation are
/// discarded by the drain.
full_res_generation: u64,
/// M15 S2 playback pre-render state, keyed by monitor (see
/// [`PreviewWindow`]). An `Arc` so the ticket completions (which fire
/// from the dispatcher's poll on the UI thread) can reach the cache;
/// the mutex keeps the engine `Sync`.
preview_windows: Arc<Mutex<HashMap<Monitor, PreviewWindow>>>,
/// Bumped whenever the preview content can change (edit / proxy
/// toggle / selection / project drop); the pre-render window rebuilds
/// against the new generation.
preview_generation: u64,
/// The channel background full-res jobs report finished frames through;
/// drained on the app tick. The mutex keeps the engine `Sync` (the
/// channel is only ever touched on the UI thread).
@@ -769,6 +858,8 @@ impl RealEngine {
meter_phase: 0,
cpu_frame_cache: Mutex::new(HashMap::new()),
full_res_generation: 0,
preview_windows: Arc::new(Mutex::new(HashMap::new())),
preview_generation: 0,
full_res_rx: Mutex::new(full_res_rx),
full_res_tx: Mutex::new(full_res_tx),
renderer: Mutex::new(RendererSlot::Untried),
@@ -858,10 +949,13 @@ impl RealEngine {
/// Renders one program-monitor frame through the oakrender ticket
/// arena: builds the sequence's montage at `frame`, renders at the
/// proxy geometry, analyzes the scope samples from the F32 RGBA
/// result, and downconverts to BGRA8. Returns `None` (the caller falls
/// back to the synthetic pattern) when no sequence is open, the render
/// manager is unavailable, or the render itself fails.
/// proxy geometry, and produces the viewer display image plus the
/// scope samples (M15 S2: the process backend delivers a BGRA8 shm
/// slot; the display image is built straight from the slot bytes —
/// the GPU-upload staging copy — and the slot released). Returns
/// `None` (the caller falls back to the synthetic pattern) when no
/// sequence is open, the render manager is unavailable, or the render
/// itself fails.
fn render_program_frame(&self, frame: Frame) -> Option<(RenderImage, ScopeData)> {
let project = self.project_ref()?.clone();
let seq = self.sequence?;
@@ -877,9 +971,9 @@ impl RealEngine {
match super::renderops::render_sequence_frame(&project, seq, frame.0, tb, width, height) {
Ok(rendered) => {
*slot = RendererSlot::Ready;
let (width, height, samples) = read_f32_frame(&rendered)?;
let scope = analyze_f32_rgba(width, height, &samples);
Some((f32_rgba_to_bgra_image(width, height, &samples), scope))
let out = rendered.to_display();
release_rendered_frame(&rendered);
out
}
Err(error) => {
println!("[real engine] render_frame failed: {error}");
@@ -902,9 +996,10 @@ impl RealEngine {
/// Renders one source-monitor frame through the ticket arena: the
/// currently selected footage node decoded at the proxy geometry (same
/// pipeline as the program monitor). Returns `None` (the caller falls
/// back to the synthetic pattern) when no footage is selected, the
/// render manager is unavailable, or the render itself fails.
/// pipeline as the program monitor, M15 S2 shm slot + release).
/// Returns `None` (the caller falls back to the synthetic pattern)
/// when no footage is selected, the render manager is unavailable, or
/// the render itself fails.
fn render_source_frame(&self, frame: Frame) -> Option<(RenderImage, ScopeData)> {
let project = self.project_ref()?.clone();
let node = self.selected_footage_node()?;
@@ -920,9 +1015,9 @@ impl RealEngine {
match super::renderops::render_footage_frame(&project, node, frame.0, tb, width, height) {
Ok(rendered) => {
*slot = RendererSlot::Ready;
let (width, height, samples) = read_f32_frame(&rendered)?;
let scope = analyze_f32_rgba(width, height, &samples);
Some((f32_rgba_to_bgra_image(width, height, &samples), scope))
let out = rendered.to_display();
release_rendered_frame(&rendered);
out
}
Err(error) => {
println!("[real engine] source render_frame failed: {error}");
@@ -981,12 +1076,17 @@ impl RealEngine {
}
};
if let Ok(rendered) = rendered {
if let Some((width, height, samples)) = read_f32_frame(&rendered) {
// M15 S2: the process backend delivers a shm slot — copy the
// pixels out once (counted, long-lived cache) and release the
// slot.
let image = rendered_to_owned_image(&rendered);
release_rendered_frame(&rendered);
if let Some(image) = image {
event = Some(FullResEvent {
monitor,
frame,
generation,
image: Arc::new(f32_rgba_to_bgra_image(width, height, &samples)),
image,
});
}
}
@@ -1040,6 +1140,216 @@ impl RealEngine {
}
}
// -----------------------------------------------------------------------
// M15 S2: playback pre-render window
// -----------------------------------------------------------------------
/// Feeds the playback pre-render window for `monitor` into the
/// scheduler (M15 S2): while playing, the forward window's frames are
/// submitted at Playback priority (ordered by playhead distance), so
/// workers render them ahead of the playhead into shm slots. The
/// window rebuilds when the node or the render-params generation
/// changed; slots that fell behind the playhead are released (credit
/// returns to the workers).
fn update_preview_window(&mut self, monitor: Monitor, cx: &mut Context<Self>) {
// Only during playback: a paused viewer uses the synchronous miss
// path (and the resting full-res fill).
let playing = self.clock(monitor).read(cx).transport.is_playing();
if !playing {
return;
}
let Some(project) = self.project.clone() else { return };
let Some(tb) = self.time_base() else { return };
let Some((width, height)) = self.proxy_render_size() else { return };
let Some(node) = (match monitor {
Monitor::Program => self.sequence,
Monitor::Source => self.selected_footage_node(),
}) else {
return;
};
let node_id = node.identity();
let playhead = self.clock_frame(monitor, cx).0;
if playhead < 0 {
return;
}
let length = match monitor {
Monitor::Program => self.sequence_length().0,
Monitor::Source => self.source_length().0,
};
let Some(m) = RenderManager::global() else { return };
let forward = config_get_int(CONFIG_KEY_PREVIEW_WINDOW, DEFAULT_PREVIEW_WINDOW_FORWARD)
.clamp(8, 1200);
let end = length.max(playhead).min(playhead + forward);
// Reset / rebuild when the node changed or an invalidation bumped the
// generation (the old pending/claimed requests are cancelled).
let mut windows = self.preview_windows.lock().unwrap_or_else(|e| e.into_inner());
let window = windows.entry(monitor).or_default();
if window.sequence != node_id || window.generation != self.preview_generation {
m.cancel_preview_sequence(window.sequence);
for slot in window.slots.values() {
m.release_frame(slot);
}
window.slots.clear();
window.submitted.clear();
window.sequence = node_id;
window.generation = self.preview_generation;
}
// Release slots that fell behind the playhead (frames passed without
// being displayed); the just-displayed frame was already consumed by
// `cpu_frame`. Prune the submitted set the same way.
let keep_from = (playhead - 2).max(0);
let stale: Vec<i64> = window
.slots
.keys()
.copied()
.filter(|f| *f < keep_from)
.collect();
for f in stale {
if let Some(slot) = window.slots.remove(&f) {
m.release_frame(&slot);
}
}
window
.submitted
.retain(|f| *f >= keep_from || (*f >= playhead && *f < end));
let new_frames: Vec<i64> = (playhead.max(0)..end)
.filter(|f| !window.submitted.contains(f))
.collect();
drop(windows);
for frame in new_frames {
let params = match monitor {
Monitor::Program => super::renderops::sequence_frame_params(
&project,
node,
frame,
tb,
width,
height,
),
Monitor::Source => {
super::renderops::footage_frame_params(&project, node, frame, tb, width, height)
}
};
let Ok(params) = params else { continue };
let distance = frame.saturating_sub(playhead).abs();
let version = self.preview_generation;
let preview_windows = self.preview_windows.clone();
let done: oakrender::ticket::Completion = Box::new(move |result| {
let mut windows =
preview_windows.lock().unwrap_or_else(|e| e.into_inner());
let window = windows.entry(monitor).or_default();
match result {
Ok(oakrender::ticket::TicketPayload::ShmFrame(slot)) => {
// The rendered frame is cached in its shm slot until
// the playhead reaches it (cpu_frame) or it falls out
// of the window.
window.slots.insert(frame, slot);
}
_ => {
// Render failed / cancelled: allow a re-request.
window.submitted.remove(&frame);
}
}
});
m.tickets.submit_playback(params, frame, distance, version, done);
self.preview_windows
.lock()
.unwrap_or_else(|e| e.into_inner())
.entry(monitor)
.or_default()
.submitted
.insert(frame);
}
}
/// Looks a frame up in the pre-rendered playback window's shm slot
/// cache (M15 S2): builds the viewer display image + scope samples
/// straight from the slot bytes (the GPU-upload staging copy) and
/// releases the slot. Returns `None` when the frame is not cached.
fn preview_slot_frame(
&self,
monitor: Monitor,
frame: Frame,
) -> Option<(Arc<RenderImage>, ScopeData)> {
let node = match monitor {
Monitor::Program => self.sequence?,
Monitor::Source => self.selected_footage_node()?,
};
let node_id = node.identity();
let mut windows = self.preview_windows.lock().unwrap_or_else(|e| e.into_inner());
let window = windows.get_mut(&monitor)?;
if window.sequence != node_id || window.generation != self.preview_generation {
return None;
}
let slot = window.slots.remove(&frame.0)?;
let out = {
let meta = &slot.meta;
let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32);
let data = slot
.shm
.slot_bytes(slot.slot)
.get(..meta.data_size.max(0) as usize)?;
let image = bgra_bytes_to_render_image(w, h, data)?;
let scope = analyze_bgra8(w, h, data);
(Arc::new(image), scope)
};
if let Some(m) = RenderManager::global() {
m.release_frame(&slot);
}
Some(out)
}
/// Cancels every monitor's pre-render window and releases its held
/// slots (edit / selection change / project drop / preview-media
/// invalidation).
fn cancel_preview_windows(&mut self) {
let mut windows = self.preview_windows.lock().unwrap_or_else(|e| e.into_inner());
let m = RenderManager::global();
for window in windows.values_mut() {
if let Some(m) = &m {
m.cancel_preview_sequence(window.sequence);
for slot in window.slots.values() {
m.release_frame(slot);
}
}
window.slots.clear();
window.submitted.clear();
}
}
/// Cancels one monitor's pre-render window (source selection change:
/// the old footage's window is stale even though the program window is
/// untouched).
fn cancel_preview_window(&mut self, monitor: Monitor) {
let mut windows = self.preview_windows.lock().unwrap_or_else(|e| e.into_inner());
let Some(window) = windows.get_mut(&monitor) else {
return;
};
if let Some(m) = RenderManager::global() {
m.cancel_preview_sequence(window.sequence);
for slot in window.slots.values() {
m.release_frame(slot);
}
}
window.slots.clear();
window.submitted.clear();
}
/// Invalidates every cached/rendered preview frame: the CPU cache is
/// cleared, the full-res and pre-render-window generations bumped, and
/// the pre-render windows cancelled (pending/claimed requests and held
/// slots released). Shared by edits, undo-stack changes, project drops
/// and preview-media invalidation.
fn invalidate_rendered_frames(&mut self) {
self.cpu_frame_cache.lock().unwrap().clear();
self.full_res_generation = self.full_res_generation.wrapping_add(1);
self.preview_generation = self.preview_generation.wrapping_add(1);
self.cancel_preview_windows();
}
/// Attaches cached thumbnails to the bin entries, spawning a background
/// generation job for every footage that has none yet. Entries without a
/// renderable frame keep the widget's placeholder.
@@ -1110,7 +1420,9 @@ impl RealEngine {
return None;
}
// Frame zero: the timebase only scales the timestamp, so any valid
// pair produces time 0.
// pair produces time 0. M15 S2: the process backend delivers a shm
// slot — copy the pixels out once (counted, the PNG must outlive the
// slot), convert BGRA→RGBA, release the slot.
let rendered = super::renderops::render_footage_frame(
project,
node,
@@ -1120,11 +1432,24 @@ impl RealEngine {
THUMBNAIL_HEIGHT,
)
.ok()?;
let (width, height, samples) = read_f32_frame(&rendered)?;
let bytes: Vec<u8> = samples
.iter()
.map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8)
.collect();
let (width, height, bytes) = match &rendered {
super::renderops::RenderedFrame::Shm(f) => {
let meta = &f.meta;
let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32);
let pixels = f.shm.slot_to_vec(f.slot);
let data = pixels.get(..meta.data_size.max(0) as usize)?;
(w, h, bgra8_to_rgba8(data))
}
super::renderops::RenderedFrame::CpuF32 { .. } => {
let (w, h, samples) = read_f32_frame(&rendered)?;
let bytes: Vec<u8> = samples
.iter()
.map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8)
.collect();
(w, h, bytes)
}
};
release_rendered_frame(&rendered);
let image = image::RgbaImage::from_raw(width, height, bytes)?;
std::fs::create_dir_all(path.parent()?).ok()?;
// Write aside then rename so readers never see a partial file.
@@ -1149,13 +1474,11 @@ impl RealEngine {
}
/// Invalidates every monitor's rendered frames (the preview media
/// changed — proxy toggled, generated or deleted): the CPU cache is
/// cleared and the full-res generation bumped so in-flight fills are
/// discarded on arrival. No timeline rebuild is needed — the montage
/// resolution reads the proxy switch on every pull.
/// changed — proxy toggled, generated or deleted). No timeline rebuild
/// is needed — the montage resolution reads the proxy switch on every
/// pull.
fn invalidate_preview_frames(&mut self, cx: &mut Context<Self>) {
self.cpu_frame_cache.lock().unwrap().clear();
self.full_res_generation = self.full_res_generation.wrapping_add(1);
self.invalidate_rendered_frames();
cx.notify();
}
@@ -1696,11 +2019,10 @@ impl RealEngine {
}
self.project = None;
self.sequence = None;
self.cpu_frame_cache.lock().unwrap().clear();
// The sequence an in-flight full-res job may still be rendering is
// gone (the job holds its own project `Arc`, so it stays valid, but
// its frame belongs to the dropped project): mark it stale.
self.full_res_generation = self.full_res_generation.wrapping_add(1);
self.invalidate_rendered_frames();
// Same for in-flight thumbnail jobs; the cache is per-project too
// (identities are only unique within one graph).
self.thumb_generation = self.thumb_generation.wrapping_add(1);
@@ -1787,8 +2109,7 @@ impl RealEngine {
fn apply_stack_change(&mut self, cx: &mut Context<Self>) {
self.refresh_sequence_info();
self.rebuild_timeline();
self.cpu_frame_cache.lock().unwrap().clear();
self.full_res_generation = self.full_res_generation.wrapping_add(1);
self.invalidate_rendered_frames();
cx.notify();
}
@@ -1966,9 +2287,9 @@ impl RealEngine {
self.refresh_sequence_info();
self.rebuild_timeline();
// The sequence content changed: cached rendered frames are stale,
// and so are any in-flight full-res renders (M12 P5a).
self.cpu_frame_cache.lock().unwrap().clear();
self.full_res_generation = self.full_res_generation.wrapping_add(1);
// and so are any in-flight full-res renders and pre-render windows
// (M12 P5a / M15 S2).
self.invalidate_rendered_frames();
cx.notify();
}
}
@@ -2075,6 +2396,16 @@ impl EngineGateway for RealEngine {
}
self.mirror_program_playhead(cx);
self.meter_phase = self.meter_phase.wrapping_add(1);
// M15 S2: pump the process dispatcher — ticket completions (the
// pre-render window, full-res fills, synchronous renders) are
// delivered from its poll loop, which must run on the UI tick.
// Then feed the playback pre-render windows before draining the
// completion channels.
if let Some(m) = RenderManager::global() {
m.poll();
}
self.update_preview_window(Monitor::Source, cx);
self.update_preview_window(Monitor::Program, cx);
// M12 P1: while the program plays, pull the audio for the
// current playhead window and queue it for the output device.
if self.program_playing {
@@ -2246,6 +2577,17 @@ impl AppEngine for RealEngine {
if let Some(image) = cache.entry(monitor).or_default().image_for(frame.0) {
return image.clone();
}
// M15 S2: try the pre-rendered playback window first — the frame's
// pixels are already in a worker shm slot (zero copy: build the
// display image from the slot bytes, then release the slot).
if let Some((image, scope)) = self.preview_slot_frame(monitor, frame) {
cache.entry(monitor).or_default().proxy = Some(ProxyEntry {
frame: frame.0,
image: image.clone(),
scope,
});
return image;
}
// Both monitors render through the oakrender ticket arena (falling
// back to the synthetic pattern when rendering is unavailable): the
// program monitor renders the current sequence, the source monitor
@@ -2329,10 +2671,12 @@ impl AppEngine for RealEngine {
// The source monitor renders the selected footage node: a new
// selection must drop the stale cached frame (the cache key only
// tracks the playhead frame), and any in-flight full-res job for
// the old selection is stale.
// the old selection is stale. The source pre-render window (M15
// S2) rebuilds against the new footage.
*self.source_renderer.lock().unwrap() = RendererSlot::Untried;
self.cpu_frame_cache.lock().unwrap().remove(&Monitor::Source);
self.full_res_generation = self.full_res_generation.wrapping_add(1);
self.cancel_preview_window(Monitor::Source);
}
cx.notify();
}
@@ -3930,6 +4274,35 @@ mod tests {
crate::oakui::graphops::test_lock()
}
/// Points the render manager's worker resolution at the built
/// oak-worker binary (dev layout `target/debug/oak-worker`) for tests
/// that render through the process backend. The default resolver looks
/// next to the *test* binary (`target/debug/deps`), which holds no
/// oak-worker. Restored on drop; only used inside [`media_lock`]
/// sections so the process env stays serialized.
struct WorkerBinGuard {
prev: Option<String>,
}
impl WorkerBinGuard {
fn set() -> Self {
let prev = std::env::var("OAK_WORKER_BIN").ok();
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("target")
.join("debug")
.join("oak-worker");
std::env::set_var("OAK_WORKER_BIN", path);
Self { prev }
}
}
impl Drop for WorkerBinGuard {
fn drop(&mut self) {
match &self.prev {
Some(p) => std::env::set_var("OAK_WORKER_BIN", p),
None => std::env::remove_var("OAK_WORKER_BIN"),
}
}
}
#[test]
fn project_format_dispatches_by_extension() {
assert_eq!(
@@ -4162,14 +4535,17 @@ mod tests {
/// End-to-end CPU render through the same path
/// [`RealEngine::render_program_frame`] uses: with the render manager
/// up, rendering an in-memory sequence produces a real F32 frame at the
/// requested proxy geometry, and the samples are well-formed (finite, in
/// range). The sequence starts empty, so the picture is black; with a
/// clip of real media on the video track the same render produces the
/// decoded footage (known content, non-black).
/// up (the default M15 S2 process backend), rendering an in-memory
/// sequence produces a real BGRA8 frame in a shared-memory slot at the
/// requested proxy geometry, the display image builds straight from
/// the slot bytes (zero copy), and the picture is well-formed. The
/// sequence starts empty, so the picture is black; with a clip of real
/// media on the video track the same render produces the decoded
/// footage (known content, non-black).
#[test]
fn real_render_frame_e2e() {
let _media = media_lock();
let _worker = WorkerBinGuard::set();
if !crate::oakui::renderops::ensure_render_manager() {
panic!("the render manager failed to start");
}
@@ -4180,21 +4556,23 @@ mod tests {
.expect("the sequence has a frame rate");
// The app's proxy size: sequence aspect (default 1920x1080) scaled
// to a 480px long edge, F32 at the sequence's rate.
// to a 480px long edge.
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270)
.expect("render_frame must produce a frame");
assert_eq!((frame.width, frame.height), (480, 270));
assert_eq!(frame.format, crate::oakui::renderops::PIXEL_FORMAT_F32);
assert!(frame.linesize >= 480 * 4 * 4, "linesize covers a full row");
let (_, _, samples) = read_f32_frame(&frame).expect("well-formed F32 frame");
assert_eq!((frame.width(), frame.height()), (480, 270));
assert!(
samples.iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)),
"samples in range"
frame.is_shm(),
"the default process backend delivers shm slots (got format {})",
frame.format()
);
let (image, _scope) = frame.to_display().expect("display image from the slot");
let bytes = image.as_bytes(0).expect("one frame");
assert_eq!(bytes.len(), 480 * 270 * 4, "BGRA8 proxy geometry");
assert!(
samples.iter().all(|&v| v == 0.0),
bytes.iter().all(|&b| b == 0),
"an empty sequence renders transparent black"
);
release_rendered_frame(&frame);
// M12 P0: with a clip of real media on the video track, the same
// render must produce the decoded footage (known content, non
@@ -4212,20 +4590,22 @@ mod tests {
.expect("clip placement");
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270)
.expect("render_frame with a clip must produce a frame");
let (_, _, samples) = read_f32_frame(&frame).expect("well-formed F32 frame");
let nonzero = samples.iter().filter(|&&v| v != 0.0).count();
let (image, _scope) = frame.to_display().expect("display image from the slot");
let bytes = image.as_bytes(0).expect("one frame");
let nonzero = bytes.chunks(4).filter(|px| px[..3].iter().any(|&c| c != 0)).count();
assert!(
nonzero > 0,
"the sequence with a footage clip must render non-black pixels"
);
// Known content: the test clip's left half is red on frame 0 —
// the center-left pixel must be red-dominant.
let px = |x: usize, y: usize| &samples[(y * 480 + x) * 4..][..4];
// the center-left pixel must be red-dominant (BGRA bytes: R at 2).
let px = |x: usize, y: usize| &bytes[(y * 480 + x) * 4..][..4];
let center_left = px(120, 135);
assert!(
center_left[0] > 0.5 && center_left[1] < 0.4 && center_left[2] < 0.4,
center_left[2] > 127 && center_left[0] < 102 && center_left[1] < 102,
"center-left pixel stays red from the decoded clip: {center_left:?}"
);
release_rendered_frame(&frame);
// A second frame at a later timestamp renders too.
assert!(crate::oakui::renderops::render_sequence_frame(&project, seq, 30, tb, 480, 270).is_ok());
@@ -4237,6 +4617,74 @@ mod tests {
let _ = std::fs::remove_file(&media);
}
/// M15 S2 acceptance: the app's onscreen path reads the process
/// backend's shm slots without the counted main-process copy. The
/// preview path (`RenderedFrame::to_display`) wraps the slot bytes
/// into the display buffer — the GPU-upload staging copy — and must
/// leave `main_heap_frame_copies == 0`; the long-lived full-res path
/// (`rendered_to_owned_image`) is the one counted copy
/// (`slot_to_vec`), released right after.
#[test]
fn process_backend_preview_path_is_zero_copy() {
let _media = media_lock();
let _worker = WorkerBinGuard::set();
use oakrender::manager::{RenderBackendChoice, RenderManager};
use oakrender::procpool::{
main_heap_frame_copies, reset_main_heap_frame_copies, DispatcherConfig,
};
RenderManager::shutdown();
let config = DispatcherConfig {
worker_bin: Some(
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("target/debug/oak-worker"),
),
workers: 1,
slots_per_worker: 2,
width: 64,
height: 64,
batch_size: 2,
..Default::default()
};
RenderManager::init_with_backend(RenderBackendChoice::Processes(config))
.expect("process backend init");
let project = graphops::create_project();
let seq = graphops::create_sequence(&project, "Zero Copy");
let tb = graphops::sequence_time_base(&graphops::lock(&project).graph, seq).unwrap();
reset_main_heap_frame_copies();
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64)
.expect("render_frame must produce a frame");
let crate::oakui::renderops::RenderedFrame::Shm(slot) = &frame else {
panic!("the process backend must deliver a shm slot");
};
assert_eq!(slot.meta.format, crate::oakui::renderops::SLOT_FORMAT_BGRA8);
let (image, _scope) = frame.to_display().expect("display image from the slot");
assert_eq!(image.as_bytes(0).expect("one frame").len(), 64 * 64 * 4);
assert_eq!(
main_heap_frame_copies(),
0,
"the preview path must not copy through slot_to_vec"
);
release_rendered_frame(&frame);
assert_eq!(main_heap_frame_copies(), 0);
// The long-lived full-res path is the one counted copy.
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64)
.expect("render_frame must produce a frame");
let image = rendered_to_owned_image(&frame).expect("owned full-res image");
assert_eq!(image.as_bytes(0).expect("one frame").len(), 64 * 64 * 4);
assert_eq!(
main_heap_frame_copies(),
1,
"the long-lived cache path is the counted copy"
);
release_rendered_frame(&frame);
RenderManager::shutdown();
oakundo::global::clear().unwrap();
}
/// M12 P3 acceptance: importing a media file makes it appear in the
/// project browser's real folder tree.
#[test]
@@ -4489,6 +4937,7 @@ mod tests {
#[test]
fn full_res_worker_outlives_a_dropped_project() {
let _media = media_lock();
let _worker = WorkerBinGuard::set();
if !crate::oakui::renderops::ensure_render_manager() {
panic!("the render manager failed to start");
}
@@ -4660,6 +5109,7 @@ mod tests {
#[test]
fn full_res_worker_renders_real_frame() {
let _media = media_lock();
let _worker = WorkerBinGuard::set();
if !crate::oakui::renderops::ensure_render_manager() {
panic!("the render manager failed to start");
}
@@ -4721,6 +5171,7 @@ mod tests {
#[gpui::test]
async fn real_engine_fills_full_res_behind_the_proxy(cx: &mut gpui::TestAppContext) {
let _media = media_lock();
let _worker = WorkerBinGuard::set();
let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx)));
cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx)));
+173 -42
View File
@@ -28,22 +28,31 @@
use std::sync::mpsc;
use gpui::RenderImage;
use oakcore_rs::{Rational, TimeRange};
use oaknode::id::NodeId;
use oaknode::track::TrackType;
use oakrender::manager::RenderManager;
use oakrender::procpool::ShmFrameRef;
use oakrender::texture::Texture;
use oakrender::ticket::{AudioTicketParams, MontageClip, TicketPayload, VideoTicketParams};
use super::engine::{ExportEvent, ExportSession};
use super::frames::{bgra_bytes_to_render_image, f32_rgba_to_bgra_image};
use super::graphops::{
clip_behavior, lock, sequence_behavior, track_behavior, track_list_behavior, ProjectRef,
};
use super::scopes::{analyze_bgra8, analyze_f32_rgba, ScopeData};
/// The pixel format the viewers render in (`oakcore_rs::PixelFormat::F32`,
/// the pipeline's internal format; the app downconverts to BGRA itself).
pub const PIXEL_FORMAT_F32: i32 = 4;
/// The BGRA8 slot wire format the process backend renders into (M15 S2):
/// the worker converts its F32 pipeline output to BGRA8 at the end of the
/// render, so the main process reads display-ready bytes from the slot.
pub const SLOT_FORMAT_BGRA8: i32 = oakrender::ipc::SLOT_FORMAT_BGRA8;
// ---------------------------------------------------------------------------
// Render manager
// ---------------------------------------------------------------------------
@@ -232,19 +241,113 @@ pub fn audio_montage(p: &ProjectRef, seq: NodeId, range: TimeRange) -> Vec<Monta
// Ticket rendering
// ---------------------------------------------------------------------------
/// A rendered frame's pixel payload (the module frame a video ticket
/// produces).
pub struct RenderedFrame {
/// A rendered frame's pixel payload (M15 S2): a process-backend shm slot
/// (BGRA8, zero-copy read) or an in-process F32 CPU frame (the test-only
/// inline backend).
pub enum RenderedFrame {
/// Process backend: a BGRA8 frame in a worker's shared-memory slot.
/// Read with `shm.slot_bytes(slot)` (no counted copy on the preview
/// path), build the display image, then release the slot through the
/// render manager.
Shm(ShmFrameRef),
/// In-process (test) backend: an F32 RGBA CPU frame.
CpuF32 {
/// Width in pixels.
width: i32,
/// Height in pixels.
height: i32,
/// Bytes per scanline (stride).
linesize: i32,
/// Pixel data (at least `linesize * height` bytes).
data: Vec<u8>,
},
}
impl RenderedFrame {
/// Width in pixels.
pub width: i32,
pub fn width(&self) -> i32 {
match self {
RenderedFrame::Shm(f) => f.meta.width,
RenderedFrame::CpuF32 { width, .. } => *width,
}
}
/// Height in pixels.
pub height: i32,
/// Pixel format (`oakcore_rs::PixelFormat` as int).
pub format: i32,
/// Bytes per scanline (stride).
pub linesize: i32,
/// Pixel data (at least `linesize * height` bytes).
pub data: Vec<u8>,
pub fn height(&self) -> i32 {
match self {
RenderedFrame::Shm(f) => f.meta.height,
RenderedFrame::CpuF32 { height, .. } => *height,
}
}
/// The pixel format: the BGRA8 slot wire format for the shm variant,
/// `PIXEL_FORMAT_F32` for the in-process variant.
pub fn format(&self) -> i32 {
match self {
RenderedFrame::Shm(_) => SLOT_FORMAT_BGRA8,
RenderedFrame::CpuF32 { .. } => PIXEL_FORMAT_F32,
}
}
/// True for the process-backend shm variant.
pub fn is_shm(&self) -> bool {
matches!(self, RenderedFrame::Shm(_))
}
/// Build the viewer display image plus the scope samples (M15 S2
/// zero-copy onscreen path). For the shm variant the slot's BGRA8
/// bytes are wrapped into the display buffer — the GPU-upload staging
/// copy, the single permitted main-process copy on the preview path
/// (design §3.5). The caller releases the slot afterwards.
pub fn to_display(&self) -> Option<(RenderImage, ScopeData)> {
match self {
RenderedFrame::Shm(f) => {
let meta = &f.meta;
let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32);
let pixels = f.shm.slot_bytes(f.slot);
let data = pixels.get(..meta.data_size.max(0) as usize)?;
let image = bgra_bytes_to_render_image(w, h, data)?;
let scope = analyze_bgra8(w, h, data);
Some((image, scope))
}
RenderedFrame::CpuF32 {
width,
height,
linesize,
data,
} => {
let (w, h) = ((*width).max(0) as u32, (*height).max(0) as u32);
let samples = repack_f32_rows(*width, *height, *linesize, data)?;
Some((
f32_rgba_to_bgra_image(w, h, &samples),
analyze_f32_rgba(w, h, &samples),
))
}
}
}
}
/// Repack one F32 RGBA rendered frame (rows padded to `linesize`) into
/// tightly packed samples. Returns `(width, height, samples)`-style
/// samples only; geometry is validated by the caller.
fn repack_f32_rows(width: i32, height: i32, linesize: i32, data: &[u8]) -> Option<Vec<f32>> {
if width <= 0 || height <= 0 {
return None;
}
let row_bytes = (width * 4 * 4) as usize;
let linesize = (linesize as usize).max(row_bytes);
if data.len() < linesize * height as usize {
return None;
}
let mut samples = vec![0.0f32; (width * height * 4) as usize];
for y in 0..height as usize {
let row = &data[y * linesize..y * linesize + row_bytes];
for (i, px) in row.chunks_exact(4).enumerate() {
let v = f32::from_ne_bytes([px[0], px[1], px[2], px[3]]);
samples[y * (width as usize) * 4 + i] = v;
}
}
Some(samples)
}
/// The renderer geometry / format validation (the facade's
@@ -256,7 +359,9 @@ fn validate_geometry(width: i32, height: i32, tb: (i64, i64)) -> Result<(), Stri
Ok(())
}
/// Drive one video ticket synchronously.
/// Drive one video ticket synchronously (M15 S2: returns the payload
/// variant without copying frame bytes — the shm slot is read zero-copy
/// by the caller and released after building the display image).
fn render_video(params: VideoTicketParams) -> Result<RenderedFrame, String> {
let m = RenderManager::global().ok_or_else(|| "render manager is not initialized".to_string())?;
let id = m.tickets.next_id();
@@ -267,33 +372,31 @@ fn render_video(params: VideoTicketParams) -> Result<RenderedFrame, String> {
.result(id)
.ok_or_else(|| "render ticket produced no result".to_string())?;
match &result {
Ok(TicketPayload::Video(Texture::Cpu(frame))) => Ok(RenderedFrame {
Ok(TicketPayload::Video(Texture::Cpu(frame))) => Ok(RenderedFrame::CpuF32 {
width: frame.width,
height: frame.height,
format: frame.format as i32,
linesize: frame.linesize_bytes() as i32,
data: frame.data.clone(),
}),
Ok(TicketPayload::ShmFrame(frame)) => Ok(RenderedFrame::Shm(frame.clone())),
Ok(TicketPayload::Video(_)) => Err("render produced a non-CPU frame".to_string()),
_ => Err("render produced no video frame".to_string()),
}
}
/// Render one frame of the sequence's montage at `frame_ts` (a timestamp
/// in the `(tb.1 / tb.0)`-per-frame timebase) into a `(width, height)`
/// F32 frame.
pub fn render_sequence_frame(
/// Build the video ticket params for one sequence-montage frame (M15 S2:
/// shared by the synchronous render and the playback pre-render window).
pub fn sequence_frame_params(
p: &ProjectRef,
seq: NodeId,
frame_ts: i64,
tb: (i64, i64),
width: i32,
height: i32,
) -> Result<RenderedFrame, String> {
) -> Result<VideoTicketParams, String> {
validate_geometry(width, height, tb)?;
let time = Rational::new(frame_ts * tb.0, tb.1);
let montage = video_montage(p, seq, time);
render_video(VideoTicketParams {
Ok(VideoTicketParams {
viewer: seq.identity(),
time,
force_size: Some((width, height)),
@@ -303,7 +406,54 @@ pub fn render_sequence_frame(
cache_id: None,
cache_timebase: None,
footage: None,
montage,
montage: video_montage(p, seq, time),
})
}
/// Render one frame of the sequence's montage at `frame_ts` (a timestamp
/// in the `(tb.1 / tb.0)`-per-frame timebase) into a `(width, height)`
/// frame.
pub fn render_sequence_frame(
p: &ProjectRef,
seq: NodeId,
frame_ts: i64,
tb: (i64, i64),
width: i32,
height: i32,
) -> Result<RenderedFrame, String> {
render_video(sequence_frame_params(p, seq, frame_ts, tb, width, height)?)
}
/// Build the video ticket params for one single-footage frame (M15 S2:
/// shared by the synchronous render and the source-monitor pre-render
/// window).
pub fn footage_frame_params(
p: &ProjectRef,
footage: NodeId,
frame_ts: i64,
tb: (i64, i64),
width: i32,
height: i32,
) -> Result<VideoTicketParams, String> {
validate_geometry(width, height, tb)?;
let (filename, stream_index) = {
let g = lock(p);
super::graphops::footage_behavior(&g.graph, footage)
.map(|f| preview_footage_media(f, true))
.ok_or_else(|| "the node is not footage".to_string())?
};
let time = Rational::new(frame_ts * tb.0, tb.1);
Ok(VideoTicketParams {
viewer: footage.identity(),
time,
force_size: Some((width, height)),
force_format: None,
cache: None,
cache_dir: None,
cache_id: None,
cache_timebase: None,
footage: Some((filename, stream_index)),
montage: Vec::new(),
})
}
@@ -317,26 +467,7 @@ pub fn render_footage_frame(
width: i32,
height: i32,
) -> Result<RenderedFrame, String> {
validate_geometry(width, height, tb)?;
let (filename, stream_index) = {
let g = lock(p);
super::graphops::footage_behavior(&g.graph, footage)
.map(|f| preview_footage_media(f, true))
.ok_or_else(|| "the node is not footage".to_string())?
};
let time = Rational::new(frame_ts * tb.0, tb.1);
render_video(VideoTicketParams {
viewer: footage.identity(),
time,
force_size: Some((width, height)),
force_format: None,
cache: None,
cache_dir: None,
cache_id: None,
cache_timebase: None,
footage: Some((filename, stream_index)),
montage: Vec::new(),
})
render_video(footage_frame_params(p, footage, frame_ts, tb, width, height)?)
}
/// Rendered interleaved f32 audio (the module audio ticket payload).
+28
View File
@@ -77,6 +77,34 @@ pub(crate) fn analyze_f32_rgba(width: u32, height: u32, samples: &[f32]) -> Scop
}
}
/// Analyzes one BGRA8 frame (the process backend's slot format, M15 S2)
/// into its [`ScopeData`]. The worker converts its F32 pipeline output to
/// BGRA8 at the end of the render, so the scopes read exactly the
/// displayed values with the viewer's 8-bit quantization — precision loss
/// vs the F32 analysis is bounded by 1/255 per channel (acceptable for
/// the scopes; the F32 path stays for the in-process test backend).
/// `bytes` must hold at least `width * height * 4` values.
pub(crate) fn analyze_bgra8(width: u32, height: u32, bytes: &[u8]) -> ScopeData {
let pixels = (width * height) as usize;
let mut luma = Vec::with_capacity(pixels);
let mut chroma = Vec::with_capacity(pixels);
let src = bytes.get(..pixels * 4).unwrap_or_default();
for px in src.chunks_exact(4) {
let b = f32::from(px[0]) / 255.0;
let g = f32::from(px[1]) / 255.0;
let r = f32::from(px[2]) / 255.0;
let y = KR * r + KG * g + KB * b;
let cb = 0.5 + (b - y) / (2.0 * (1.0 - KB));
let cr = 0.5 + (r - y) / (2.0 * (1.0 - KR));
luma.push(y);
chroma.push((cb.clamp(0.0, 1.0), cr.clamp(0.0, 1.0)));
}
ScopeData {
luma: Arc::new(luma),
chroma: Arc::new(chroma),
}
}
#[cfg(test)]
mod tests {
use super::*;