From f908724a7aaf80dfb0b9b00b03eccfa8902224d3 Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Tue, 11 Aug 2026 16:10:14 +0800 Subject: [PATCH] feat(app): real engine renders true frames to the viewers - RealEngine caches a per-sequence renderer and serves cpu_frame via oakengine_renderer_render_frame (F32 RGBA, 480px-long-edge proxy) instead of the synthetic test pattern; edits/undo invalidate the per-frame cache - frames.rs: F32 RGBA -> BGRA8 downconvert extracted as f32_rgba_to_bgra_image (shared with the synthetic fallback) - ffi.rs: renderer/frame C ABI declarations + OakEngineFrame / OakEngineRenderer handle mirrors, oakrender_manager_init/available - new real_render_frame_e2e test renders through the dylib; mock engine and headless tests unaffected - known gaps (documented in code): source monitor still synthetic (facade renders sequences only), render is synchronous on the UI thread, renderer not rebuilt on sequence geometry change --- src/oakui/ffi.rs | 62 +++++++++ src/oakui/frames.rs | 43 ++++++- src/oakui/real.rs | 297 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 386 insertions(+), 16 deletions(-) diff --git a/src/oakui/ffi.rs b/src/oakui/ffi.rs index 89aed3e69..6af8674c1 100644 --- a/src/oakui/ffi.rs +++ b/src/oakui/ffi.rs @@ -102,7 +102,9 @@ macro_rules! engine_handle { engine_handle! { OakEngineClip, OakEngineEncodingParams, + OakEngineFrame, OakEngineProject, + OakEngineRenderer, OakEngineSequence, OakEngineTask, } @@ -478,6 +480,66 @@ unsafe extern "C" { /// `oakengine_track_height_pixels_to_internal`. pub fn oakengine_track_height_pixels_to_internal(pixels: c_int) -> f64; + // -- oakengine::render (CPU frame renderer) -- + // + // The renderer binds a sequence handle to an output geometry; each + // `render_frame` submits an oakrender ticket, waits for it and returns + // the produced frame (F32 RGBA when created with pixel format 4). The + // renderer box is opaque: it is freed ONLY with + // `oakengine_renderer_free` (never [`free_box`] — the facade box is not + // a module-handle box). Frames are freed with `oakengine_frame_free`. + + /// `oakengine_renderer_create` — owned renderer box (NULL on invalid + /// arguments: NULL sequence, non-positive geometry/rate, or a pixel + /// format outside 0..=4). `output_colorspace` may be NULL. + pub fn oakengine_renderer_create( + seq: *mut OakEngineSequence, + width: c_int, + height: c_int, + pixel_format: c_int, + frame_rate_num: c_int, + frame_rate_den: c_int, + output_colorspace: *const c_char, + ) -> *mut OakEngineRenderer; + /// `oakengine_renderer_free` — consuming free (NULL no-op). + pub fn oakengine_renderer_free(self_: *mut OakEngineRenderer); + /// `oakengine_renderer_render_frame` — synchronous render of the frame at + /// `timestamp` (in the renderer's frame-rate units). NULL on failure + /// (see `oakengine_renderer_last_error`). + pub fn oakengine_renderer_render_frame( + self_: *mut OakEngineRenderer, + timestamp: i64, + ) -> *mut OakEngineFrame; + /// `oakengine_renderer_last_error` (buf/size). + pub fn oakengine_renderer_last_error( + self_: *const OakEngineRenderer, + buf: *mut c_char, + buf_size: c_int, + ) -> c_int; + /// `oakengine_frame_width`. + pub fn oakengine_frame_width(self_: *const OakEngineFrame) -> c_int; + /// `oakengine_frame_height`. + pub fn oakengine_frame_height(self_: *const OakEngineFrame) -> c_int; + /// `oakengine_frame_format` — `PixelFormat::Format` (4 = F32). + pub fn oakengine_frame_format(self_: *const OakEngineFrame) -> c_int; + /// `oakengine_frame_linesize_bytes` — row stride in bytes. + pub fn oakengine_frame_linesize_bytes(self_: *const OakEngineFrame) -> c_int; + /// `oakengine_frame_data` — borrowed pixel data (valid until free). + pub fn oakengine_frame_data(self_: *const OakEngineFrame) -> *const c_void; + /// `oakengine_frame_free` — consuming free (NULL no-op). + pub fn oakengine_frame_free(self_: *mut OakEngineFrame); + + // -- oakrender module C ABI (carried by the dylib) -- + + /// `oakrender_manager_init` — bring up the module's process-global + /// render manager. Without it `render_frame` fails with NULL + + /// last_error. The facade does not wrap this; like the `oaktask_*` + /// entries below, the symbol is exported by the dylib itself. Fails + /// (nonzero) when the manager is already initialized. + pub fn oakrender_manager_init() -> c_int; + /// `oakrender_manager_available` — 1 when the render manager is up. + pub fn oakrender_manager_available() -> c_int; + // -- oaktask module C ABI (carried by the dylib) -- /// `oaktask_load_take_project` — take the project an interchange diff --git a/src/oakui/frames.rs b/src/oakui/frames.rs index 78ca11972..c24297137 100644 --- a/src/oakui/frames.rs +++ b/src/oakui/frames.rs @@ -84,13 +84,44 @@ pub(crate) fn synthetic_frame(frame: Frame) -> RenderImage { } } - let mut bytes = Vec::with_capacity((width * height * 4) as usize); + f32_rgba_to_bgra_image(width, height, &samples) +} + +/// Downconverts an F32 RGBA frame (the engine pipeline's pixel format) to a +/// BGRA8 [`RenderImage`] for the viewers' CPU-frame path. Samples are +/// clamped to `0.0..=1.0` before quantization; `samples` must hold exactly +/// `width * height * 4` values (tightly packed rows). +/// +/// Shared by the synthetic test pattern and the real engine's rendered +/// frames ([`super::real`]). +pub(crate) fn f32_rgba_to_bgra_image(width: u32, height: u32, samples: &[f32]) -> RenderImage { + assert_eq!( + samples.len(), + (width * height * 4) as usize, + "F32 RGBA frame must be tightly packed" + ); + let mut bytes = Vec::with_capacity(samples.len()); for i in (0..samples.len()).step_by(4) { - bytes.push((samples[i + 2] * 255.0) as u8); // B - bytes.push((samples[i + 1] * 255.0) as u8); // G - bytes.push((samples[i] * 255.0) as u8); // R - bytes.push((samples[i + 3] * 255.0) as u8); // A + bytes.push((samples[i + 2].clamp(0.0, 1.0) * 255.0) as u8); // B + bytes.push((samples[i + 1].clamp(0.0, 1.0) * 255.0) as u8); // G + bytes.push((samples[i].clamp(0.0, 1.0) * 255.0) as u8); // R + bytes.push((samples[i + 3].clamp(0.0, 1.0) * 255.0) as u8); // A } - let buffer = image::RgbaImage::from_raw(width, height, bytes).expect("synthetic frame"); + let buffer = image::RgbaImage::from_raw(width, height, bytes).expect("BGRA frame"); RenderImage::new(smallvec::SmallVec::from_elem(image::Frame::new(buffer), 1)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn f32_rgba_converts_to_bgra_bytes() { + // One red-ish pixel and one pixel exercising clamping. + let samples = [1.0, 0.0, 0.5, 1.0, 2.0, -1.0, 0.25, 1.0]; + let image = f32_rgba_to_bgra_image(2, 1, &samples); + let frame = image.as_bytes(0).expect("one frame"); // BGRA8, tightly packed + assert_eq!(&frame[0..4], &[127, 0, 255, 255]); + assert_eq!(&frame[4..8], &[63, 0, 255, 255]); + } +} diff --git a/src/oakui/real.rs b/src/oakui/real.rs index b65a1789e..d44430896 100644 --- a/src/oakui/real.rs +++ b/src/oakui/real.rs @@ -41,10 +41,13 @@ //! //! # What is still mock/stub //! -//! * The viewer frames are the shared synthetic SMPTE pattern ([`frames`]), -//! driven by the real sequence frame rate — the real frame transport -//! (render worker over shared memory, the facade's worker module) is a -//! separate process surface not bound yet. +//! * The source monitor's viewer frames are the shared synthetic SMPTE +//! pattern ([`frames`]): the facade renderer binds a *sequence* handle +//! only, so there is no surface to render a single footage node for the +//! material viewer. The program monitor renders real frames through the +//! facade CPU renderer ([`RealEngine::render_program_frame`]) at a proxy +//! resolution; the full-resolution async render worker (the facade's +//! worker module) is a separate process surface not bound yet. //! * Effect stack, node graph and audio meter feed empty/silent data: the //! facade surfaces for them (effect chains, graph nodes, audio levels) //! are not bound in this increment. @@ -84,7 +87,7 @@ use super::ffi::*; use super::engine::{ AppEngine, EngineGateway, ExportEvent, ExportSession, Monitor, Project, Sequence, VideoFormat, }; -use super::frames::synthetic_frame; +use super::frames::{f32_rgba_to_bgra_image, synthetic_frame}; use super::transport::TransportState; /// `oakengine_timeline.h` track-type constants. @@ -102,6 +105,11 @@ const EXPORT_SAMPLE_FORMAT: c_int = 0; /// The project name of a blank project before it is saved. const UNTITLED: &str = "Untitled Project"; +/// `PixelFormat::F32` (the render pipeline's internal format): F32 RGBA, +/// 16 bytes per pixel. The viewer renderer is created with this so the +/// frame accessors hand back float samples the app downconverts itself. +const PIXEL_FORMAT_F32: c_int = 4; + // --------------------------------------------------------------------------- // oaktask module C ABI entries the facade does not wrap // --------------------------------------------------------------------------- @@ -202,6 +210,45 @@ impl Drop for SequenceHandle { } } +/// An owned facade renderer handle; freed with `oakengine_renderer_free`. +/// +/// The facade renderer box is NOT a module-handle box (it is the facade's +/// own `RendererBox`), so it must never go through [`free_box`]; the +/// dedicated free is the only valid deallocator. The renderer borrows the +/// sequence handle it was created from, so it must be dropped before the +/// sequence (see [`RealEngine::drop_project`]). +struct RendererHandle(*mut OakEngineRenderer); + +// SAFETY: see [`ProjectHandle`]. +unsafe impl Send for RendererHandle {} +unsafe impl Sync for RendererHandle {} + +impl RendererHandle { + fn ptr(&self) -> *mut OakEngineRenderer { + self.0 + } +} + +impl Drop for RendererHandle { + fn drop(&mut self) { + unsafe { + oakengine_renderer_free(self.0); + } + } +} + +/// The lifecycle state of the program monitor's lazily created renderer. +enum RendererSlot { + /// No renderer yet; the next `cpu_frame` tries to create one. + Untried, + /// The live per-sequence renderer. + Ready(RendererHandle), + /// Creation failed (or no sequence is open); don't retry until the + /// project changes, so a broken setup doesn't retry — and log — on + /// every frame. + Unavailable, +} + // --------------------------------------------------------------------------- // FFI helpers // --------------------------------------------------------------------------- @@ -543,10 +590,19 @@ pub struct RealEngine { program_playing: bool, /// Phase counter driving the (silent) audio levels. meter_phase: u32, - /// Cache of the synthetic CPU frames handed to the viewers, keyed by - /// monitor. Entries are the playhead frame that produced the image, so a - /// paused viewer never regenerates its picture. + /// Cache of the CPU frames handed to the viewers, keyed by monitor. + /// Entries are the playhead frame that produced the image, so a paused + /// viewer never regenerates its picture. The program monitor's entries + /// are real rendered frames (see [`RealEngine::render_program_frame`]); + /// the source monitor's are the synthetic pattern (the facade renderer + /// binds a sequence only — footage-node rendering is a documented gap). cpu_frame_cache: Mutex)>>, + /// The program monitor's cached renderer, created lazily from the + /// current sequence at a proxy resolution. The mutex both provides the + /// interior mutability `cpu_frame` (a `&self` read) needs and serializes + /// the synchronous render calls. Reset to [`RendererSlot::Untried`] + /// (before the sequence is freed) in [`RealEngine::drop_project`]. + renderer: Mutex, /// Whether the project has unsaved changes (mirrors the facade flag). modified: bool, } @@ -572,6 +628,7 @@ impl RealEngine { program_playing: false, meter_phase: 0, cpu_frame_cache: Mutex::new(HashMap::new()), + renderer: Mutex::new(RendererSlot::Untried), modified: false, } } @@ -609,6 +666,131 @@ impl RealEngine { } } + /// Brings up the module's process-global render manager if it is not + /// running yet (without it `render_frame` fails with NULL + last_error). + /// Returns false when the manager could not be started. + fn ensure_render_manager() -> bool { + unsafe { + if oakrender_manager_available() != 0 { + return true; + } + oakrender_manager_init(); + oakrender_manager_available() != 0 + } + } + + /// The proxy resolution the viewer renderer runs at: the sequence's + /// aspect scaled to a small long edge. Rendering is a synchronous call + /// made from `cpu_frame` (a `&self` read on the UI thread), so the + /// geometry stays tiny to keep the block short; the async render worker + /// (full-resolution, off-thread) is a separate transport surface not + /// bound yet. + fn proxy_render_size(&self) -> Option<(c_int, c_int)> { + let info = self.sequence_info.as_ref()?; + let (w, h) = (info.format.width.max(1), info.format.height.max(1)); + const MAX_LONG_EDGE: u32 = 480; + let scale = MAX_LONG_EDGE as f64 / w.max(h) as f64; + let width = ((w as f64 * scale).round() as u32).max(2); + let height = ((h as f64 * scale).round() as u32).max(2); + Some((width as c_int, height as c_int)) + } + + /// Renders one program-monitor frame through the facade CPU renderer: + /// creates the per-sequence renderer lazily (cached in `self.renderer`), + /// renders `frame`, and downconverts the F32 RGBA result to BGRA8. + /// Returns `None` (the caller falls back to the synthetic pattern) when + /// no sequence is open, the render manager is unavailable, or the render + /// itself fails. + fn render_program_frame(&self, frame: Frame) -> Option { + let seq = self.seq_ptr()?; + if !Self::ensure_render_manager() { + return None; + } + let mut slot = self.renderer.lock().unwrap(); + match &*slot { + RendererSlot::Unavailable => return None, + RendererSlot::Untried => { + let created = self.create_renderer(seq); + *slot = match created { + Some(handle) => RendererSlot::Ready(handle), + None => RendererSlot::Unavailable, + }; + } + RendererSlot::Ready(_) => {} + } + let RendererSlot::Ready(handle) = &*slot else { + return None; + }; + let renderer = handle.ptr(); + let frame_ptr = unsafe { oakengine_renderer_render_frame(renderer, frame.0) }; + if frame_ptr.is_null() { + let error = read_string(|buf, size| unsafe { + oakengine_renderer_last_error(renderer, buf, size) + }); + println!("[real engine] render_frame failed: {error}"); + // Don't retry (and re-log) on every frame. + *slot = RendererSlot::Unavailable; + return None; + } + // Read the frame (F32 RGBA, rows padded to linesize), repack it + // tightly, then downconvert. + let (width, height, linesize, format) = unsafe { + ( + oakengine_frame_width(frame_ptr), + oakengine_frame_height(frame_ptr), + oakengine_frame_linesize_bytes(frame_ptr), + oakengine_frame_format(frame_ptr), + ) + }; + let data = unsafe { oakengine_frame_data(frame_ptr) }; + let mut image = None; + if width > 0 && height > 0 && format == PIXEL_FORMAT_F32 && !data.is_null() { + let row_bytes = (width * 4 * 4) as usize; + let linesize = (linesize as usize).max(row_bytes); + let mut samples = vec![0.0f32; (width * height * 4) as usize]; + for y in 0..height as usize { + unsafe { + std::ptr::copy_nonoverlapping( + (data as *const u8).add(y * linesize), + samples.as_mut_ptr().add(y * row_bytes / 4) as *mut u8, + row_bytes, + ); + } + } + image = Some(f32_rgba_to_bgra_image(width as u32, height as u32, &samples)); + } + unsafe { + oakengine_frame_free(frame_ptr); + } + image + } + + /// Creates the per-sequence renderer at the proxy resolution (see + /// [`RealEngine::proxy_render_size`]). The renderer borrows the sequence + /// handle; the caller owns the slot it is stored in. + fn create_renderer(&self, seq: *mut OakEngineSequence) -> Option { + let (width, height) = self.proxy_render_size()?; + let rate = self.sequence_info.as_ref()?.format.rate; + // Pixel format 4 = PixelFormat::F32 (the pipeline format); + // timestamp units are frames at this rate. + let renderer = unsafe { + oakengine_renderer_create( + seq, + width, + height, + PIXEL_FORMAT_F32, + rate.num as c_int, + rate.den as c_int, + std::ptr::null(), + ) + }; + if renderer.is_null() { + println!("[real engine] renderer_create failed; viewer keeps the synthetic frame"); + return None; + } + Some(RendererHandle(renderer)) + } + /// Adopts a newly created/loaded facade project, freeing any previous /// one, and rebuilds every snapshot. `blank` projects get a default /// sequence; loaded ones use the first sequence. @@ -643,10 +825,14 @@ impl RealEngine { cx.notify(); } - /// Frees the project and every borrowed handle (sequence first). + /// Frees the project and every borrowed handle (renderer and sequence + /// first: the renderer borrows the sequence, and the sequence is + /// borrowed from the project). fn drop_project(&mut self) { + *self.renderer.lock().unwrap() = RendererSlot::Untried; drop(self.sequence.take()); drop(self.project.take()); + self.cpu_frame_cache.lock().unwrap().clear(); self.tracks.clear(); self.bin_roots.clear(); self.bin_children.clear(); @@ -841,6 +1027,8 @@ impl RealEngine { } self.refresh_sequence_info(); self.rebuild_timeline(); + // The sequence content changed: cached rendered frames are stale. + self.cpu_frame_cache.lock().unwrap().clear(); self.modified = true; cx.notify(); } @@ -1055,7 +1243,19 @@ impl AppEngine for RealEngine { return image.clone(); } } - let image = Arc::new(synthetic_frame(frame)); + // The program monitor renders the current sequence through the + // facade CPU renderer (falling back to the synthetic pattern when + // rendering is unavailable). The source monitor keeps the synthetic + // pattern: the facade renderer binds a *sequence* handle only, so + // there is currently no surface to render a single footage node for + // the material viewer — that is a documented facade gap. + let image = match monitor { + Monitor::Program => self + .render_program_frame(frame) + .map(Arc::new) + .unwrap_or_else(|| Arc::new(synthetic_frame(frame))), + Monitor::Source => Arc::new(synthetic_frame(frame)), + }; cache.insert(monitor, (frame.0, image.clone())); image } @@ -1320,6 +1520,7 @@ impl AppEngine for RealEngine { } self.refresh_sequence_info(); self.rebuild_timeline(); + self.cpu_frame_cache.lock().unwrap().clear(); self.modified = true; cx.notify(); } @@ -1332,6 +1533,7 @@ impl AppEngine for RealEngine { } self.refresh_sequence_info(); self.rebuild_timeline(); + self.cpu_frame_cache.lock().unwrap().clear(); self.modified = true; cx.notify(); } @@ -1817,4 +2019,79 @@ mod tests { unsafe { oakengine_project_free(project2) }; let _ = std::fs::remove_file(&save_path); } + + /// End-to-end CPU render through the same facade path + /// [`RealEngine::render_program_frame`] uses: with the render manager up, + /// `render_frame` on an in-memory sequence produces a real F32 frame at + /// the renderer's proxy geometry, and the samples are well-formed + /// (finite, in range). The sequence is empty, so the picture is black — content + /// correctness with real footage needs the footage-input surface the + /// facade does not bind yet (documented gap); what is asserted here is + /// the full transport: renderer lifecycle, frame geometry/format/stride + /// and sane sample values. + #[test] + fn real_render_frame_e2e() { + if !RealEngine::ensure_render_manager() { + panic!("the render manager failed to start"); + } + + let project = unsafe { oakengine_project_create() }; + assert!(!project.is_null()); + assert_eq!(unsafe { oakengine_project_new(project) }, 0); + let name = CString::new("Render E2E").unwrap(); + let sequence = unsafe { oakengine_sequence_new(project, name.as_ptr()) }; + assert!(!sequence.is_null()); + + // The app's proxy size: sequence aspect (default 1920x1080) scaled + // to a 480px long edge, F32 at 25 fps. + let renderer = unsafe { + oakengine_renderer_create(sequence, 480, 270, PIXEL_FORMAT_F32, 25, 1, std::ptr::null()) + }; + assert!(!renderer.is_null(), "renderer_create must succeed"); + + let frame = unsafe { oakengine_renderer_render_frame(renderer, 0) }; + assert!(!frame.is_null(), "render_frame must produce a frame"); + assert_eq!(unsafe { oakengine_frame_width(frame) }, 480); + assert_eq!(unsafe { oakengine_frame_height(frame) }, 270); + assert_eq!(unsafe { oakengine_frame_format(frame) }, PIXEL_FORMAT_F32); + let linesize = unsafe { oakengine_frame_linesize_bytes(frame) }; + assert!(linesize >= 480 * 4 * 4, "linesize covers a full row"); + let data = unsafe { oakengine_frame_data(frame) } as *const f32; + assert!(!data.is_null()); + + // Sample pixels across the frame: all values must be finite and in + // range; an empty sequence renders transparent black (all zeros). + let stride = linesize as usize / 4; + let mut nonzero = 0usize; + for &(x, y) in &[(0usize, 0usize), (240, 135), (479, 269)] { + let base = y * stride + x * 4; + let px = unsafe { std::slice::from_raw_parts(data.add(base), 4) }; + assert!(px.iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)), "samples in range: {px:?}"); + nonzero += px.iter().filter(|&&v| v != 0.0).count(); + } + // Document the current empty-sequence behavior: a transparent-black + // program. When footage input is bound, extend this to assert + // non-black content. + assert_eq!(nonzero, 0, "an empty sequence renders transparent black"); + + // A second frame at a later timestamp renders too. + let frame2 = unsafe { oakengine_renderer_render_frame(renderer, 30) }; + assert!(!frame2.is_null()); + unsafe { oakengine_frame_free(frame2) }; + + // Invalid arguments are rejected (geometry, pixel format). + assert!(unsafe { + oakengine_renderer_create(sequence, 0, 270, PIXEL_FORMAT_F32, 25, 1, std::ptr::null()) + } + .is_null()); + assert!(unsafe { + oakengine_renderer_create(sequence, 480, 270, 99999, 25, 1, std::ptr::null()) + } + .is_null()); + + unsafe { oakengine_frame_free(frame) }; + unsafe { oakengine_renderer_free(renderer) }; + unsafe { free_box(sequence) }; + unsafe { oakengine_project_free(project) }; + } }