diff --git a/crates/oak-app/src/oakui/renderops.rs b/crates/oak-app/src/oakui/renderops.rs index fec225498..c0ab56916 100644 --- a/crates/oak-app/src/oakui/renderops.rs +++ b/crates/oak-app/src/oakui/renderops.rs @@ -142,6 +142,38 @@ fn clip_preview_media( Some(preview_footage_media(f, is_video)) } +/// The clip block's effect stack as montage effect descriptors +/// (source-first order — the chain walk's signal order, so the renderer +/// applies them media-side first). The chain walk runs all the way to +/// the media source, so its first element is the footage node feeding +/// the clip; the montage decodes that footage itself, so the source +/// node (the one WITHOUT an effect input) is dropped here. Disabled +/// effects are carried with `enabled = false`; the renderer bypasses +/// them (the C++ traverser's bypass pushes the effect input through +/// unchanged). Parameters are the inspector's parameter set (non-hidden, +/// non-connection inputs at their standard values; keyframed values are +/// not time-resolved on this path). +fn clip_effects(g: &oak_node::graph::Graph, block_id: NodeId) -> Vec { + use super::effectchain; + effectchain::chain(g, block_id) + .into_iter() + .filter(|&fx| effectchain::effect_input_of(g, fx).is_some()) + .filter_map(|fx| { + let entry = g.get(fx)?; + Some(oak_render::ticket::MontageEffect { + type_id: entry.behavior.type_id().to_string(), + enabled: effectchain::is_enabled(g, fx), + effect_input_id: effectchain::effect_input_of(g, fx), + params: effectchain::effect_params(g, fx) + .unwrap_or_default() + .into_iter() + .map(|p| (p.input_id, p.value)) + .collect(), + }) + }) + .collect() +} + /// The video montage at sequence time `time`: every clip covering `time` /// on video tracks, ordered bottom-to-top (track index 0 is topmost, so /// it is composited last). Hidden tracks (the muted flag doubles as the @@ -187,6 +219,7 @@ pub fn video_montage(p: &ProjectRef, seq: NodeId, time: Rational) -> Vec Vec effect -> clip). + let fx = crate::oakui::effectchain::insert( + &project, + block, + usize::MAX, + "org.olivevideoeditor.Olive.opacity", + ) + .expect("append the opacity effect"); + crate::oakui::effectchain::set_input_value( + &project, + fx, + "opacity_in", + oak_node::value::NodeValue::Float(0.5), + ) + .expect("set the opacity value"); + + let tb = graphops::sequence_time_base(&lock(&project).graph, seq).unwrap(); + let time = graphops::ts_to_rational(0, tb); + + // Render the montage through the same entry point the render worker + // uses (`render_montage_frame_into`, F32 RGBA rows). + let render = |montage: Vec| { + let params = VideoTicketParams { + viewer: 0, + time, + force_size: Some((64, 64)), + force_format: Some(oak_core::PixelFormat::F32), + cache: None, + cache_dir: None, + cache_id: None, + cache_timebase: None, + footage: None, + montage, + }; + let mut dst = vec![0u8; 64 * 64 * 16]; + oak_render::eval::render_montage_frame_into(time, ¶ms, (64, 64), &mut dst, 64 * 16) + .expect("montage render"); + dst + }; + // A left-half pixel of the test pattern (known content, r ~= 0.9). + let pixel = |frame: &[u8]| { + let off = (8 * 64 + 8) * 16; + [0, 1, 2, 3].map(|i| f32::from_le_bytes(frame[off + i * 4..off + i * 4 + 4].try_into().unwrap())) + }; + + let with_fx = video_montage(&project, seq, time); + assert_eq!(with_fx.len(), 1, "one clip covers frame 0"); + assert_eq!( + with_fx[0].effects.len(), + 1, + "the montage carries the clip's effect stack (not the footage source node)" + ); + assert_eq!(with_fx[0].effects[0].type_id, "org.olivevideoeditor.Olive.opacity"); + assert!(with_fx[0].effects[0].enabled); + + let plain = render( + with_fx + .iter() + .cloned() + .map(|mut c| { + c.effects.clear(); + c + }) + .collect(), + ); + let effected = render(with_fx); + let (p, e) = (pixel(&plain), pixel(&effected)); + assert!(p[0] > 0.6, "the plain render shows the test pattern (r={})", p[0]); + // 50% opacity: the shader halves every channel, then the composite + // over transparent black halves it again via the halved alpha. + assert!( + (e[0] - p[0] * 0.25).abs() < 0.1, + "50% opacity quarters the output ({} vs {})", + e[0], + p[0] * 0.25 + ); + assert!( + (e[3] - 0.5).abs() < 0.1, + "the composited alpha halves (a={})", + e[3] + ); + + // Disabled effect = bypass: pixels match the plain render. + crate::oakui::effectchain::set_enabled(&project, fx, false).expect("disable the effect"); + let disabled_montage = video_montage(&project, seq, time); + assert_eq!(disabled_montage[0].effects.len(), 1); + assert!(!disabled_montage[0].effects[0].enabled); + let disabled = render(disabled_montage); + let d = pixel(&disabled); + assert!( + (d[0] - p[0]).abs() < 0.05, + "a disabled effect leaves the pixels alone ({} vs {})", + d[0], + p[0] + ); + + oak_undo::global::clear().unwrap(); + let _ = std::fs::remove_file(&media); + } + /// The original media decodes from the footage's actual first stream of /// the kind (not the hardcoded 0/1 of a typical layout): a file whose /// video stream is not stream 0 must still decode its own video when diff --git a/crates/oak-cli/src/engine.rs b/crates/oak-cli/src/engine.rs index 2d72c63b6..27c419ac3 100644 --- a/crates/oak-cli/src/engine.rs +++ b/crates/oak-cli/src/engine.rs @@ -535,6 +535,9 @@ pub fn video_montage(p: &ProjectRef, seq_id: NodeId, time: Rational) -> Vec Vec = OnceLock::new(); pub fn install_render_executor() { EXECUTOR_INSTALLED.get_or_init(|| { oak_render::eval::set_plugin_executor(Some(Arc::new(execute_plugin_job))); + oak_render::eval::set_plugin_instance_factory(Some(Arc::new(shared_plugin_instance))); oak_node::nodes::plugin::set_plugin_duplicator(Some(Arc::new(duplicate_instance))); }); } +// --------------------------------------------------------------------------- +// montage 渲染路径的共享实例（渲染进程按插件标识惰性创建 + 进程级缓存） +// --------------------------------------------------------------------------- + +/// montage 效果栈携带插件标识而非实例 id（montage 在主进程从时间线 +/// 解析，实例活在渲染进程）。渲染进程经此工厂按需创建实例并缓存： +/// 参数每次渲染前由 montage 注入（[`execute_plugin_job`] 的 values +/// 覆盖），worker 单线程顺序渲染，共享实例无跨帧状态冲突。 +fn shared_plugin_instance(identifier: &str) -> Option { + static SHARED: OnceLock>> = OnceLock::new(); + let cache = SHARED.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(&id) = cache + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(identifier) + { + return Some(id); + } + let host = Host::global(); + let plugin = host.cache.find(identifier)?; + // 上下文选择与 [`register_plugin_nodes`] 一致（filter 优先，否则 + // 首个支持上下文；factory.cpp:171-177）。 + let context = if plugin.contexts.iter().any(|c| c == "OfxImageEffectContextFilter") { + "OfxImageEffectContextFilter".to_string() + } else { + plugin.contexts.first()?.clone() + }; + let inst = host.create_instance(identifier, Some(&context)).ok()?; + let id = register_instance(inst); + cache + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(identifier.to_string(), id); + Some(id) +} + // --------------------------------------------------------------------------- // 测试 // --------------------------------------------------------------------------- @@ -1085,6 +1122,13 @@ mod tests { unregister_instance(u64::MAX); } + #[test] + fn shared_instance_unknown_identifier_yields_none() { + // 缓存里查无此标识 → None（montage 路径据此按"无求值器" + // 告警并直通）。 + assert!(shared_plugin_instance("com.example.definitely-missing").is_none()); + } + /// 构造一个只含 push-button 参数的最小实例（直接登记进注册表）。 fn instance_with_push_button() -> u64 { use std::ffi::{c_char, c_void}; diff --git a/crates/oak-render/src/eval.rs b/crates/oak-render/src/eval.rs index 41d77ef43..e302e2d4d 100644 --- a/crates/oak-render/src/eval.rs +++ b/crates/oak-render/src/eval.rs @@ -149,6 +149,36 @@ pub fn plugin_executor() -> Option> { .clone() } +/// Plugin instance factory: resolves an OFX plugin identifier to a live +/// instance-registry id, creating (and caching) the instance lazily. +/// Implemented by the oakplugin crate — the montage effect path carries +/// plugin identifiers, not instance ids (the montage is resolved from +/// the timeline in the main process; the instance lives in whichever +/// process renders the frame). +pub type PluginInstanceFactory = dyn Fn(&str) -> Option + Send + Sync; + +static PLUGIN_INSTANCE_FACTORY: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +fn instance_factory_slot() -> &'static std::sync::Mutex>> { + PLUGIN_INSTANCE_FACTORY.get_or_init(|| std::sync::Mutex::new(None)) +} + +/// Install the plugin instance factory (oakplugin registration point; +/// `None` clears it). +pub fn set_plugin_instance_factory(factory: Option>) { + *instance_factory_slot().lock().unwrap_or_else(|e| e.into_inner()) = factory; +} + +/// The installed plugin instance factory, if any. +pub fn plugin_instance_factory() -> Option> { + instance_factory_slot() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() +} + /// The failure marker frame: solid magenta (1, 0, 1, 1) F32 RGBA — /// the C++ plugin renderer paints failed plugin output purple so a /// broken plugin is visible instead of silently black. @@ -775,7 +805,11 @@ pub fn render_montage_frame_into( (w, h), PixelFormat::F32, )?; - let (src_data, src_stride) = match &decoded { + // The clip's effect stack runs between decode and compositing + // (C++ semantics: the clip texture passes through the chain + // bottom-up, the chain top feeds the track composite). + let effected = apply_clip_effects(decoded, clip, time); + let (src_data, src_stride) = match &effected { Texture::Cpu(src) => (&src.data, src.linesize_bytes() as i32), _ => continue, }; @@ -825,6 +859,139 @@ pub fn composite_over( } } +// --------------------------------------------------------------------------- +// Montage clip effect stacks +// --------------------------------------------------------------------------- + +/// The built-in Opacity effect's type id (oaknode `OpacityEffect`) — the +/// one built-in video effect with a CPU evaluator on the montage path. +const OPACITY_EFFECT_TYPE_ID: &str = "org.olivevideoeditor.Olive.opacity"; + +/// The Opacity effect's value input id (oaknode `opacity_in`). +const OPACITY_VALUE_INPUT: &str = "opacity_in"; + +/// The effect type ids the montage path already warned about (one log +/// line per type per process instead of one per frame). +fn unsupported_warned() -> std::sync::MutexGuard<'static, std::collections::HashSet> { + static WARNED: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + WARNED + .get_or_init(|| std::sync::Mutex::new(std::collections::HashSet::new())) + .lock() + .unwrap_or_else(|e| e.into_inner()) +} + +/// Log an unsupported-effect passthrough once per type id. +fn warn_unsupported_once(type_id: &str, reason: &str) { + if unsupported_warned().insert(type_id.to_string()) { + eprintln!("montage effect \"{type_id}\" passes through unchanged: {reason}"); + } +} + +/// Run a clip's effect stack over its decoded frame (source-first order; +/// disabled effects are bypassed — the C++ traverser's bypass pushes the +/// effect input through unchanged). Effects the montage path cannot +/// evaluate log a warning once and pass the frame through. +fn apply_clip_effects( + src: Texture, + clip: &crate::ticket::MontageClip, + time: Rational, +) -> Texture { + let mut tex = src; + for effect in &clip.effects { + if !effect.enabled { + continue; + } + tex = apply_montage_effect(tex, effect, time); + } + tex +} + +/// Apply one effect to `src` (an F32 RGBA CPU frame of the montage +/// pipeline). `time` is the sequence time the frame is rendered at (the +/// C++ node evaluation time). +fn apply_montage_effect( + src: Texture, + effect: &crate::ticket::MontageEffect, + time: Rational, +) -> Texture { + // Built-in Opacity: multiply every channel by the opacity factor + // (C++ `:/shaders/opacity.frag`: `frag_color = texture(tex_in, …) * + // opacity_in` — the shader scales the whole vec4, alpha included). + if effect.type_id == OPACITY_EFFECT_TYPE_ID { + let factor = effect + .params + .iter() + .find(|(id, _)| id == OPACITY_VALUE_INPUT) + .map(|(_, v)| v.to_double()) + .unwrap_or(1.0); + // Unity is a pass-through (C++ `qFuzzyCompare(opacity, 1.0)`). + if (factor - 1.0).abs() * 1e12 <= factor.abs().min(1.0) { + return src; + } + // Texture implements Drop, so scale in place through a mutable + // borrow instead of moving the frame out. + let mut tex = src; + match &mut tex { + Texture::Cpu(frame) => { + let stride = frame.linesize_bytes(); + for row in frame.data.chunks_exact_mut(stride).take(frame.height.max(0) as usize) { + for px in row[..(frame.width.max(0) as usize) * 16].chunks_exact_mut(16) { + for c in px.chunks_exact_mut(4) { + let v = f32::from_le_bytes(c.try_into().unwrap()); + c.copy_from_slice(&(v * factor as f32).to_le_bytes()); + } + } + } + } + _ => { + warn_unsupported_once( + &effect.type_id, + "opacity on a non-CPU texture is not supported by the montage path", + ); + } + } + return tex; + } + + // Everything else: an OFX plugin effect. The montage carries the + // plugin identifier; the rendering process resolves it to a live + // instance through the oakplugin-installed factory (lazily created + // and cached per identifier), then dispatches through the plugin + // executor exactly like the graph path's plugin jobs. + let Some(factory) = plugin_instance_factory() else { + warn_unsupported_once( + &effect.type_id, + "no plugin instance factory installed (oakplugin init missing in this process)", + ); + return src; + }; + let Some(instance) = factory(&effect.type_id) else { + warn_unsupported_once( + &effect.type_id, + "no evaluator: unknown built-in effect or OFX plugin unavailable in this process", + ); + return src; + }; + let spec = JobSpec::Plugin { + instance, + time: time.to_f64(), + effect_input_id: effect.effect_input_id.clone(), + inputs: Vec::new(), + values: effect.params.clone(), + }; + let size = src.size(); + match RenderEvalHooks::new().process_plugin_job(src, &spec) { + Ok(texture) => texture, + Err(err) => { + // Unreachable for a Plugin spec (the executor failure path + // yields a purple frame); stay loud rather than silent. + eprintln!("montage plugin job failed to dispatch: {err:#}"); + purple_frame(time, size) + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -1165,4 +1332,118 @@ mod tests { let mut dst = [0u8; 8]; // far too small for 2000x2 f32 samples assert!(render_audio_samples_into(¶ms, &mut dst).is_err()); } + + // ---- Montage clip effect stacks ------------------------------------- + + /// A 2x1 F32 texture filled with a known color. + fn solid_texture(r: f32, g: f32, b: f32, a: f32) -> Texture { + let mut frame = generate_frame(Rational::new(0, 1), (2, 1), PixelFormat::F32).unwrap(); + for px in frame.data.chunks_exact_mut(16) { + for (c, v) in px.chunks_exact_mut(4).zip([r, g, b, a]) { + c.copy_from_slice(&v.to_le_bytes()); + } + } + Texture::Cpu(frame) + } + + fn opacity_effect(enabled: bool, value: f64) -> crate::ticket::MontageEffect { + crate::ticket::MontageEffect { + type_id: OPACITY_EFFECT_TYPE_ID.to_string(), + enabled, + effect_input_id: Some("tex_in".to_string()), + params: vec![(OPACITY_VALUE_INPUT.to_string(), NodeValue::Float(value))], + } + } + + fn clip_with_effects(effects: Vec) -> crate::ticket::MontageClip { + crate::ticket::MontageClip { + filename: String::new(), + stream_index: 0, + in_time: Rational::new(0, 1), + out_time: Rational::new(1, 1), + media_in: Rational::new(0, 1), + gain: 1.0, + effects, + } + } + + /// The built-in Opacity effect really scales the decoded pixels + /// (every channel, C++ `opacity.frag` parity). + #[test] + fn montage_opacity_effect_scales_pixels() { + let clip = clip_with_effects(vec![opacity_effect(true, 0.5)]); + let out = apply_clip_effects(solid_texture(0.8, 0.4, 0.2, 1.0), &clip, Rational::new(0, 1)); + assert_eq!(first_pixel(&out), [0.4, 0.2, 0.1, 0.5]); + } + + /// Disabled effects are bypassed (C++ traverser parity), and unity + /// opacity is a pass-through. + #[test] + fn montage_disabled_or_unity_effects_pass_through() { + let disabled = clip_with_effects(vec![opacity_effect(false, 0.5)]); + let out = apply_clip_effects(solid_texture(0.8, 0.4, 0.2, 1.0), &disabled, Rational::new(0, 1)); + assert_eq!(first_pixel(&out), [0.8, 0.4, 0.2, 1.0]); + + let unity = clip_with_effects(vec![opacity_effect(true, 1.0)]); + let out = apply_clip_effects(solid_texture(0.8, 0.4, 0.2, 1.0), &unity, Rational::new(0, 1)); + assert_eq!(first_pixel(&out), [0.8, 0.4, 0.2, 1.0]); + } + + /// An OFX effect (any non-built-in type id) resolves its instance + /// through the installed factory and dispatches through the executor, + /// carrying the montage's parameter values. + #[test] + fn montage_plugin_effect_dispatches_with_params() { + let _guard = PLUGIN_TEST_LOCK.lock().unwrap(); + set_plugin_instance_factory(Some(Arc::new(|identifier: &str| { + (identifier == "com.example.darken").then_some(7) + }))); + set_plugin_executor(Some(Arc::new(|req: &PluginJobRequest<'_>| { + let JobSpec::Plugin { instance, values, .. } = req.spec else { + return Err(Error::Invalid); + }; + assert_eq!(*instance, 7); + // The injected parameter drives the output: paint the frame + // with the "gain" value so the test observes the param path. + let gain = values + .iter() + .find(|(k, _)| k == "gain") + .map(|(_, v)| v.to_double() as f32) + .unwrap_or(1.0); + let mut frame = generate_frame(Rational::new(0, 1), req.src.size(), PixelFormat::F32)?; + for px in frame.data.chunks_exact_mut(16) { + for (c, v) in px.chunks_exact_mut(4).zip([gain, gain, gain, 1.0]) { + c.copy_from_slice(&v.to_le_bytes()); + } + } + Ok(Texture::Cpu(frame)) + }))); + let clip = clip_with_effects(vec![crate::ticket::MontageEffect { + type_id: "com.example.darken".to_string(), + enabled: true, + effect_input_id: Some("Source".to_string()), + params: vec![("gain".to_string(), NodeValue::Float(0.25))], + }]); + let out = apply_clip_effects(solid_texture(0.8, 0.4, 0.2, 1.0), &clip, Rational::new(0, 1)); + assert_eq!(first_pixel(&out), [0.25, 0.25, 0.25, 1.0]); + set_plugin_executor(None); + set_plugin_instance_factory(None); + } + + /// An effect nobody can evaluate (no factory / unknown type) passes + /// the frame through unchanged — loudly (the warn-once log), never a + /// silent no-op. + #[test] + fn montage_unknown_effect_passes_through() { + let _guard = PLUGIN_TEST_LOCK.lock().unwrap(); + set_plugin_instance_factory(None); + let clip = clip_with_effects(vec![crate::ticket::MontageEffect { + type_id: "com.example.missing".to_string(), + enabled: true, + effect_input_id: Some("Source".to_string()), + params: Vec::new(), + }]); + let out = apply_clip_effects(solid_texture(0.8, 0.4, 0.2, 1.0), &clip, Rational::new(0, 1)); + assert_eq!(first_pixel(&out), [0.8, 0.4, 0.2, 1.0]); + } } diff --git a/crates/oak-render/src/ipc.rs b/crates/oak-render/src/ipc.rs index 3099118c5..ba04f184a 100644 --- a/crates/oak-render/src/ipc.rs +++ b/crates/oak-render/src/ipc.rs @@ -272,6 +272,122 @@ pub struct HelloCapsMsg { pub max_slot_bytes: i64, } +/// A node value on the wire (protocol v2, montage effect parameters). +/// `oak_node::value::NodeValue` itself is not serde-able (texture/sample +/// payloads, handles); this enum covers the plain-data variants an effect +/// parameter can carry. Connection/handle variants (texture, samples, +/// node refs, video/audio params, push buttons) have no wire +/// representation and are dropped by [`WireNodeValue::from_node_value`]. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "t", content = "v", rename_all = "snake_case")] +pub enum WireNodeValue { + /// No value. + None, + /// Integer. + Int(i64), + /// Float. + Float(f64), + /// RGBA color. + Color([f64; 4]), + /// Text. + Text(String), + /// Boolean. + Boolean(bool), + /// Rational (num, den). + Rational(i64, i64), + /// Vec2. + Vec2([f64; 2]), + /// Vec3. + Vec3([f64; 3]), + /// Vec4. + Vec4([f64; 4]), + /// 4x4 matrix, row-major. + Matrix([f64; 16]), + /// Combo index. + Combo(i64), + /// String combo. + StrCombo(String), + /// Opaque bytes. + Binary(Vec), +} + +impl Default for WireNodeValue { + fn default() -> Self { + WireNodeValue::None + } +} + +impl WireNodeValue { + /// The wire form of a node value, or `None` when the variant has no + /// wire representation (the parameter is then not carried). + pub fn from_node_value(v: &oak_node::value::NodeValue) -> Option { + use oak_node::value::NodeValue as NV; + Some(match v { + NV::None => WireNodeValue::None, + NV::Int(i) => WireNodeValue::Int(*i), + NV::Float(f) => WireNodeValue::Float(*f), + NV::Color(c) => WireNodeValue::Color(*c), + NV::Text(s) => WireNodeValue::Text(s.clone()), + NV::Boolean(b) => WireNodeValue::Boolean(*b), + NV::Rational(r) => WireNodeValue::Rational(r.numerator(), r.denominator()), + NV::Vec2(v2) => WireNodeValue::Vec2(*v2), + NV::Vec3(v3) => WireNodeValue::Vec3(*v3), + NV::Vec4(v4) => WireNodeValue::Vec4(*v4), + NV::Matrix(m) => WireNodeValue::Matrix(*m), + NV::Combo(i) => WireNodeValue::Combo(*i), + NV::StrCombo(s) => WireNodeValue::StrCombo(s.clone()), + NV::Binary(b) => WireNodeValue::Binary(b.clone()), + _ => return None, + }) + } + + /// Back to a node value (worker side). + pub fn to_node_value(&self) -> oak_node::value::NodeValue { + use oak_node::value::NodeValue as NV; + match self { + WireNodeValue::None => NV::None, + WireNodeValue::Int(i) => NV::Int(*i), + WireNodeValue::Float(f) => NV::Float(*f), + WireNodeValue::Color(c) => NV::Color(*c), + WireNodeValue::Text(s) => NV::Text(s.clone()), + WireNodeValue::Boolean(b) => NV::Boolean(*b), + WireNodeValue::Rational(n, d) => NV::Rational(oak_core::Rational::new(*n, *d)), + WireNodeValue::Vec2(v2) => NV::Vec2(*v2), + WireNodeValue::Vec3(v3) => NV::Vec3(*v3), + WireNodeValue::Vec4(v4) => NV::Vec4(*v4), + WireNodeValue::Matrix(m) => NV::Matrix(*m), + WireNodeValue::Combo(i) => NV::Combo(*i), + WireNodeValue::StrCombo(s) => NV::StrCombo(s.clone()), + WireNodeValue::Binary(b) => NV::Binary(b.clone()), + } + } +} + +/// One effect parameter on the wire (input id + value). +#[derive(Serialize, Deserialize, Default, Debug, Clone, PartialEq)] +#[serde(default)] +pub struct WireEffectParam { + /// Node input id. + pub input: String, + /// Parameter value. + pub value: WireNodeValue, +} + +/// One effect of a montage clip's effect stack on the wire (protocol v2 +/// additive field of [`WireMontageClip`]; source-first order). +#[derive(Serialize, Deserialize, Default, Debug, Clone, PartialEq)] +#[serde(default)] +pub struct WireMontageEffect { + /// Built-in node type id or OFX plugin identifier. + pub type_id: String, + /// Enabled flag (disabled effects are bypassed). + pub enabled: bool, + /// Effect input (clip) name; "" = none. + pub effect_input_id: String, + /// Parameter values. + pub params: Vec, +} + /// One montage clip on the wire (rationals flattened to num/den pairs). #[derive(Serialize, Deserialize, Default, Debug, Clone, PartialEq)] #[serde(default)] @@ -294,6 +410,47 @@ pub struct WireMontageClip { pub media_in_den: i64, /// Playback gain (1.0 = unity). pub gain: f32, + /// The clip's effect stack (protocol v2 additive: older peers omit the + /// field and it defaults to an empty stack). + pub effects: Vec, +} + +/// Map a ticket-side montage effect to its wire form (main process; +/// parameters without a wire representation are dropped). +pub fn wire_effect_from(effect: &crate::ticket::MontageEffect) -> WireMontageEffect { + WireMontageEffect { + type_id: effect.type_id.clone(), + enabled: effect.enabled, + effect_input_id: effect.effect_input_id.clone().unwrap_or_default(), + params: effect + .params + .iter() + .filter_map(|(input, value)| { + WireNodeValue::from_node_value(value).map(|value| WireEffectParam { + input: input.clone(), + value, + }) + }) + .collect(), + } +} + +/// Map a wire effect back to the ticket-side form (worker). +pub fn montage_effect_from(wire: &WireMontageEffect) -> crate::ticket::MontageEffect { + crate::ticket::MontageEffect { + type_id: wire.type_id.clone(), + enabled: wire.enabled, + effect_input_id: if wire.effect_input_id.is_empty() { + None + } else { + Some(wire.effect_input_id.clone()) + }, + params: wire + .params + .iter() + .map(|p| (p.input.clone(), p.value.to_node_value())) + .collect(), + } } /// One frame ticket inside a [`RenderBatchMsg`] — the main process @@ -1710,6 +1867,15 @@ mod tests { media_in_num: 10, media_in_den: 24, gain: 0.5, + effects: vec![WireMontageEffect { + type_id: "org.olivevideoeditor.Olive.opacity".into(), + enabled: true, + effect_input_id: "tex_in".into(), + params: vec![WireEffectParam { + input: "opacity_in".into(), + value: WireNodeValue::Float(0.5), + }], + }], }], }, ], @@ -1721,6 +1887,15 @@ mod tests { assert_eq!(value["tickets"][0]["format"], SLOT_FORMAT_BGRA8); assert_eq!(value["tickets"][1]["montage"][0]["filename"], "b.mp4"); assert_eq!(value["tickets"][1]["montage"][0]["media_in_num"], 10); + // The effect stack rides the clip as an additive v2 field. + assert_eq!( + value["tickets"][1]["montage"][0]["effects"][0]["type_id"], + "org.olivevideoeditor.Olive.opacity" + ); + assert_eq!( + value["tickets"][1]["montage"][0]["effects"][0]["params"][0]["value"], + json!({ "t": "float", "v": 0.5 }) + ); // Round-trip back to the struct. let round: RenderBatchMsg = serde_json::from_value(value).unwrap(); assert_eq!(round, batch); @@ -1733,6 +1908,49 @@ mod tests { assert!(bare.montage.is_empty()); } + /// Protocol v2 compatibility: a v1-shaped montage clip (no `effects` + /// field) parses with an empty effect stack, and the node-value wire + /// enum round-trips every plain-data variant. + #[test] + fn montage_clip_effects_are_additive_and_values_roundtrip() { + let legacy = json!({ + "filename": "a.mp4", + "stream_index": 0, + "in_num": 0, "in_den": 24, + "out_num": 48, "out_den": 24, + "media_in_num": 0, "media_in_den": 24, + "gain": 1.0, + }); + let parsed: WireMontageClip = serde_json::from_value(legacy).unwrap(); + assert!(parsed.effects.is_empty(), "v1 clips carry no effects"); + + use oak_node::value::NodeValue as NV; + let cases = [ + NV::None, + NV::Int(-3), + NV::Float(0.25), + NV::Color([0.1, 0.2, 0.3, 1.0]), + NV::Text("hello".into()), + NV::Boolean(true), + NV::Rational(oak_core::Rational::new(1, 24)), + NV::Vec2([1.0, 2.0]), + NV::Vec3([1.0, 2.0, 3.0]), + NV::Vec4([1.0, 2.0, 3.0, 4.0]), + NV::Matrix([0.0; 16]), + NV::Combo(2), + NV::StrCombo("choice".into()), + NV::Binary(vec![1, 2, 3]), + ]; + for value in cases { + let wire = WireNodeValue::from_node_value(&value).expect("plain data has a wire form"); + let json = serde_json::to_string(&wire).unwrap(); + let back: WireNodeValue = serde_json::from_str(&json).unwrap(); + assert_eq!(back.to_node_value(), value, "wire round-trip of {value:?}"); + } + // Connection/handle variants have no wire form. + assert!(WireNodeValue::from_node_value(&NV::PushButton).is_none()); + } + #[test] fn batch_accepted_and_frame_failed_wire_roundtrip() { let accepted = BatchAcceptedMsg { @@ -1790,6 +2008,7 @@ mod tests { media_in_num: 10, media_in_den: 24, gain: 0.5, + effects: vec![], }], }], }; diff --git a/crates/oak-render/src/procpool.rs b/crates/oak-render/src/procpool.rs index 2b68360a7..aaea81b7d 100644 --- a/crates/oak-render/src/procpool.rs +++ b/crates/oak-render/src/procpool.rs @@ -1936,6 +1936,7 @@ fn build_ticket_spec( media_in_num: c.media_in.numerator(), media_in_den: c.media_in.denominator(), gain: c.gain, + effects: c.effects.iter().map(crate::ipc::wire_effect_from).collect(), }) .collect(); BatchTicketSpec { @@ -1970,6 +1971,7 @@ fn build_audio_ticket_spec(ticket: i64, slot: u32, params: &AudioTicketParams) - media_in_num: c.media_in.numerator(), media_in_den: c.media_in.denominator(), gain: c.gain, + effects: c.effects.iter().map(crate::ipc::wire_effect_from).collect(), }) .collect(); AudioTicketSpec { diff --git a/crates/oak-render/src/ticket.rs b/crates/oak-render/src/ticket.rs index 617d23ff4..8190c9b1c 100644 --- a/crates/oak-render/src/ticket.rs +++ b/crates/oak-render/src/ticket.rs @@ -35,6 +35,31 @@ use crate::eval; use crate::texture::Texture; use crate::worker::{JobDispatch, JobSchedule}; +/// One effect of a montage clip's effect stack (M14 R3 effect-chain +/// wiring into the montage render path). The stack is ordered +/// source-first (the chain walk's signal order: the first element feeds +/// the media side, the last feeds the clip's effect input); the renderer +/// applies them in sequence after decode, before compositing. +#[derive(Clone, Debug)] +pub struct MontageEffect { + /// Node type id: the built-in factory type id (e.g. + /// `org.olivevideoeditor.Olive.opacity`) or the OFX plugin identifier. + pub type_id: String, + /// The effect's enabled flag. Disabled effects are bypassed (the C++ + /// traverser's bypass pushes the effect input through unchanged), so + /// the renderer skips them; they are carried anyway so the render side + /// can log what it skipped. + pub enabled: bool, + /// The clip/input name the source texture arrives on (the node's + /// effect input id — "tex_in" for built-ins, "Source" for typical OFX + /// filters; `None` when the node has no effect input). + pub effect_input_id: Option, + /// Parameter values (input id -> value): the node's non-hidden, + /// non-connection data inputs at their standard (non-keyframed) + /// values — the same parameter set the inspector exposes. + pub params: Vec<(String, oak_node::value::NodeValue)>, +} + /// One clip of a sequence montage (M12 P0): the facade resolves the /// timeline into an ordered list of clips; the producer decodes each and /// composites them topmost-last. @@ -52,6 +77,9 @@ pub struct MontageClip { pub media_in: Rational, /// Playback gain (1.0 = unity). pub gain: f32, + /// The clip's effect stack (source-first; empty for audio clips and + /// for montage builders that do not resolve effect chains). + pub effects: Vec, } /// Audio ticket parameters (M12 P1): the output format plus the audio diff --git a/crates/oak-task/src/render.rs b/crates/oak-task/src/render.rs index 3a0ca5171..01493dcca 100644 --- a/crates/oak-task/src/render.rs +++ b/crates/oak-task/src/render.rs @@ -61,7 +61,7 @@ use oak_render::procpool::{ bgra8_to_f32_rgba, DispatcherConfig, ProcessDispatcher, ShmAudioRef, ShmFrameRef, }; use oak_render::ticket::{ - ticket_kind, AudioTicketParams, MontageClip, TicketArena, TicketId, TicketPayload, + ticket_kind, AudioTicketParams, MontageClip, MontageEffect, TicketArena, TicketId, TicketPayload, TicketResult, VideoTicketParams, }; use oak_render::worker::JobDispatch; @@ -245,6 +245,90 @@ impl RenderTask { } } + /// The effect stack of a clip block as montage effect descriptors + /// (source-first order). Mirrors the app's `effectchain` walk — + /// oak-task cannot link oak-app, so the export path keeps its own + /// copy over the public oaknode graph API. Disabled effects ride + /// along with `enabled = false`; the worker bypasses them (the C++ + /// traverser's bypass pushes the effect input through unchanged). + /// Parameters are the non-hidden, non-connection data inputs at + /// their standard (non-keyframed) values. + fn clip_effects(graph: &oak_node::graph::Graph, host: oak_node::id::NodeId) -> Vec { + // Walk the chain: from the host's effect input upstream until an + // unconnected input or a node without an effect input; a `seen` + // guard protects against malformed cycles. + let mut chain = Vec::new(); + let mut cur = host; + let mut seen: Vec = Vec::new(); + loop { + if seen.contains(&cur) { + break; + } + seen.push(cur); + let Some(entry) = graph.get(cur) else { + break; + }; + let input = &entry.core.effect_input; + if input.is_empty() { + break; + } + let Some(up) = graph.connected_output(cur, input, -1) else { + break; + }; + chain.push(up); + cur = up; + } + chain.reverse(); + chain + .into_iter() + // The walk runs all the way to the media source; drop the + // source node (the one WITHOUT an effect input) — the montage + // decodes the footage itself. + .filter(|&fx| { + graph + .get(fx) + .map(|e| !e.core.effect_input.is_empty()) + .unwrap_or(false) + }) + .filter_map(|fx| { + let entry = graph.get(fx)?; + let enabled = matches!( + entry.core.standard_value(oak_node::node::ENABLED_INPUT, -1), + oak_node::value::NodeValue::Boolean(true) + ); + let effect_input_id = if entry.core.effect_input.is_empty() { + None + } else { + Some(entry.core.effect_input.clone()) + }; + let mut params = Vec::new(); + for input in &entry.core.inputs { + if matches!( + input.value_type, + oak_node::value::ValueType::Texture + | oak_node::value::ValueType::Samples + | oak_node::value::ValueType::Matrix + ) { + continue; + } + if input.flags & oak_node::input::flags::HIDDEN != 0 { + continue; + } + if input.id == oak_node::node::ENABLED_INPUT { + continue; + } + params.push((input.id.clone(), entry.core.standard_value(&input.id, -1))); + } + Some(MontageEffect { + type_id: entry.behavior.type_id().to_string(), + enabled, + effect_input_id, + params, + }) + }) + .collect() + } + /// Flatten the video tracks of `sequence` into an ordered montage /// (bottom-most track first so the topmost track composites last; /// `// CPP-PARITY: M12 P0 montage contract`). @@ -316,6 +400,7 @@ impl RenderTask { out_time: core.out(), media_in: core.media_in, gain: 1.0, + effects: Self::clip_effects(&guard.graph, *block_id), }); } } @@ -393,6 +478,9 @@ impl RenderTask { out_time: core.out(), media_in: core.media_in, gain: 1.0, + // Audio clips carry no effect stack (video-side + // concept; the mixer never consults it). + effects: Vec::new(), }); } } diff --git a/crates/oak-worker/src/worker.rs b/crates/oak-worker/src/worker.rs index 49f0d46d2..56365912a 100644 --- a/crates/oak-worker/src/worker.rs +++ b/crates/oak-worker/src/worker.rs @@ -975,6 +975,7 @@ impl WorkerSession { out_time: Rational::new(c.out_num, c.out_den), media_in: Rational::new(c.media_in_num, c.media_in_den), gain: c.gain, + effects: c.effects.iter().map(crate::ipc::montage_effect_from).collect(), }) .collect(); Ok(AudioTicketParams { @@ -1069,6 +1070,7 @@ impl WorkerSession { out_time: Rational::new(c.out_num, c.out_den), media_in: Rational::new(c.media_in_num, c.media_in_den), gain: c.gain, + effects: c.effects.iter().map(crate::ipc::montage_effect_from).collect(), }) .collect(); VideoTicketParams { diff --git a/crates/oak-worker/tests/procpool_integration.rs b/crates/oak-worker/tests/procpool_integration.rs index 4af964c86..7c18a0837 100644 --- a/crates/oak-worker/tests/procpool_integration.rs +++ b/crates/oak-worker/tests/procpool_integration.rs @@ -321,6 +321,105 @@ fn worker_decodes_real_footage_into_slot() { dispatcher.shutdown(); } +/// M14 R3 end-to-end: a montage clip carrying an effect stack renders +/// through a REAL worker process — the effects ride the wire (protocol +/// v2 additive `effects` field on the montage clip), the worker applies +/// the stack between decode and compositing, and the main process reads +/// the changed pixels back from the shm slot. 50% Opacity quarters the +/// output (the shader halves every channel; the composite over +/// transparent black halves again via the halved alpha). +#[test] +fn montage_effects_render_through_the_worker() { + let _guard = lock_test(); + let demo = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oak-app/tests/demo.mp4"); + assert!(demo.exists(), "repo fixture tests/demo.mp4 missing"); + + let dispatcher = ProcessDispatcher::new(config(1, 4)).expect("dispatcher config"); + dispatcher.start().expect("worker starts"); + + let clip = |effects| oak_render::ticket::MontageClip { + filename: demo.display().to_string(), + stream_index: 0, + in_time: Rational::new(0, 1), + out_time: Rational::new(10, 1), + media_in: Rational::new(0, 1), + gain: 1.0, + effects, + }; + let montages = [ + vec![clip(Vec::new())], + vec![clip(vec![oak_render::ticket::MontageEffect { + type_id: "org.olivevideoeditor.Olive.opacity".into(), + enabled: true, + effect_input_id: Some("tex_in".into()), + params: vec![( + "opacity_in".into(), + oak_node::value::NodeValue::Float(0.5), + )], + }])], + ]; + + let results = Arc::new(Mutex::new(Vec::new())); + for (i, montage) in montages.into_iter().enumerate() { + let results = results.clone(); + let time = Rational::new(i as i64, 25); + let mut p = params(time, None).as_ref().clone(); + p.montage = montage; + let job = Job { + node_identity: 1, + time, + params: Arc::new(p), + audio: None, + produce: Arc::new(|_, _| { + Err(oak_render::error::Error::Failed( + "process backend does not use the in-process producer".into(), + )) + }), + done: Box::new(move |result| { + results.lock().unwrap_or_else(|e| e.into_inner()).push(result); + }), + schedule: JobSchedule::seek(), + }; + assert!(dispatcher.post(job), "post accepted while alive"); + } + pump_until(&dispatcher, &results, 2); + + // Results arrive in ticket order (one batch, in-order rendering). + let mut frames: Vec> = Vec::new(); + for result in results.lock().unwrap().drain(..) { + let payload = result.expect("montage frame rendered"); + let TicketPayload::ShmFrame(frame) = payload else { + panic!("ShmFrame payload"); + }; + frames.push(frame.shm.slot_bytes(frame.slot)[..frame.meta.data_size as usize].to_vec()); + dispatcher.release_frame(&frame); + } + dispatcher.shutdown(); + + let [plain, dimmed] = &frames[..] else { + panic!("two frames rendered"); + }; + // Pick an opaque, non-black pixel in the plain render as the probe. + let probe = plain + .chunks_exact(4) + .position(|px| px[3] == 255 && px[0] > 40) + .expect("the fixture frame has an opaque non-black pixel"); + let p = &plain[probe * 4..probe * 4 + 4]; + let d = &dimmed[probe * 4..probe * 4 + 4]; + assert_eq!(p[3], 255, "the plain montage render is opaque"); + assert!( + (d[3] as i32 - 128).abs() <= 3, + "50% opacity halves the alpha ({} vs 128)", + d[3] + ); + assert!( + (d[0] as i32 - p[0] as i32 / 4).abs() <= 6, + "50% opacity quarters the color ({} vs {}/4)", + d[0], + p[0] + ); +} + /// Submit an empty-montage audio range pull through the dispatcher /// (M15 S3): the worker mixes silence into a shm slot and the main /// process reads it back as [`TicketPayload::ShmAudio`].