render: stack higher-numbered video tracks on top
Match the timeline UI (V_max drawn topmost): composite tracks from V1 up to V_max so the highest-numbered track is composited last, in both the montage path and direct graph evaluation.
This commit is contained in:
@@ -175,9 +175,11 @@ fn clip_effects(g: &oak_node::graph::Graph, block_id: NodeId) -> Vec<oak_render:
|
||||
}
|
||||
|
||||
/// The video montage at sequence time `time`: every clip covering `time`
|
||||
/// on video tracks, ordered bottom-to-top (track index 0 is topmost, so
|
||||
/// it is composited last). Hidden tracks (the muted flag doubles as the
|
||||
/// video visibility toggle, Olive parity) contribute nothing.
|
||||
/// on video tracks, ordered bottom-to-top (the highest-numbered track —
|
||||
/// the list's last — is topmost, so it is composited last; NLE stacking
|
||||
/// matches the timeline UI, which shows the highest-numbered track on
|
||||
/// top). Hidden tracks (the muted flag doubles as the video visibility
|
||||
/// toggle, Olive parity) contribute nothing.
|
||||
pub fn video_montage(p: &ProjectRef, seq: NodeId, time: Rational) -> Vec<MontageClip> {
|
||||
let g = lock(p);
|
||||
let mut clips = Vec::new();
|
||||
@@ -191,7 +193,7 @@ pub fn video_montage(p: &ProjectRef, seq: NodeId, time: Rational) -> Vec<Montage
|
||||
if list.kind != TrackType::Video {
|
||||
continue;
|
||||
}
|
||||
for &track_id in list.tracks.iter().rev() {
|
||||
for &track_id in list.tracks.iter() {
|
||||
let Some(track) = track_behavior(&g.graph, track_id) else {
|
||||
continue;
|
||||
};
|
||||
@@ -959,6 +961,73 @@ mod tests {
|
||||
let _ = std::fs::remove_file(&media);
|
||||
}
|
||||
|
||||
/// NLE stacking regression: two OPAQUE solid-color clips covering the
|
||||
/// same time on V1 (red) and V2 (blue) — the montage lists V1's clip
|
||||
/// first and V2's LAST (the topmost composites last), and the rendered
|
||||
/// frame shows V2's blue (the highest-numbered track wins, matching
|
||||
/// the timeline UI).
|
||||
#[test]
|
||||
fn video_montage_stacks_highest_track_on_top() {
|
||||
let _media = media_lock();
|
||||
oak_undo::global::clear().unwrap();
|
||||
let red =
|
||||
std::env::temp_dir().join(format!("oakapp_stack_red_{}.mp4", std::process::id()));
|
||||
let blue =
|
||||
std::env::temp_dir().join(format!("oakapp_stack_blue_{}.mp4", std::process::id()));
|
||||
oak_codec::testmedia::write_test_clip_solid(&red, 64, 64, 10, 10, [0.9, 0.1, 0.1, 1.0])
|
||||
.expect("generate the red media");
|
||||
oak_codec::testmedia::write_test_clip_solid(&blue, 64, 64, 10, 10, [0.1, 0.1, 0.9, 1.0])
|
||||
.expect("generate the blue media");
|
||||
|
||||
let project = graphops::create_project();
|
||||
let seq = graphops::create_sequence(&project, "Stack Montage");
|
||||
let red_footage = graphops::import_footage(&project, &red).expect("import the red media");
|
||||
let blue_footage = graphops::import_footage(&project, &blue).expect("import the blue media");
|
||||
graphops::place_footage_clip(&project, seq, red_footage, TrackType::Video, 0, 0, 10, 0)
|
||||
.expect("place the V1 (red) clip");
|
||||
graphops::place_footage_clip(&project, seq, blue_footage, TrackType::Video, 1, 0, 10, 0)
|
||||
.expect("place the V2 (blue) clip");
|
||||
let tb = graphops::sequence_time_base(&lock(&project).graph, seq).unwrap();
|
||||
let time = graphops::ts_to_rational(0, tb);
|
||||
|
||||
// Bottom-to-top: V1's (red) clip first, V2's (blue) last.
|
||||
let montage = video_montage(&project, seq, time);
|
||||
assert_eq!(montage.len(), 2, "both clips cover frame 0");
|
||||
assert_eq!(montage[0].filename, red.to_string_lossy(), "V1's clip is the bottom of the stack");
|
||||
assert_eq!(
|
||||
montage[1].filename,
|
||||
blue.to_string_lossy(),
|
||||
"V2's clip is the top of the stack (composited last)"
|
||||
);
|
||||
|
||||
// Render through the same entry point the render worker uses: V2's
|
||||
// blue covers V1's red.
|
||||
let params = VideoTicketParams {
|
||||
viewer: 0,
|
||||
project: String::new(),
|
||||
time,
|
||||
force_size: Some((64, 64)),
|
||||
force_format: Some(oak_core::PixelFormat::F32),
|
||||
cache: None,
|
||||
cache_dir: None,
|
||||
cache_id: None,
|
||||
cache_timebase: None,
|
||||
footage: None,
|
||||
montage,
|
||||
};
|
||||
let mut dst = vec![0u8; 64 * 64 * 16];
|
||||
oak_render::eval::render_montage_frame_into(time, ¶ms, (64, 64), &mut dst, 64 * 16)
|
||||
.expect("montage render");
|
||||
let off = (8 * 64 + 8) * 16;
|
||||
let r = f32::from_le_bytes(dst[off..off + 4].try_into().unwrap());
|
||||
let b = f32::from_le_bytes(dst[off + 8..off + 12].try_into().unwrap());
|
||||
assert!(b > 0.5 && r < 0.4, "V2's blue covers V1's red (r={r}, b={b})");
|
||||
|
||||
oak_undo::global::clear().unwrap();
|
||||
let _ = std::fs::remove_file(&red);
|
||||
let _ = std::fs::remove_file(&blue);
|
||||
}
|
||||
|
||||
/// The acceptance gate for montage effect rendering: a 50% Opacity on
|
||||
/// a real clip (generated media, real graph, real decode, real
|
||||
/// compositing — nothing mocked) must change the rendered pixels, and
|
||||
|
||||
@@ -492,8 +492,8 @@ fn clip_media(g: &Graph, block_id: NodeId) -> Option<(String, i32)> {
|
||||
}
|
||||
|
||||
/// The video montage at sequence time `time`: every clip covering `time`
|
||||
/// on video tracks, ordered bottom-to-top (track index 0 is topmost, so
|
||||
/// it is composited last).
|
||||
/// on video tracks, ordered bottom-to-top (the highest-numbered track —
|
||||
/// the list's last — is topmost, so it is composited last).
|
||||
pub fn video_montage(p: &ProjectRef, seq_id: NodeId, time: Rational) -> Vec<MontageClip> {
|
||||
let g = lock(p);
|
||||
let mut clips = Vec::new();
|
||||
@@ -507,7 +507,7 @@ pub fn video_montage(p: &ProjectRef, seq_id: NodeId, time: Rational) -> Vec<Mont
|
||||
if list.kind != TrackType::Video {
|
||||
continue;
|
||||
}
|
||||
for &track_id in list.tracks.iter().rev() {
|
||||
for &track_id in list.tracks.iter() {
|
||||
let Some(track) = track_behavior(&g.graph, track_id) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
@@ -58,23 +58,7 @@ pub fn write_test_clip(
|
||||
if width <= 0 || height <= 0 || frame_count <= 0 || fps <= 0 {
|
||||
return Err(Error::Invalid);
|
||||
}
|
||||
let mut params = EncodingParams::default();
|
||||
let name = out.as_os_str().as_encoded_bytes();
|
||||
if name.len() >= params.filename.len() {
|
||||
return Err(Error::Failed("output path too long".into()));
|
||||
}
|
||||
params.filename[..name.len()].copy_from_slice(name);
|
||||
params.format = 2; // MPEG-4 video container
|
||||
params.video_enabled = 1;
|
||||
params.video_codec = 10; // MPEG-2 (B-frame-free; the H.264 B-frame streams hit a muxer timing bug)
|
||||
params.video_width = width;
|
||||
params.video_height = height;
|
||||
params.video_time_base_num = 1;
|
||||
params.video_time_base_den = fps;
|
||||
params.video_pixel_format = PixelFormat::F32;
|
||||
params.video_interlacing = 0;
|
||||
params.video_pixel_aspect_num = 1;
|
||||
params.video_pixel_aspect_den = 1;
|
||||
let mut params = video_params(out, width, height, fps)?;
|
||||
// A stereo PCM audio track with a known 440 Hz sine (M12 P1: the
|
||||
// audio-render path needs a decodable audio stream; PCM avoids
|
||||
// codec sample-format negotiation issues in this FFmpeg pairing).
|
||||
@@ -103,6 +87,75 @@ pub fn write_test_clip(
|
||||
encoder.flush()
|
||||
}
|
||||
|
||||
/// Encode `frame_count` frames of a solid `rgba` color into `out` (same
|
||||
/// MPEG-2/MP4 pairing as [`write_test_clip`], video only). The stacking
|
||||
/// tests need two OPAQUE clips of different known colors covering the
|
||||
/// same time — the sweeping pattern cannot provide that (every clip's
|
||||
/// frame 0 is identical, and decode-time seeking is not exercised here).
|
||||
pub fn write_test_clip_solid(
|
||||
out: &Path,
|
||||
width: i32,
|
||||
height: i32,
|
||||
frame_count: i32,
|
||||
fps: i32,
|
||||
rgba: [f32; 4],
|
||||
) -> Result<()> {
|
||||
if width <= 0 || height <= 0 || frame_count <= 0 || fps <= 0 {
|
||||
return Err(Error::Invalid);
|
||||
}
|
||||
let params = video_params(out, width, height, fps)?;
|
||||
let encoder = create_from_params(¶ms)
|
||||
.ok_or_else(|| Error::Failed("no encoder for test clip params".into()))?;
|
||||
encoder.configure(¶ms)?;
|
||||
encoder.open()?;
|
||||
for i in 0..frame_count {
|
||||
encoder.write_video(&solid_frame(i, width, height, fps, rgba))?;
|
||||
}
|
||||
encoder.flush()
|
||||
}
|
||||
|
||||
/// The shared MPEG-2-video-in-MP4 encoder params of the test clips
|
||||
/// (B-frame-free; the H.264 B-frame streams hit a muxer timing bug).
|
||||
fn video_params(out: &Path, width: i32, height: i32, fps: i32) -> Result<EncodingParams> {
|
||||
let mut params = EncodingParams::default();
|
||||
let name = out.as_os_str().as_encoded_bytes();
|
||||
if name.len() >= params.filename.len() {
|
||||
return Err(Error::Failed("output path too long".into()));
|
||||
}
|
||||
params.filename[..name.len()].copy_from_slice(name);
|
||||
params.format = 2; // MPEG-4 video container
|
||||
params.video_enabled = 1;
|
||||
params.video_codec = 10; // MPEG-2 (B-frame-free; the H.264 B-frame streams hit a muxer timing bug)
|
||||
params.video_width = width;
|
||||
params.video_height = height;
|
||||
params.video_time_base_num = 1;
|
||||
params.video_time_base_den = fps;
|
||||
params.video_pixel_format = PixelFormat::F32;
|
||||
params.video_interlacing = 0;
|
||||
params.video_pixel_aspect_num = 1;
|
||||
params.video_pixel_aspect_den = 1;
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
/// One frame of solid `rgba` (F32 RGBA rows, like [`pattern_frame`]).
|
||||
fn solid_frame(i: i32, width: i32, height: i32, fps: i32, rgba: [f32; 4]) -> Frame {
|
||||
let mut f = pattern_frame(0, width, height, fps);
|
||||
f.set_timestamp(Rational::new(i as i64, fps as i64));
|
||||
let linesize = f.linesize_bytes() as usize;
|
||||
let data = f.data_mut().expect("test frame buffer");
|
||||
let [r, g, b, a] = rgba;
|
||||
for y in 0..height as usize {
|
||||
for x in 0..width as usize {
|
||||
let off = y * linesize + x * 16;
|
||||
data[off..off + 4].copy_from_slice(&r.to_le_bytes());
|
||||
data[off + 4..off + 8].copy_from_slice(&g.to_le_bytes());
|
||||
data[off + 8..off + 12].copy_from_slice(&b.to_le_bytes());
|
||||
data[off + 12..off + 16].copy_from_slice(&a.to_le_bytes());
|
||||
}
|
||||
}
|
||||
f
|
||||
}
|
||||
|
||||
/// One frame of the known pattern (see module doc).
|
||||
fn pattern_frame(i: i32, width: i32, height: i32, fps: i32) -> Frame {
|
||||
let mut vp = VideoParams::new_basic(
|
||||
|
||||
@@ -1008,8 +1008,9 @@ fn composite_tracks(frames: Vec<Frame>, size: (i32, i32)) -> Frame {
|
||||
/// Render one frame of `viewer` (a sequence) at `time`: evaluate every
|
||||
/// enabled clip overlapping `time` through the node graph (one traverser
|
||||
/// pass per clip; the hooks' decoder cache is shared across clips) and
|
||||
/// composite the resulting frames topmost-first — in the video track
|
||||
/// list, track 0 is the topmost stack element (C++ `TrackList` order).
|
||||
/// composite the resulting frames bottommost-first — in the video track
|
||||
/// list the LAST track (the highest-numbered one) is the topmost stack
|
||||
/// element (NLE stacking, matching the timeline UI).
|
||||
///
|
||||
/// `size` is the decode target for every clip, so all frames composite
|
||||
/// without per-frame scaling. Errors: `Invalid` for a non-F32 format or a
|
||||
@@ -1038,8 +1039,9 @@ pub fn render_graph_frame(
|
||||
|
||||
// Collect the clips covering `time`: the sequence's track lists (video
|
||||
// then audio — C++ `Sequence` keeps them in the `k_track_input_format`
|
||||
// array order), the video list's tracks in stack order, then each
|
||||
// track's blocks.
|
||||
// array order), the video list's tracks topmost-first (the list's last
|
||||
// track is the top of the stack; `composite_tracks` walks the frames in
|
||||
// reverse and draws the first frame last), then each track's blocks.
|
||||
let mut clips: Vec<oak_node::id::NodeId> = Vec::new();
|
||||
for tl_id in &sequence.track_lists {
|
||||
let Some(tl) = graph.get(*tl_id) else {
|
||||
@@ -1055,7 +1057,7 @@ pub fn render_graph_frame(
|
||||
if tl.kind != oak_node::track::TrackType::Video {
|
||||
continue;
|
||||
}
|
||||
for track_id in &tl.tracks {
|
||||
for track_id in tl.tracks.iter().rev() {
|
||||
let Some(track) = graph.get(*track_id) else {
|
||||
continue;
|
||||
};
|
||||
@@ -1297,7 +1299,8 @@ fn scale_rgba_f32(
|
||||
}
|
||||
|
||||
/// Composite the montage at `time`: decode each covering clip and
|
||||
/// alpha-composite topmost-last (C++ track order: track 0 is topmost).
|
||||
/// alpha-composite topmost-last (NLE track order: the highest-numbered
|
||||
/// track is topmost).
|
||||
fn render_montage_frame(
|
||||
time: Rational,
|
||||
params: &crate::ticket::VideoTicketParams,
|
||||
|
||||
@@ -19,7 +19,8 @@
|
||||
//! Builds a real node graph (sequence -> video track list -> video tracks
|
||||
//! -> clip blocks -> footage), evaluates the clip overlapping the request
|
||||
//! time through the traverser and composites the decoded frames — the same
|
||||
//! path the engine's viewer uses. Track 0 is the topmost stack element.
|
||||
//! path the engine's viewer uses. The LAST track is the topmost stack
|
||||
//! element (NLE stacking: the highest-numbered track wins).
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -43,7 +44,8 @@ fn clip_path(tag: &str) -> std::path::PathBuf {
|
||||
}
|
||||
|
||||
/// One sequence + one video track list with one track per clip
|
||||
/// `(filename, [in, out))`. Track 0 (the first entry) composites on top.
|
||||
/// `(filename, [in, out))`. The LAST entry's track composites on top
|
||||
/// (NLE stacking: the highest-numbered track is topmost).
|
||||
fn build_project(clips: &[(&str, Rational, Rational)]) -> (Arc<Mutex<Project>>, NodeId) {
|
||||
let project = Project::new();
|
||||
let seq;
|
||||
@@ -191,6 +193,51 @@ fn graph_sequence_renders_two_tracks() {
|
||||
let _ = std::fs::remove_file(&path_b);
|
||||
}
|
||||
|
||||
/// NLE stacking regression: two OPAQUE solid-color clips covering the
|
||||
/// same time on two video tracks — the clip on the LAST track (V2, blue)
|
||||
/// composites on top of the clip on the first track (V1, red), matching
|
||||
/// the timeline UI (the highest-numbered track displays on top).
|
||||
#[test]
|
||||
fn graph_sequence_stacks_highest_track_on_top() {
|
||||
let red = clip_path("stack_red");
|
||||
let blue = clip_path("stack_blue");
|
||||
oak_codec::testmedia::write_test_clip_solid(&red, 64, 64, 10, 10, [0.9, 0.1, 0.1, 1.0])
|
||||
.expect("red clip generation");
|
||||
oak_codec::testmedia::write_test_clip_solid(&blue, 64, 64, 10, 10, [0.1, 0.1, 0.9, 1.0])
|
||||
.expect("blue clip generation");
|
||||
|
||||
// V1 = red (bottom), V2 = blue (top).
|
||||
let (project, seq) = build_project(&[
|
||||
(&red.to_string_lossy(), Rational::new(0, 1), Rational::new(1, 1)),
|
||||
(&blue.to_string_lossy(), Rational::new(0, 1), Rational::new(1, 1)),
|
||||
]);
|
||||
let tex = oak_render::eval::render_graph_frame(&project, seq, Rational::new(0, 1), (64, 64), PixelFormat::F32)
|
||||
.expect("stacked render");
|
||||
let data = frame_data(&tex);
|
||||
assert!(
|
||||
channel(data, 8, 8, 2) > 0.5 && channel(data, 8, 8, 0) < 0.4,
|
||||
"V2's blue covers V1's red (r={}, b={})",
|
||||
channel(data, 8, 8, 0),
|
||||
channel(data, 8, 8, 2)
|
||||
);
|
||||
|
||||
// Distinguishability guard: solo, the V1 clip really is red (the two
|
||||
// tracks carry different content).
|
||||
let (solo, solo_seq) = build_project(&[(&red.to_string_lossy(), Rational::new(0, 1), Rational::new(1, 1))]);
|
||||
let solo_tex = oak_render::eval::render_graph_frame(&solo, solo_seq, Rational::new(0, 1), (64, 64), PixelFormat::F32)
|
||||
.expect("solo V1 render");
|
||||
let solo_data = frame_data(&solo_tex);
|
||||
assert!(
|
||||
channel(solo_data, 8, 8, 0) > 0.5 && channel(solo_data, 8, 8, 2) < 0.4,
|
||||
"solo V1 is red (r={}, b={})",
|
||||
channel(solo_data, 8, 8, 0),
|
||||
channel(solo_data, 8, 8, 2)
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&red);
|
||||
let _ = std::fs::remove_file(&blue);
|
||||
}
|
||||
|
||||
/// The driver rejects bad arguments explainably: non-F32 format, a
|
||||
/// non-positive size, and a missing viewer.
|
||||
#[test]
|
||||
|
||||
@@ -330,7 +330,8 @@ impl RenderTask {
|
||||
}
|
||||
|
||||
/// Flatten the video tracks of `sequence` into an ordered montage
|
||||
/// (bottom-most track first so the topmost track composites last;
|
||||
/// (bottom-most track first so the topmost track — the highest-numbered
|
||||
/// one, the list's last — composites last;
|
||||
/// `// CPP-PARITY: M12 P0 montage contract`).
|
||||
fn video_montage(project: &ProjectRef, sequence: oak_node::id::NodeId, time: Rational) -> Vec<MontageClip> {
|
||||
let guard = project
|
||||
@@ -361,7 +362,7 @@ impl RenderTask {
|
||||
if list.kind != TrackType::Video {
|
||||
continue;
|
||||
}
|
||||
for track_id in list.tracks.iter().rev() {
|
||||
for track_id in list.tracks.iter() {
|
||||
let Some(te) = guard.graph.get(*track_id) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user