render: stack higher-numbered video tracks on top

Match the timeline UI (V_max drawn topmost): composite tracks from
V1 up to V_max so the highest-numbered track is composited last, in
both the montage path and direct graph evaluation.
This commit is contained in:
2026-08-27 15:20:18 +08:00
parent 663d879908
commit c03f1ec604
6 changed files with 207 additions and 34 deletions
+73 -4
View File
@@ -175,9 +175,11 @@ fn clip_effects(g: &oak_node::graph::Graph, block_id: NodeId) -> Vec<oak_render:
}
/// The video montage at sequence time `time`: every clip covering `time`
/// on video tracks, ordered bottom-to-top (track index 0 is topmost, so
/// it is composited last). Hidden tracks (the muted flag doubles as the
/// video visibility toggle, Olive parity) contribute nothing.
/// on video tracks, ordered bottom-to-top (the highest-numbered track —
/// the list's last — is topmost, so it is composited last; NLE stacking
/// matches the timeline UI, which shows the highest-numbered track on
/// top). Hidden tracks (the muted flag doubles as the video visibility
/// toggle, Olive parity) contribute nothing.
pub fn video_montage(p: &ProjectRef, seq: NodeId, time: Rational) -> Vec<MontageClip> {
let g = lock(p);
let mut clips = Vec::new();
@@ -191,7 +193,7 @@ pub fn video_montage(p: &ProjectRef, seq: NodeId, time: Rational) -> Vec<Montage
if list.kind != TrackType::Video {
continue;
}
for &track_id in list.tracks.iter().rev() {
for &track_id in list.tracks.iter() {
let Some(track) = track_behavior(&g.graph, track_id) else {
continue;
};
@@ -959,6 +961,73 @@ mod tests {
let _ = std::fs::remove_file(&media);
}
/// NLE stacking regression: two OPAQUE solid-color clips covering the
/// same time on V1 (red) and V2 (blue) — the montage lists V1's clip
/// first and V2's LAST (the topmost composites last), and the rendered
/// frame shows V2's blue (the highest-numbered track wins, matching
/// the timeline UI).
#[test]
fn video_montage_stacks_highest_track_on_top() {
let _media = media_lock();
oak_undo::global::clear().unwrap();
let red =
std::env::temp_dir().join(format!("oakapp_stack_red_{}.mp4", std::process::id()));
let blue =
std::env::temp_dir().join(format!("oakapp_stack_blue_{}.mp4", std::process::id()));
oak_codec::testmedia::write_test_clip_solid(&red, 64, 64, 10, 10, [0.9, 0.1, 0.1, 1.0])
.expect("generate the red media");
oak_codec::testmedia::write_test_clip_solid(&blue, 64, 64, 10, 10, [0.1, 0.1, 0.9, 1.0])
.expect("generate the blue media");
let project = graphops::create_project();
let seq = graphops::create_sequence(&project, "Stack Montage");
let red_footage = graphops::import_footage(&project, &red).expect("import the red media");
let blue_footage = graphops::import_footage(&project, &blue).expect("import the blue media");
graphops::place_footage_clip(&project, seq, red_footage, TrackType::Video, 0, 0, 10, 0)
.expect("place the V1 (red) clip");
graphops::place_footage_clip(&project, seq, blue_footage, TrackType::Video, 1, 0, 10, 0)
.expect("place the V2 (blue) clip");
let tb = graphops::sequence_time_base(&lock(&project).graph, seq).unwrap();
let time = graphops::ts_to_rational(0, tb);
// Bottom-to-top: V1's (red) clip first, V2's (blue) last.
let montage = video_montage(&project, seq, time);
assert_eq!(montage.len(), 2, "both clips cover frame 0");
assert_eq!(montage[0].filename, red.to_string_lossy(), "V1's clip is the bottom of the stack");
assert_eq!(
montage[1].filename,
blue.to_string_lossy(),
"V2's clip is the top of the stack (composited last)"
);
// Render through the same entry point the render worker uses: V2's
// blue covers V1's red.
let params = VideoTicketParams {
viewer: 0,
project: String::new(),
time,
force_size: Some((64, 64)),
force_format: Some(oak_core::PixelFormat::F32),
cache: None,
cache_dir: None,
cache_id: None,
cache_timebase: None,
footage: None,
montage,
};
let mut dst = vec![0u8; 64 * 64 * 16];
oak_render::eval::render_montage_frame_into(time, &params, (64, 64), &mut dst, 64 * 16)
.expect("montage render");
let off = (8 * 64 + 8) * 16;
let r = f32::from_le_bytes(dst[off..off + 4].try_into().unwrap());
let b = f32::from_le_bytes(dst[off + 8..off + 12].try_into().unwrap());
assert!(b > 0.5 && r < 0.4, "V2's blue covers V1's red (r={r}, b={b})");
oak_undo::global::clear().unwrap();
let _ = std::fs::remove_file(&red);
let _ = std::fs::remove_file(&blue);
}
/// The acceptance gate for montage effect rendering: a 50% Opacity on
/// a real clip (generated media, real graph, real decode, real
/// compositing — nothing mocked) must change the rendered pixels, and
+3 -3
View File
@@ -492,8 +492,8 @@ fn clip_media(g: &Graph, block_id: NodeId) -> Option<(String, i32)> {
}
/// The video montage at sequence time `time`: every clip covering `time`
/// on video tracks, ordered bottom-to-top (track index 0 is topmost, so
/// it is composited last).
/// on video tracks, ordered bottom-to-top (the highest-numbered track —
/// the list's last — is topmost, so it is composited last).
pub fn video_montage(p: &ProjectRef, seq_id: NodeId, time: Rational) -> Vec<MontageClip> {
let g = lock(p);
let mut clips = Vec::new();
@@ -507,7 +507,7 @@ pub fn video_montage(p: &ProjectRef, seq_id: NodeId, time: Rational) -> Vec<Mont
if list.kind != TrackType::Video {
continue;
}
for &track_id in list.tracks.iter().rev() {
for &track_id in list.tracks.iter() {
let Some(track) = track_behavior(&g.graph, track_id) else {
continue;
};
+70 -17
View File
@@ -58,23 +58,7 @@ pub fn write_test_clip(
if width <= 0 || height <= 0 || frame_count <= 0 || fps <= 0 {
return Err(Error::Invalid);
}
let mut params = EncodingParams::default();
let name = out.as_os_str().as_encoded_bytes();
if name.len() >= params.filename.len() {
return Err(Error::Failed("output path too long".into()));
}
params.filename[..name.len()].copy_from_slice(name);
params.format = 2; // MPEG-4 video container
params.video_enabled = 1;
params.video_codec = 10; // MPEG-2 (B-frame-free; the H.264 B-frame streams hit a muxer timing bug)
params.video_width = width;
params.video_height = height;
params.video_time_base_num = 1;
params.video_time_base_den = fps;
params.video_pixel_format = PixelFormat::F32;
params.video_interlacing = 0;
params.video_pixel_aspect_num = 1;
params.video_pixel_aspect_den = 1;
let mut params = video_params(out, width, height, fps)?;
// A stereo PCM audio track with a known 440 Hz sine (M12 P1: the
// audio-render path needs a decodable audio stream; PCM avoids
// codec sample-format negotiation issues in this FFmpeg pairing).
@@ -103,6 +87,75 @@ pub fn write_test_clip(
encoder.flush()
}
/// Encode `frame_count` frames of a solid `rgba` color into `out` (same
/// MPEG-2/MP4 pairing as [`write_test_clip`], video only). The stacking
/// tests need two OPAQUE clips of different known colors covering the
/// same time — the sweeping pattern cannot provide that (every clip's
/// frame 0 is identical, and decode-time seeking is not exercised here).
pub fn write_test_clip_solid(
out: &Path,
width: i32,
height: i32,
frame_count: i32,
fps: i32,
rgba: [f32; 4],
) -> Result<()> {
if width <= 0 || height <= 0 || frame_count <= 0 || fps <= 0 {
return Err(Error::Invalid);
}
let params = video_params(out, width, height, fps)?;
let encoder = create_from_params(&params)
.ok_or_else(|| Error::Failed("no encoder for test clip params".into()))?;
encoder.configure(&params)?;
encoder.open()?;
for i in 0..frame_count {
encoder.write_video(&solid_frame(i, width, height, fps, rgba))?;
}
encoder.flush()
}
/// The shared MPEG-2-video-in-MP4 encoder params of the test clips
/// (B-frame-free; the H.264 B-frame streams hit a muxer timing bug).
fn video_params(out: &Path, width: i32, height: i32, fps: i32) -> Result<EncodingParams> {
let mut params = EncodingParams::default();
let name = out.as_os_str().as_encoded_bytes();
if name.len() >= params.filename.len() {
return Err(Error::Failed("output path too long".into()));
}
params.filename[..name.len()].copy_from_slice(name);
params.format = 2; // MPEG-4 video container
params.video_enabled = 1;
params.video_codec = 10; // MPEG-2 (B-frame-free; the H.264 B-frame streams hit a muxer timing bug)
params.video_width = width;
params.video_height = height;
params.video_time_base_num = 1;
params.video_time_base_den = fps;
params.video_pixel_format = PixelFormat::F32;
params.video_interlacing = 0;
params.video_pixel_aspect_num = 1;
params.video_pixel_aspect_den = 1;
Ok(params)
}
/// One frame of solid `rgba` (F32 RGBA rows, like [`pattern_frame`]).
fn solid_frame(i: i32, width: i32, height: i32, fps: i32, rgba: [f32; 4]) -> Frame {
let mut f = pattern_frame(0, width, height, fps);
f.set_timestamp(Rational::new(i as i64, fps as i64));
let linesize = f.linesize_bytes() as usize;
let data = f.data_mut().expect("test frame buffer");
let [r, g, b, a] = rgba;
for y in 0..height as usize {
for x in 0..width as usize {
let off = y * linesize + x * 16;
data[off..off + 4].copy_from_slice(&r.to_le_bytes());
data[off + 4..off + 8].copy_from_slice(&g.to_le_bytes());
data[off + 8..off + 12].copy_from_slice(&b.to_le_bytes());
data[off + 12..off + 16].copy_from_slice(&a.to_le_bytes());
}
}
f
}
/// One frame of the known pattern (see module doc).
fn pattern_frame(i: i32, width: i32, height: i32, fps: i32) -> Frame {
let mut vp = VideoParams::new_basic(
+9 -6
View File
@@ -1008,8 +1008,9 @@ fn composite_tracks(frames: Vec<Frame>, size: (i32, i32)) -> Frame {
/// Render one frame of `viewer` (a sequence) at `time`: evaluate every
/// enabled clip overlapping `time` through the node graph (one traverser
/// pass per clip; the hooks' decoder cache is shared across clips) and
/// composite the resulting frames topmost-first — in the video track
/// list, track 0 is the topmost stack element (C++ `TrackList` order).
/// composite the resulting frames bottommost-first — in the video track
/// list the LAST track (the highest-numbered one) is the topmost stack
/// element (NLE stacking, matching the timeline UI).
///
/// `size` is the decode target for every clip, so all frames composite
/// without per-frame scaling. Errors: `Invalid` for a non-F32 format or a
@@ -1038,8 +1039,9 @@ pub fn render_graph_frame(
// Collect the clips covering `time`: the sequence's track lists (video
// then audio — C++ `Sequence` keeps them in the `k_track_input_format`
// array order), the video list's tracks in stack order, then each
// track's blocks.
// array order), the video list's tracks topmost-first (the list's last
// track is the top of the stack; `composite_tracks` walks the frames in
// reverse and draws the first frame last), then each track's blocks.
let mut clips: Vec<oak_node::id::NodeId> = Vec::new();
for tl_id in &sequence.track_lists {
let Some(tl) = graph.get(*tl_id) else {
@@ -1055,7 +1057,7 @@ pub fn render_graph_frame(
if tl.kind != oak_node::track::TrackType::Video {
continue;
}
for track_id in &tl.tracks {
for track_id in tl.tracks.iter().rev() {
let Some(track) = graph.get(*track_id) else {
continue;
};
@@ -1297,7 +1299,8 @@ fn scale_rgba_f32(
}
/// Composite the montage at `time`: decode each covering clip and
/// alpha-composite topmost-last (C++ track order: track 0 is topmost).
/// alpha-composite topmost-last (NLE track order: the highest-numbered
/// track is topmost).
fn render_montage_frame(
time: Rational,
params: &crate::ticket::VideoTicketParams,
+49 -2
View File
@@ -19,7 +19,8 @@
//! Builds a real node graph (sequence -> video track list -> video tracks
//! -> clip blocks -> footage), evaluates the clip overlapping the request
//! time through the traverser and composites the decoded frames — the same
//! path the engine's viewer uses. Track 0 is the topmost stack element.
//! path the engine's viewer uses. The LAST track is the topmost stack
//! element (NLE stacking: the highest-numbered track wins).
use std::sync::{Arc, Mutex};
@@ -43,7 +44,8 @@ fn clip_path(tag: &str) -> std::path::PathBuf {
}
/// One sequence + one video track list with one track per clip
/// `(filename, [in, out))`. Track 0 (the first entry) composites on top.
/// `(filename, [in, out))`. The LAST entry's track composites on top
/// (NLE stacking: the highest-numbered track is topmost).
fn build_project(clips: &[(&str, Rational, Rational)]) -> (Arc<Mutex<Project>>, NodeId) {
let project = Project::new();
let seq;
@@ -191,6 +193,51 @@ fn graph_sequence_renders_two_tracks() {
let _ = std::fs::remove_file(&path_b);
}
/// NLE stacking regression: two OPAQUE solid-color clips covering the
/// same time on two video tracks — the clip on the LAST track (V2, blue)
/// composites on top of the clip on the first track (V1, red), matching
/// the timeline UI (the highest-numbered track displays on top).
#[test]
fn graph_sequence_stacks_highest_track_on_top() {
let red = clip_path("stack_red");
let blue = clip_path("stack_blue");
oak_codec::testmedia::write_test_clip_solid(&red, 64, 64, 10, 10, [0.9, 0.1, 0.1, 1.0])
.expect("red clip generation");
oak_codec::testmedia::write_test_clip_solid(&blue, 64, 64, 10, 10, [0.1, 0.1, 0.9, 1.0])
.expect("blue clip generation");
// V1 = red (bottom), V2 = blue (top).
let (project, seq) = build_project(&[
(&red.to_string_lossy(), Rational::new(0, 1), Rational::new(1, 1)),
(&blue.to_string_lossy(), Rational::new(0, 1), Rational::new(1, 1)),
]);
let tex = oak_render::eval::render_graph_frame(&project, seq, Rational::new(0, 1), (64, 64), PixelFormat::F32)
.expect("stacked render");
let data = frame_data(&tex);
assert!(
channel(data, 8, 8, 2) > 0.5 && channel(data, 8, 8, 0) < 0.4,
"V2's blue covers V1's red (r={}, b={})",
channel(data, 8, 8, 0),
channel(data, 8, 8, 2)
);
// Distinguishability guard: solo, the V1 clip really is red (the two
// tracks carry different content).
let (solo, solo_seq) = build_project(&[(&red.to_string_lossy(), Rational::new(0, 1), Rational::new(1, 1))]);
let solo_tex = oak_render::eval::render_graph_frame(&solo, solo_seq, Rational::new(0, 1), (64, 64), PixelFormat::F32)
.expect("solo V1 render");
let solo_data = frame_data(&solo_tex);
assert!(
channel(solo_data, 8, 8, 0) > 0.5 && channel(solo_data, 8, 8, 2) < 0.4,
"solo V1 is red (r={}, b={})",
channel(solo_data, 8, 8, 0),
channel(solo_data, 8, 8, 2)
);
let _ = std::fs::remove_file(&red);
let _ = std::fs::remove_file(&blue);
}
/// The driver rejects bad arguments explainably: non-F32 format, a
/// non-positive size, and a missing viewer.
#[test]
+3 -2
View File
@@ -330,7 +330,8 @@ impl RenderTask {
}
/// Flatten the video tracks of `sequence` into an ordered montage
/// (bottom-most track first so the topmost track composites last;
/// (bottom-most track first so the topmost track — the highest-numbered
/// one, the list's last — composites last;
/// `// CPP-PARITY: M12 P0 montage contract`).
fn video_montage(project: &ProjectRef, sequence: oak_node::id::NodeId, time: Rational) -> Vec<MontageClip> {
let guard = project
@@ -361,7 +362,7 @@ impl RenderTask {
if list.kind != TrackType::Video {
continue;
}
for track_id in list.tracks.iter().rev() {
for track_id in list.tracks.iter() {
let Some(te) = guard.graph.get(*track_id) else {
continue;
};