optimize: video and audio render
This commit is contained in:
+176
-25
@@ -105,6 +105,7 @@ pub(crate) mod menu_ids {
|
||||
pub const FOCUS_TIMELINE: usize = ActionId::FocusTimeline.menu_id();
|
||||
pub const FOCUS_EFFECT_LIBRARY: usize = ActionId::FocusEffectLibrary.menu_id();
|
||||
pub const FOCUS_MULTICAM: usize = ActionId::FocusMulticam.menu_id();
|
||||
pub const RESET_DEFAULT_LAYOUT: usize = ActionId::ResetDefaultLayout.menu_id();
|
||||
}
|
||||
|
||||
/// Modal-dialog control ids (see [`ModalEvent::control`]).
|
||||
@@ -636,32 +637,10 @@ impl<E: AppEngine> OakApp<E> {
|
||||
);
|
||||
});
|
||||
|
||||
// Tune the default split ratios: viewers 60% / timeline 40%, project
|
||||
// bin 17% of the row. The timeline share leaves room for all four
|
||||
// tracks (V2/V1 video + A1/A2 audio) plus the ruler and toolbar at
|
||||
// 1600×900; the viewers keep the remaining ~60%. The program viewer
|
||||
// (with its audio level strip) is the active tab of its group, so the
|
||||
// shell opens on the design's visible 素材查看器 | 序列查看器 row rather
|
||||
// than on the node editor.
|
||||
// Tune the default split ratios and active tabs — the shared tuning
|
||||
// block (`tune_default_layout`) that 窗口 → 重置布局 also applies.
|
||||
let mut layout: DockLayout = dock.read(cx).layout().clone();
|
||||
layout.resize_split(&NodePath(vec![]), 0.60);
|
||||
layout.resize_split(&NodePath(vec![0]), 0.17);
|
||||
// The program viewer's transport row (six transport buttons, the
|
||||
// timecode, the 安全框/缩放 toggles) plus its 26px meter strip needs
|
||||
// ~430px at 1600×900 — more than an equal share of the row gives it,
|
||||
// and the design makes the program monitor the prominent viewer. Tilt
|
||||
// the source/program and program/inspector boundaries accordingly so
|
||||
// the transport's trailing toggles are not clipped.
|
||||
layout.resize_split_child(&NodePath(vec![0]), 1, 0.52);
|
||||
layout.resize_split_child(&NodePath(vec![0]), 2, 0.62);
|
||||
if let Some(path) = layout.find_panel(PROGRAM_VIEWER) {
|
||||
layout.set_tabs_active(&path, PROGRAM_VIEWER);
|
||||
}
|
||||
// The project bin is the active tab of its group (the effect library
|
||||
// sits behind it).
|
||||
if let Some(path) = layout.find_panel(PROJECT) {
|
||||
layout.set_tabs_active(&path, PROJECT);
|
||||
}
|
||||
tune_default_layout(&mut layout);
|
||||
dock.update(cx, |dock, cx| dock.set_layout(layout, cx));
|
||||
|
||||
// --- status bar ----------------------------------------------------
|
||||
@@ -1104,6 +1083,7 @@ impl<E: AppEngine> OakApp<E> {
|
||||
A::FocusTimeline => self.toggle_panel(TIMELINE, cx),
|
||||
A::FocusEffectLibrary => self.toggle_panel(EFFECT_LIBRARY, cx),
|
||||
A::FocusMulticam => self.toggle_panel(MULTICAM, cx),
|
||||
A::ResetDefaultLayout => self.reset_default_layout(cx),
|
||||
// --- Tools -----------------------------------------------------
|
||||
A::Snapping => {
|
||||
let enabled = !self.timeline.read(cx).state.snap_enabled;
|
||||
@@ -1431,6 +1411,53 @@ impl<E: AppEngine> OakApp<E> {
|
||||
}
|
||||
}
|
||||
|
||||
/// 窗口 → 重置布局: rebuilds the dock from scratch with every panel at its
|
||||
/// design position and the default ratios and active tabs restored.
|
||||
/// Docked panels are removed through the dock's remove flow; floating
|
||||
/// (tear-off) panels have their window closed for good.
|
||||
fn reset_default_layout(&mut self, cx: &mut Context<Self>) {
|
||||
let dock = self.dock.clone();
|
||||
let ids: Vec<PanelId> = WINDOW_PANELS
|
||||
.iter()
|
||||
.map(|(id, _)| *id)
|
||||
.filter(|id| dock.read(cx).is_panel_visible(*id))
|
||||
.collect();
|
||||
dock.update(cx, |dock, cx| {
|
||||
for id in ids {
|
||||
if dock.is_floating(id) {
|
||||
dock.close_floating(id, cx);
|
||||
} else {
|
||||
let _ = dock.remove_panel(id, cx);
|
||||
}
|
||||
}
|
||||
});
|
||||
self.apply_default_layout(cx);
|
||||
self.rebuild_menu_bar(cx);
|
||||
cx.notify();
|
||||
}
|
||||
|
||||
/// (Re-)seeds every panel at its design position, mirroring the seeding in
|
||||
/// [`OakApp::new`]: panels are added in the 窗口 menu's order
|
||||
/// ([`WINDOW_PANELS`]) at their [`default_dock_target`] placement, then
|
||||
/// the default split ratios and active tabs are applied.
|
||||
fn apply_default_layout(&mut self, cx: &mut Context<Self>) {
|
||||
let mut panels = Vec::new();
|
||||
for (id, _) in WINDOW_PANELS {
|
||||
if let Some(handle) = self.panel_handle(id, cx) {
|
||||
panels.push((handle, default_dock_target(id)));
|
||||
}
|
||||
}
|
||||
let dock = self.dock.clone();
|
||||
dock.update(cx, |dock, cx| {
|
||||
for (handle, target) in panels {
|
||||
let _ = dock.add_panel(handle, target, cx);
|
||||
}
|
||||
let mut layout = dock.layout().clone();
|
||||
tune_default_layout(&mut layout);
|
||||
dock.set_layout(layout, cx);
|
||||
});
|
||||
}
|
||||
|
||||
/// Switches the UI language live: updates the [`i18n`] global, rebuilds
|
||||
/// the menu bar (so the menu labels and the language checkmark move
|
||||
/// immediately), and repaints the whole shell.
|
||||
@@ -2602,6 +2629,29 @@ fn panel_bit(id: PanelId) -> u16 {
|
||||
1u16 << (id.raw() as u16)
|
||||
}
|
||||
|
||||
/// Applies the design's default split ratios and active tabs to `layout`,
|
||||
/// shared by [`OakApp::new`] and the 窗口 → 重置布局 action.
|
||||
///
|
||||
/// Viewers 60% / timeline 40%, project bin 17% of the row — the timeline
|
||||
/// share leaves room for all four tracks (V2/V1 video + A1/A2 audio) plus
|
||||
/// the ruler and toolbar at 1600×900, the viewers keep the remaining ~60%.
|
||||
/// The program viewer's transport row plus its 26px meter strip needs ~430px
|
||||
/// at 1600×900, so the source/program and program/inspector boundaries are
|
||||
/// tilted (0.52 / 0.62) to keep the transport's trailing toggles unclipped.
|
||||
/// The program viewer and project bin end up active in their groups.
|
||||
fn tune_default_layout(layout: &mut DockLayout) {
|
||||
layout.resize_split(&NodePath(vec![]), 0.60);
|
||||
layout.resize_split(&NodePath(vec![0]), 0.17);
|
||||
layout.resize_split_child(&NodePath(vec![0]), 1, 0.52);
|
||||
layout.resize_split_child(&NodePath(vec![0]), 2, 0.62);
|
||||
if let Some(path) = layout.find_panel(PROGRAM_VIEWER) {
|
||||
layout.set_tabs_active(&path, PROGRAM_VIEWER);
|
||||
}
|
||||
if let Some(path) = layout.find_panel(PROJECT) {
|
||||
layout.set_tabs_active(&path, PROJECT);
|
||||
}
|
||||
}
|
||||
|
||||
/// The design's default dock position for each panel, mirroring the seeding
|
||||
/// in [`OakApp::new`] — 项目 | 素材查看器 | 序列查看器+节点编辑器 |
|
||||
/// 检查器+历史记录 row, timeline full width at the bottom. Used to re-open a
|
||||
@@ -3321,6 +3371,107 @@ mod tests {
|
||||
assert_eq!(cx.read(|app| inspector_checked(app)), Some(true));
|
||||
}
|
||||
|
||||
/// 窗口 → 重置布局 tears the current dock arrangement down and rebuilds
|
||||
/// every panel at its design position with the default ratios: panels
|
||||
/// dismissed via the 窗口 menu come back, floating (tear-off) panels'
|
||||
/// windows are closed for good, and tilted splits are restored.
|
||||
#[gpui::test]
|
||||
async fn reset_layout_restores_the_default_workspace(cx: &mut TestAppContext) {
|
||||
let _guard = crate::actions::shortcuts_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
let _guard = crate::i18n::lang_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
let (_window, root) = mock_shell(cx);
|
||||
|
||||
// Distort the workspace on every axis: dismiss two panels, tear the
|
||||
// timeline off into its own window, and tilt the root split to 0.5.
|
||||
cx.update(|app| root.update(app, |app, cx| app.on_menu(menu_ids::FOCUS_INSPECTOR, cx)));
|
||||
cx.update(|app| root.update(app, |app, cx| app.on_menu(menu_ids::FOCUS_HISTORY, cx)));
|
||||
cx.run_until_parked();
|
||||
cx.update(|app| {
|
||||
root.update(app, |app, cx| {
|
||||
let dock = app.dock.clone();
|
||||
dock.update(cx, |dock, cx| {
|
||||
assert!(dock.float_panel(TIMELINE, cx));
|
||||
});
|
||||
});
|
||||
});
|
||||
cx.run_until_parked();
|
||||
cx.update(|app| {
|
||||
root.update(app, |app, cx| {
|
||||
let mut layout = app.dock.read(cx).layout().clone();
|
||||
layout.resize_split(&NodePath(vec![]), 0.5);
|
||||
let dock = app.dock.clone();
|
||||
dock.update(cx, |dock, cx| dock.set_layout(layout, cx));
|
||||
});
|
||||
});
|
||||
cx.run_until_parked();
|
||||
|
||||
// The distorted state is in place.
|
||||
assert_eq!(
|
||||
cx.read(|app| {
|
||||
let dock = root.read(app).dock.read(app);
|
||||
(
|
||||
dock.is_docked(INSPECTOR),
|
||||
dock.is_docked(HISTORY),
|
||||
dock.is_floating(TIMELINE),
|
||||
dock
|
||||
.layout()
|
||||
.split_ratios(&NodePath(vec![]))
|
||||
.map(|r| r[0]),
|
||||
)
|
||||
}),
|
||||
(false, false, true, Some(0.5))
|
||||
);
|
||||
|
||||
// 窗口 → 重置布局 restores every panel to its design position.
|
||||
cx.update(|app| {
|
||||
root.update(app, |app, cx| app.on_menu(menu_ids::RESET_DEFAULT_LAYOUT, cx));
|
||||
});
|
||||
cx.run_until_parked();
|
||||
|
||||
assert!(
|
||||
cx.read(|app| {
|
||||
let dock = root.read(app).dock.read(app);
|
||||
let all_docked =
|
||||
WINDOW_PANELS.iter().all(|(panel, _)| dock.is_docked(*panel));
|
||||
let none_floating =
|
||||
WINDOW_PANELS.iter().all(|(panel, _)| !dock.is_floating(*panel));
|
||||
let root_ratio = dock
|
||||
.layout()
|
||||
.split_ratios(&NodePath(vec![]))
|
||||
.map(|r| r[0]);
|
||||
all_docked && none_floating && (root_ratio.unwrap_or(0.0) - 0.60).abs() < 1e-4
|
||||
}),
|
||||
"every panel is docked at the default 0.60 split after the reset"
|
||||
);
|
||||
|
||||
// The 窗口 menu's checkmarks follow the restored visible-panel set.
|
||||
let mask = cx.read(|app| open_panels_mask(root.read(app).dock.read(app)));
|
||||
let mut state = MenuState::new(true);
|
||||
state.open_panels = mask;
|
||||
let window_menu = make_menus(state)
|
||||
.into_iter()
|
||||
.find(|e| e.title == crate::i18n::tr("menu.window"))
|
||||
.expect("Window menu exists")
|
||||
.menu;
|
||||
for (panel, action) in WINDOW_PANELS {
|
||||
let item = window_menu
|
||||
.items
|
||||
.iter()
|
||||
.find(|item| item.id == action.menu_id())
|
||||
.unwrap_or_else(|| panic!("Window menu is missing panel {}", panel.raw()));
|
||||
assert_eq!(
|
||||
item.checked,
|
||||
Some(true),
|
||||
"{} is re-checked after the reset",
|
||||
panel.raw()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Closing a panel via its tab ✕ — the dock's own close flow — also
|
||||
/// refreshes the 窗口 menu: the dock's structural event rebuilds the menu
|
||||
/// bar, so the dismissed panel loses its checkmark without waiting for any
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
// Oak Video Editor - Non-Linear Video Editor
|
||||
// Copyright (C) 2026 Oak Team
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Dedicated audio-render thread (M16 S3): audio chunk renders never run
|
||||
//! on the UI tick.
|
||||
//!
|
||||
//! Playback prefetch submits one audio chunk per frame tick. Those chunks
|
||||
//! used to be rendered synchronously by the ticket arena's inline audio
|
||||
//! dispatcher — i.e. inside the UI tick itself: a slow decode (opening a
|
||||
//! file, hardware probe) froze the UI and delayed the next chunk, starving
|
||||
//! the audio device (underruns → pops/clicks). This module owns a single
|
||||
//! background thread that runs the same [`oak_render::eval::render_audio_samples`]
|
||||
//! path the ticket arena uses; the UI tick only queues the job and returns
|
||||
//! immediately.
|
||||
//!
|
||||
//! The thread starts lazily on the first submit and lives for the process.
|
||||
//! A single thread is deliberate: the montage decoders are not safe to run
|
||||
//! concurrently, and one chunk per frame tick stays far below real-time
|
||||
//! decode throughput. Render errors degrade to silence of the exact length
|
||||
//! the success path would produce, so the playback buffer never desyncs.
|
||||
|
||||
use std::sync::{mpsc, OnceLock};
|
||||
|
||||
use oak_render::ticket::{AudioTicketParams, TicketPayload};
|
||||
|
||||
use super::renderops::RenderedAudio;
|
||||
|
||||
/// One queued audio chunk: render `params`, then send `(start_ts, data)`
|
||||
/// on `done`.
|
||||
struct AudioJob {
|
||||
params: AudioTicketParams,
|
||||
start_ts: i64,
|
||||
done: mpsc::Sender<(i64, RenderedAudio)>,
|
||||
}
|
||||
|
||||
/// The audio-render thread's job queue (set on first use; the sender is
|
||||
/// kept alive by this `OnceLock` for the whole process).
|
||||
static TX: OnceLock<mpsc::Sender<AudioJob>> = OnceLock::new();
|
||||
|
||||
/// Queue `params` for rendering on the audio thread; the rendered samples
|
||||
/// (or aligned silence on failure) are sent on `done` as
|
||||
/// `(start_ts, data)`. Never blocks — a UI tick must not wait on a worker.
|
||||
pub fn submit(
|
||||
params: AudioTicketParams,
|
||||
start_ts: i64,
|
||||
done: mpsc::Sender<(i64, RenderedAudio)>,
|
||||
) -> Result<(), String> {
|
||||
let tx = ensure_thread();
|
||||
tx.send(AudioJob { params, start_ts, done })
|
||||
.map_err(|_| "the audio render thread has exited".to_string())
|
||||
}
|
||||
|
||||
/// The queue sender, starting the render thread on first use.
|
||||
fn ensure_thread() -> mpsc::Sender<AudioJob> {
|
||||
if let Some(tx) = TX.get() {
|
||||
return tx.clone();
|
||||
}
|
||||
let (tx, rx) = mpsc::channel::<AudioJob>();
|
||||
if TX.set(tx.clone()).is_ok() {
|
||||
// We won the race and own the receiver; start the thread. The
|
||||
// loser's `rx` is dropped right here.
|
||||
std::thread::Builder::new()
|
||||
.name("oak-audio-render".to_string())
|
||||
.spawn(move || render_loop(rx))
|
||||
.expect("spawn the audio render thread");
|
||||
}
|
||||
TX.get().expect("audio thread sender was set above").clone()
|
||||
}
|
||||
|
||||
/// The render loop: one job at a time; failures degrade to silence.
|
||||
fn render_loop(rx: mpsc::Receiver<AudioJob>) {
|
||||
while let Ok(job) = rx.recv() {
|
||||
let data = match oak_render::eval::render_audio_samples(&job.params) {
|
||||
Ok(TicketPayload::Audio(samples)) => RenderedAudio {
|
||||
data: samples.samples,
|
||||
sample_rate: samples.sample_rate,
|
||||
channel_count: samples.channel_count,
|
||||
},
|
||||
_ => silence_for(&job.params),
|
||||
};
|
||||
let _ = job.done.send((job.start_ts, data));
|
||||
}
|
||||
}
|
||||
|
||||
/// Aligned silence for `params`: the same anchored-grid length
|
||||
/// (`round(out·rate) − round(in·rate)`) the successful path produces, so
|
||||
/// a failed chunk keeps the playback buffer aligned.
|
||||
fn silence_for(params: &AudioTicketParams) -> RenderedAudio {
|
||||
let rate = params.sample_rate.max(1) as f64;
|
||||
let channels = params.channel_layout.count_ones().max(1) as i32;
|
||||
let frames = ((params.range.out().to_f64() * rate).round()
|
||||
- (params.range.in_().to_f64() * rate).round())
|
||||
.max(0.0) as usize;
|
||||
RenderedAudio {
|
||||
data: vec![0.0; frames * channels as usize],
|
||||
sample_rate: params.sample_rate.max(1),
|
||||
channel_count: channels,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use oak_core::{Rational, TimeRange};
|
||||
use oak_render::ticket::MontageClip;
|
||||
|
||||
/// Stereo 48 kHz empty-montage params over `[in_sec, out_sec)`.
|
||||
fn params(in_sec: i64, out_sec: i64) -> AudioTicketParams {
|
||||
AudioTicketParams {
|
||||
viewer: 7,
|
||||
range: TimeRange::new(Rational::new(in_sec, 1), Rational::new(out_sec, 1)),
|
||||
sample_rate: 48000,
|
||||
channel_layout: 0x3,
|
||||
montage: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Submitting returns immediately (no blocking render on the caller)
|
||||
/// and the result arrives asynchronously on the completion channel.
|
||||
#[test]
|
||||
fn submit_returns_immediately_and_delivers_async() {
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
let start = std::time::Instant::now();
|
||||
submit(params(0, 1), 12, done_tx).expect("submit");
|
||||
assert!(
|
||||
start.elapsed() < std::time::Duration::from_millis(50),
|
||||
"submit must not block"
|
||||
);
|
||||
|
||||
let (start_ts, audio) = done_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("rendered audio delivered");
|
||||
assert_eq!(start_ts, 12);
|
||||
assert_eq!(audio.sample_rate, 48000);
|
||||
assert_eq!(audio.channel_count, 2);
|
||||
// Empty montage → pure silence, anchored-grid length (1 s).
|
||||
assert_eq!(audio.data.len(), 48000 * 2);
|
||||
assert!(audio.data.iter().all(|&s| s == 0.0));
|
||||
}
|
||||
|
||||
/// A clip that cannot be opened (missing file) degrades to silence of
|
||||
/// the expected anchored-grid length — the playback buffer stays
|
||||
/// aligned.
|
||||
#[test]
|
||||
fn error_produces_silence_of_expected_length() {
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
let mut p = params(0, 1);
|
||||
p.montage.push(MontageClip {
|
||||
filename: "/nonexistent/oak-audio-test.wav".to_string(),
|
||||
stream_index: 0,
|
||||
in_time: Rational::new(0, 1),
|
||||
out_time: Rational::new(1, 1),
|
||||
media_in: Rational::new(0, 1),
|
||||
gain: 1.0,
|
||||
effects: Vec::new(),
|
||||
});
|
||||
submit(p, 0, done_tx).expect("submit");
|
||||
|
||||
let (_, audio) = done_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
.expect("silence delivered");
|
||||
assert_eq!(audio.data.len(), 48000 * 2);
|
||||
assert!(audio.data.iter().all(|&s| s == 0.0));
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@
|
||||
//! * [`timecode`] — timecode / duration / fps / resolution formatting (pure,
|
||||
//! unit tested).
|
||||
|
||||
pub mod audio_thread;
|
||||
pub mod component;
|
||||
pub mod displaycolor;
|
||||
pub mod effectchain;
|
||||
|
||||
+436
-108
@@ -241,7 +241,8 @@ enum FullResTarget {
|
||||
// workers, so by the time the playhead reaches a frame its pixels are
|
||||
// already sitting in a shm slot. `cpu_frame` hits this slot cache first
|
||||
// (zero copy — build the display image from the slot bytes, then release
|
||||
// the slot); the synchronous render path is the miss fallback. Frames
|
||||
// the slot); a miss falls back to the last displayed proxy frame (the
|
||||
// synchronous render is the cold-cache fallback only). Frames
|
||||
// that fall out of the window (or whose params were invalidated) release
|
||||
// their slots back to the workers.
|
||||
|
||||
@@ -268,24 +269,39 @@ struct PreviewWindow {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Playback audio prefetch (M15 S3; M16 S2: rendered inline)
|
||||
// Playback audio prefetch (M15 S3; M16 S3: rendered on the audio thread)
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// Real-time audio is pulled by the UI tick. Through the worker pool it had
|
||||
// one IPC round trip and could wait behind a busy render worker, so the
|
||||
// chunks are rendered AHEAD (tickets complete on the dispatcher's poll) and
|
||||
// buffered here. M16 S2 renders audio on the UI tick itself (the manager's
|
||||
// audio dispatch is inline — see oak_render::manager), so the prefetch
|
||||
// window now serves as the scheduling cushion instead of the delivery
|
||||
// latency: the pull never blocks on a worker. `AUDIO_PREFETCH_CHUNKS` ahead
|
||||
// ≈ 64 ms of audio at 60 fps — enough to cover the delivery latency while
|
||||
// keeping at most a handful of audio chunks in flight.
|
||||
// buffered here. M16 S3 renders each chunk on the dedicated audio-render
|
||||
// thread (see oakui::audio_thread) — never on the UI tick — and delivers it
|
||||
// into the channel asynchronously, so the prefetch window serves as the
|
||||
// scheduling cushion: the pull never blocks on a worker. `AUDIO_PREFETCH_CHUNKS`
|
||||
// ahead ≈ 64 ms of audio at 60 fps — enough to cover the delivery latency
|
||||
// while keeping at most a handful of audio chunks in flight.
|
||||
|
||||
/// How many audio chunks are kept rendered ahead of the playhead (M15
|
||||
/// S3; M16 S2: rendered inline on the UI tick, so the window is the
|
||||
/// S3; M16 S3: rendered on the audio thread, so the window is the
|
||||
/// scheduling cushion). Each chunk is one sequence frame of audio;
|
||||
/// 4 frames ahead ≈ 66 ms at 60 fps and ≈ 160 ms at 25 fps.
|
||||
const AUDIO_PREFETCH_CHUNKS: i64 = 4;
|
||||
/// 6 frames ahead ≈ 100 ms at 60 fps and ≈ 240 ms at 25 fps — the
|
||||
/// output-device queue holds the whole window (chunks are pushed as soon
|
||||
/// as rendered), so this is also how long a UI stall can last before the
|
||||
/// device underruns.
|
||||
const AUDIO_PREFETCH_CHUNKS: i64 = 6;
|
||||
|
||||
/// The playhead frame whose audio is playing at audio-clock time `secs`:
|
||||
/// `start_frame + (secs - start_secs)` seconds at `rate`. The wall clock
|
||||
/// anchors play() at the click instant, but the first chunk only reaches
|
||||
/// the device ~50-150 ms later, so the anchor is taken at the first push —
|
||||
/// `start_secs` is the output clock at that moment, NOT zero (on a replay
|
||||
/// the device has already padded underruns onto `output_frames_consumed`).
|
||||
/// The multiply is split into `num as f64 / den as f64` first so large
|
||||
/// frame counts cannot overflow i64.
|
||||
fn audio_target_frame(start_frame: i64, start_secs: f64, secs: f64, rate: FrameRate) -> i64 {
|
||||
start_frame + ((secs - start_secs) * rate.num as f64 / rate.den as f64).round() as i64
|
||||
}
|
||||
|
||||
/// The playback-audio prefetch buffer: rendered chunks ordered by start
|
||||
/// timestamp, plus the submission cursor. UI-thread-only (guarded by the
|
||||
@@ -298,6 +314,12 @@ struct AudioPrefetch {
|
||||
next_submit: i64,
|
||||
/// Chunk length (sequence frames per tick) the buffer is built with.
|
||||
chunk: i64,
|
||||
/// Sequence frame of the chunk most recently served to the output
|
||||
/// device (None until the first pop). A playhead stalled between
|
||||
/// sequence frames (a 60 Hz tick at a 25 fps rate) presents the same
|
||||
/// frame again; [`AudioPrefetch::needs_reset`] distinguishes that from
|
||||
/// a genuine seek so the frame is not re-pushed.
|
||||
last_consumed: Option<i64>,
|
||||
/// Rendered chunks in start-ts order.
|
||||
buffered: VecDeque<(i64, super::renderops::RenderedAudio)>,
|
||||
}
|
||||
@@ -309,14 +331,25 @@ impl AudioPrefetch {
|
||||
front_ts: i64::MIN,
|
||||
next_submit: i64::MIN,
|
||||
chunk: 1,
|
||||
last_consumed: None,
|
||||
buffered: VecDeque::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// True when `frame` lies inside the submitted window
|
||||
/// `[front_ts, next_submit)` — the playhead is being served.
|
||||
fn covers(&self, frame: i64) -> bool {
|
||||
self.front_ts != i64::MIN && self.front_ts <= frame && frame < self.next_submit
|
||||
/// True when the playhead jumped beyond what serving can follow — a
|
||||
/// genuine seek or restart. Serving runs AHEAD of the playhead by up
|
||||
/// to `AUDIO_PREFETCH_CHUNKS` chunks (rendered chunks are pushed to the
|
||||
/// output device immediately so its queue rides out UI stalls), so the
|
||||
/// served cursor normally leads the playhead; only a backward jump past
|
||||
/// that lead (loop wrap, scrub) or a forward jump past the submitted
|
||||
/// window means a real seek. The audio-master correction's small
|
||||
/// re-anchors (a frame or two) stay inside the tolerance.
|
||||
fn needs_reset(&self, frame: i64) -> bool {
|
||||
if self.front_ts == i64::MIN {
|
||||
return true;
|
||||
}
|
||||
let served = self.last_consumed.unwrap_or(self.front_ts - self.chunk);
|
||||
frame < served - (AUDIO_PREFETCH_CHUNKS + 2) || frame >= self.next_submit
|
||||
}
|
||||
|
||||
/// Reset for a (re)start at `frame`: drop every buffered chunk and
|
||||
@@ -325,6 +358,7 @@ impl AudioPrefetch {
|
||||
self.front_ts = frame;
|
||||
self.next_submit = frame;
|
||||
self.chunk = chunk.max(1);
|
||||
self.last_consumed = None;
|
||||
self.buffered.clear();
|
||||
}
|
||||
|
||||
@@ -345,27 +379,26 @@ impl AudioPrefetch {
|
||||
self.buffered.insert(pos, (ts, data));
|
||||
}
|
||||
|
||||
/// Pop the chunk at `frame` (dropping any stale leading chunks).
|
||||
/// Returns `None` when the chunk has not been rendered yet.
|
||||
fn pop_at(&mut self, frame: i64) -> Option<super::renderops::RenderedAudio> {
|
||||
/// Serve every rendered chunk in start-ts order, returning the ones to
|
||||
/// push to the output device. Chunks whose sequence time is already
|
||||
/// behind the playhead arrived too late — they are DROPPED rather than
|
||||
/// played late, so an underrun/production gap costs content but never
|
||||
/// accumulates A/V desync. On-time and future chunks are pushed
|
||||
/// immediately (not one-per-tick): the output device's queue then
|
||||
/// holds up to the whole prefetch window, so a stalled UI tick can no
|
||||
/// longer starve the device into crackling.
|
||||
fn drain_ready(&mut self, playhead: i64) -> Vec<super::renderops::RenderedAudio> {
|
||||
let mut out = Vec::new();
|
||||
while let Some((ts, _)) = self.buffered.front() {
|
||||
if *ts < frame {
|
||||
let ts = self.buffered.pop_front().unwrap().0;
|
||||
self.front_ts = ts + self.chunk;
|
||||
} else {
|
||||
break;
|
||||
let ts = *ts;
|
||||
let (_, data) = self.buffered.pop_front().unwrap();
|
||||
if ts >= playhead {
|
||||
out.push(data);
|
||||
}
|
||||
}
|
||||
let (ts, data) = self.buffered.pop_front()?;
|
||||
if ts == frame {
|
||||
self.front_ts = ts + self.chunk;
|
||||
Some(data)
|
||||
} else {
|
||||
// Gap: the chunk at `frame` is still rendering. Re-insert and
|
||||
// report nothing (the output device zero-fills this tick).
|
||||
self.buffered.push_front((ts, data));
|
||||
None
|
||||
self.last_consumed = Some(ts);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
@@ -720,6 +753,19 @@ impl RealClock {
|
||||
self.started = None;
|
||||
}
|
||||
|
||||
/// Re-anchors a PLAYING clock at `frame`: subsequent wall-clock ticks
|
||||
/// advance from here. A bare `transport.seek` while playing is
|
||||
/// overwritten by the next [`tick`](Self::tick) (it recomputes the
|
||||
/// playhead from the `started` anchor), so external clock followers
|
||||
/// (the audio-master correction) must re-anchor instead.
|
||||
pub fn reanchor_while_playing(&mut self, frame: Frame, length: Frame) {
|
||||
if self.started.is_none() {
|
||||
return;
|
||||
}
|
||||
self.transport.seek(frame, length);
|
||||
self.started = Some((Instant::now(), self.transport.frame()));
|
||||
}
|
||||
|
||||
/// Advances the playhead from the wall clock while playing, looping at
|
||||
/// `length` — or pausing on the last frame when `stop_on_last` is set.
|
||||
/// No-op when stopped. The advance is clamped per tick: a
|
||||
@@ -1117,18 +1163,31 @@ pub struct RealEngine {
|
||||
audio_tx: Mutex<mpsc::Sender<(i64, super::renderops::RenderedAudio)>>,
|
||||
/// The audio prefetch buffer (see [`AudioPrefetch`]).
|
||||
audio_prefetch: Mutex<AudioPrefetch>,
|
||||
/// Audio-mastered playback anchor `(playhead_frame, output_clock_secs)`
|
||||
/// recorded at the first chunk push of a play run (M12 P1a). While set,
|
||||
/// the tick loop re-anchors the program playhead onto the frame the
|
||||
/// output device is actually consuming (`manager.seconds()`), fixing
|
||||
/// the fixed startup offset and letting playback catch up after
|
||||
/// underruns. Cleared on pause / seek / stop; `None` falls back to the
|
||||
/// wall clock.
|
||||
audio_playback: Option<(i64, f64)>,
|
||||
/// Cooldown for the underrun self-heal resync in `pull_audio_tick`
|
||||
/// (without it a resync retriggers every tick while the fresh chunks
|
||||
/// are still rendering).
|
||||
last_audio_resync: Option<std::time::Instant>,
|
||||
}
|
||||
|
||||
impl RealEngine {
|
||||
/// Render one tick's worth of audio at the program playhead and queue
|
||||
/// it for playback (M12 P1; M15 S3: worker-pool prefetch; M16 S2: the
|
||||
/// manager's audio dispatch is inline, so each chunk renders
|
||||
/// synchronously on this tick and lands in the channel before the drain
|
||||
/// below). The chunks are submitted AHEAD of the playhead and buffered
|
||||
/// by [`AudioPrefetch`], so the real-time pull never blocks on a render
|
||||
/// worker or an IPC round trip. Failures degrade to silence; when the
|
||||
/// manager is down the channel stays empty and playback continues
|
||||
/// video-only.
|
||||
/// it for playback (M12 P1; M15 S3: worker-pool prefetch; M16 S3: each
|
||||
/// chunk is queued on the dedicated audio-render thread — see
|
||||
/// [`renderops::submit_audio_chunk`](super::renderops::submit_audio_chunk)
|
||||
/// — and delivered asynchronously into the channel, drained below, so
|
||||
/// the render never runs on the UI tick). The chunks are submitted AHEAD
|
||||
/// of the playhead and buffered by [`AudioPrefetch`], so the real-time
|
||||
/// pull never blocks on a render worker or an IPC round trip. Failures
|
||||
/// degrade to silence; when the audio thread is down the channel stays
|
||||
/// empty and playback continues video-only.
|
||||
fn pull_audio_tick(&mut self, cx: &mut Context<Self>) {
|
||||
let (Some(project), Some(seq)) = (self.project.clone(), self.sequence) else {
|
||||
return;
|
||||
@@ -1151,9 +1210,9 @@ impl RealEngine {
|
||||
.audio_prefetch
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
// Reset on seek / (re)start: the playhead must lie inside the
|
||||
// submitted window [front_ts, next_submit).
|
||||
if !st.covers(frame) {
|
||||
// Reset on seek / restart (see needs_reset): serving runs ahead of
|
||||
// the playhead, so only genuine jumps re-arm the buffer.
|
||||
if st.needs_reset(frame) {
|
||||
st.reset(frame, chunk);
|
||||
}
|
||||
// Submit chunks to cover [next_submit, frame + PREFETCH ahead). In
|
||||
@@ -1173,35 +1232,104 @@ impl RealEngine {
|
||||
}
|
||||
st.next_submit += chunk;
|
||||
}
|
||||
// Drain completed chunks. With the M16 S2 inline audio dispatch the
|
||||
// submit above already ran each chunk synchronously into the channel;
|
||||
// the drain stays for the worker-pool path (Threads/other backends).
|
||||
// Drain completed chunks. With the M16 S3 audio thread the submit
|
||||
// above only queues; the render lands here asynchronously, so the
|
||||
// drain is the sole delivery path.
|
||||
let rx = self.audio_rx.lock().unwrap_or_else(|e| e.into_inner());
|
||||
while let Ok((ts, data)) = rx.try_recv() {
|
||||
st.insert(ts, data);
|
||||
}
|
||||
drop(rx);
|
||||
// Push the chunk at the current playhead to the output device.
|
||||
let Some(buf) = st.pop_at(frame) else {
|
||||
return;
|
||||
};
|
||||
if buf.sample_rate <= 0 || buf.channel_count <= 0 || buf.data.is_empty() {
|
||||
return;
|
||||
// Underrun self-heal: zero-padding advanced the output clock past
|
||||
// the queued content, so everything audible from here on would
|
||||
// play late (the classic "audio drifts behind video"). Resync by
|
||||
// dropping the device queue and re-rendering from the playhead;
|
||||
// the anchor is re-recorded at the next push. Cooldown prevents a
|
||||
// resync storm while the fresh chunks are still rendering.
|
||||
if self.audio_playback.is_some() {
|
||||
let cooldown_ok = self
|
||||
.last_audio_resync
|
||||
.map(|t| t.elapsed() > std::time::Duration::from_secs(1))
|
||||
.unwrap_or(true);
|
||||
let underrun = oak_audio::manager::instance()
|
||||
.map(|m| m.take_output_underrun_frames())
|
||||
.unwrap_or(0);
|
||||
if cooldown_ok && underrun > 480 {
|
||||
if let Some(mut manager) = oak_audio::manager::instance() {
|
||||
let _ = manager.clear_buffered_output();
|
||||
}
|
||||
st.reset(frame, chunk);
|
||||
self.audio_playback = None;
|
||||
self.last_audio_resync = Some(std::time::Instant::now());
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Packed F32; layout: 1ch → mono mask, else stereo.
|
||||
let layout: u64 = if buf.channel_count == 1 { 0x4 } else { 0x3 };
|
||||
let bytes: Vec<u8> = buf.data.iter().flat_map(|v| v.to_ne_bytes()).collect();
|
||||
// Push every rendered chunk to the output device in ts order (the
|
||||
// device queue is the cushion that rides out UI stalls); chunks
|
||||
// whose time already passed are dropped inside drain_ready.
|
||||
for buf in st.drain_ready(frame) {
|
||||
if buf.sample_rate <= 0 || buf.channel_count <= 0 || buf.data.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Packed F32; layout: 1ch → mono mask, else stereo.
|
||||
let layout: u64 = if buf.channel_count == 1 { 0x4 } else { 0x3 };
|
||||
let bytes: Vec<u8> = buf.data.iter().flat_map(|v| v.to_ne_bytes()).collect();
|
||||
if let Some(mut manager) = oak_audio::manager::instance() {
|
||||
let _ = manager.push_to_output(
|
||||
oak_audio::params::AudioParams {
|
||||
sample_rate: buf.sample_rate,
|
||||
channel_layout: layout,
|
||||
format: oak_core::SampleFormat::F32,
|
||||
},
|
||||
&bytes,
|
||||
&mut [],
|
||||
);
|
||||
// M12 P1a: anchor the audio master clock at the first push of a
|
||||
// play run. Recording the CURRENT output clock (not 0) keeps the
|
||||
// anchor honest on a replay — the device has been padding
|
||||
// underruns onto `output_frames_consumed` since reset, so the
|
||||
// correction starts from the frame actually audible right now
|
||||
// instead of teleporting the playhead back to the push position.
|
||||
drop(manager);
|
||||
if self.audio_playback.is_none() {
|
||||
let mut secs = -1.0;
|
||||
if let Some(manager) = oak_audio::manager::instance() {
|
||||
if manager.seconds(&mut secs).is_ok() && secs >= 0.0 {
|
||||
self.audio_playback = Some((frame, secs));
|
||||
// Zeros padded before this first push (stream
|
||||
// startup, or the whole paused interval) are
|
||||
// not a playback underrun — reset the counter
|
||||
// so the self-heal doesn't fire spuriously.
|
||||
let _ = manager.take_output_underrun_frames();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop program playback: clear the play flag and silence the audio
|
||||
/// output (drop the queued samples and re-anchor the output clock so
|
||||
/// a later play() starts clean). Shared by [`AppEngine::pause`] and
|
||||
/// the Stop-on-Last auto-stop — a clock that stopped itself leaves
|
||||
/// `program_playing` set, and without this the last frame's audio
|
||||
/// re-pushes forever.
|
||||
fn stop_program_playback(&mut self, cx: &mut Context<Self>) {
|
||||
self.program_playing = false;
|
||||
self.audio_playback = None;
|
||||
if let Some(mut manager) = oak_audio::manager::instance() {
|
||||
let _ = manager.push_to_output(
|
||||
oak_audio::params::AudioParams {
|
||||
sample_rate: buf.sample_rate,
|
||||
channel_layout: layout,
|
||||
format: oak_core::SampleFormat::F32,
|
||||
},
|
||||
&bytes,
|
||||
&mut [],
|
||||
);
|
||||
let _ = manager.stop_output();
|
||||
let _ = manager.reset_output_clock();
|
||||
}
|
||||
// Reset the prefetch at the resting playhead so a later play()
|
||||
// re-renders it: `last_consumed` would otherwise suppress the
|
||||
// re-push of the very frame still on screen.
|
||||
let frame = self.program_clock.read(cx).transport.frame().0;
|
||||
let mut st = self
|
||||
.audio_prefetch
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
st.reset(frame, 1);
|
||||
}
|
||||
|
||||
/// Builds an engine with no project open.
|
||||
@@ -1255,6 +1383,8 @@ impl RealEngine {
|
||||
audio_rx: Mutex::new(audio_rx),
|
||||
audio_tx: Mutex::new(audio_tx),
|
||||
audio_prefetch: Mutex::new(AudioPrefetch::new()),
|
||||
audio_playback: None,
|
||||
last_audio_resync: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1750,8 +1880,8 @@ impl RealEngine {
|
||||
/// changed; slots that fell behind the playhead are released (credit
|
||||
/// returns to the workers).
|
||||
fn update_preview_window(&mut self, monitor: Monitor, cx: &mut Context<Self>) {
|
||||
// Only during playback: a paused viewer uses the synchronous miss
|
||||
// path (and the resting full-res fill).
|
||||
// Only during playback: a paused viewer displays the last proxy
|
||||
// frame (the resting full-res fill replaces it).
|
||||
let playing = self.clock(monitor).read(cx).transport.is_playing();
|
||||
if !playing {
|
||||
return;
|
||||
@@ -3332,6 +3462,24 @@ impl EngineGateway for RealEngine {
|
||||
}
|
||||
cx.notify();
|
||||
});
|
||||
if monitor == Monitor::Program {
|
||||
// A seek must not keep playing audio queued for the old
|
||||
// playhead: clear the output buffer and reset the prefetch so
|
||||
// the new position renders from scratch (without the reset the
|
||||
// dedup guard could suppress the re-push of the seek target).
|
||||
// The audio master anchor is dropped too — the next play run
|
||||
// re-anchors at its own first push.
|
||||
self.audio_playback = None;
|
||||
if let Some(mut manager) = oak_audio::manager::instance() {
|
||||
let _ = manager.clear_buffered_output();
|
||||
let _ = manager.reset_output_clock();
|
||||
}
|
||||
let mut st = self
|
||||
.audio_prefetch
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
st.reset(frame.0, 1);
|
||||
}
|
||||
self.mirror_program_playhead(cx);
|
||||
self.update_ofx_viewer_time(cx);
|
||||
cx.notify();
|
||||
@@ -3340,6 +3488,10 @@ impl EngineGateway for RealEngine {
|
||||
fn play(&mut self, monitor: Monitor, cx: &mut Context<Self>) {
|
||||
if monitor == Monitor::Program {
|
||||
self.program_playing = true;
|
||||
// A new play run re-anchors the audio master clock at its first
|
||||
// chunk push; a stale anchor (previous run, since cleared by
|
||||
// pause/stop/seek) must not leak into the new one.
|
||||
self.audio_playback = None;
|
||||
}
|
||||
let clock = self.clock(monitor).clone();
|
||||
clock.update(cx, |clock, cx| {
|
||||
@@ -3351,14 +3503,14 @@ impl EngineGateway for RealEngine {
|
||||
}
|
||||
|
||||
fn pause(&mut self, monitor: Monitor, cx: &mut Context<Self>) {
|
||||
if monitor == Monitor::Program {
|
||||
self.program_playing = false;
|
||||
}
|
||||
let clock = self.clock(monitor).clone();
|
||||
clock.update(cx, |clock, cx| {
|
||||
clock.pause();
|
||||
cx.notify();
|
||||
});
|
||||
if monitor == Monitor::Program {
|
||||
self.stop_program_playback(cx);
|
||||
}
|
||||
cx.notify();
|
||||
}
|
||||
|
||||
@@ -3394,6 +3546,56 @@ impl EngineGateway for RealEngine {
|
||||
cx.notify();
|
||||
});
|
||||
}
|
||||
// M12 P1a: while the program plays with an audio anchor, re-anchor
|
||||
// the playhead onto the frame the output device is actually
|
||||
// consuming. The wall clock anchored play() at the click instant,
|
||||
// but the first chunk reaches the device 50-150 ms later (tick
|
||||
// interval + inline mix + cpal open) — a fixed startup offset every
|
||||
// play run — and underruns pad the output with silence while still
|
||||
// advancing `output_frames_consumed`, so playback also drifts.
|
||||
// `manager.seconds()` is the truth: the playhead is set to the
|
||||
// audible frame (loop playback wraps with the sequence). The strict
|
||||
// `secs > start_secs` guard is the fail-safe: if the device never
|
||||
// opens, seconds() stays 0/-1 and playback falls back to the wall
|
||||
// clock untouched.
|
||||
if self.program_playing && self.program_clock.read(cx).is_playing() {
|
||||
if let Some((start_frame, start_secs)) = self.audio_playback {
|
||||
let mut secs = -1.0;
|
||||
if let Some(manager) = oak_audio::manager::instance() {
|
||||
if manager.seconds(&mut secs).is_ok() && secs > start_secs {
|
||||
let rate = self.program_clock.read(cx).frame_rate();
|
||||
let mut target =
|
||||
audio_target_frame(start_frame, start_secs, secs, rate);
|
||||
if length.0 > 0 && target >= length.0 {
|
||||
target %= length.0;
|
||||
}
|
||||
let clock = self.program_clock.clone();
|
||||
clock.update(cx, |clock, cx| {
|
||||
// Re-anchor (not transport.seek): the next
|
||||
// wall-clock tick recomputes from this anchor,
|
||||
// so the playhead keeps following the output
|
||||
// device's consumed-sample clock instead of
|
||||
// snapping back to the click-instant anchor.
|
||||
clock.reanchor_while_playing(Frame(target), length);
|
||||
cx.notify();
|
||||
});
|
||||
} else if secs < 0.0 {
|
||||
// No running output stream (device closed): drop the
|
||||
// anchor and stay on the wall clock.
|
||||
self.audio_playback = None;
|
||||
}
|
||||
} else {
|
||||
self.audio_playback = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
// A Stop-on-Last clock pauses itself at the last frame; mirror
|
||||
// that into the engine's play state like pause() (the transport is
|
||||
// already stopped, so the output must go silent too — otherwise
|
||||
// the last frame's audio re-pushes and the UI re-renders forever).
|
||||
if self.program_playing && !self.program_clock.read(cx).is_playing() {
|
||||
self.stop_program_playback(cx);
|
||||
}
|
||||
self.mirror_program_playhead(cx);
|
||||
self.update_ofx_viewer_time(cx);
|
||||
self.meter_phase = self.meter_phase.wrapping_add(1);
|
||||
@@ -3619,8 +3821,8 @@ impl AppEngine for RealEngine {
|
||||
gpui_widgets::viewer::clear_gpu_frames();
|
||||
}
|
||||
// The full-resolution fill replaces the proxy when its frame matches
|
||||
// the playhead; otherwise the proxy frame is displayed (rendered
|
||||
// synchronously below on a cache miss, filled by the background
|
||||
// the playhead; otherwise the proxy frame is displayed (cached
|
||||
// misses fall back to the last proxy below, filled by the background
|
||||
// worker once the playhead rests).
|
||||
if let Some(image) = cache.entry(monitor).or_default().image_for(frame.0) {
|
||||
return image.clone();
|
||||
@@ -3636,21 +3838,23 @@ impl AppEngine for RealEngine {
|
||||
});
|
||||
return image;
|
||||
}
|
||||
// During playback a cache miss must NOT block the UI thread on a
|
||||
// synchronous render: the wait starves the tick loop that feeds
|
||||
// the pre-render window, and the seek-priority ticket steals
|
||||
// worker capacity from it, so the window never catches up (every
|
||||
// painted frame blocked in `TicketArena::wait` — the choppy
|
||||
// playback regression). Show the last displayed frame while the
|
||||
// window warms up; the workers catch up within a few frames and
|
||||
// the window then serves every playhead frame.
|
||||
if self.clock(monitor).read(cx).transport.is_playing() {
|
||||
if let Some(image) = cache
|
||||
.get(&monitor)
|
||||
.and_then(|e| e.proxy.as_ref().map(|p| p.image.clone()))
|
||||
{
|
||||
return image;
|
||||
}
|
||||
// A cache miss must NOT block the UI thread on a synchronous render
|
||||
// while the cache has ever displayed a frame: during playback the
|
||||
// wait starves the tick loop that feeds the pre-render window, and
|
||||
// the seek-priority ticket steals worker capacity from it, so the
|
||||
// window never catches up (every painted frame blocked in
|
||||
// `TicketArena::wait` — the choppy playback regression); after a
|
||||
// pause or a paused seek the same wait stalls the UI for as long as
|
||||
// the worker takes to render the resting playhead (the "freeze on
|
||||
// pause" report). Show the last displayed frame instead — the
|
||||
// background full-res fill (schedule_full_res on the tick) replaces
|
||||
// it within a few frames. Only a fully cold cache (nothing ever
|
||||
// displayed) falls through to the synchronous render below.
|
||||
if let Some(image) = cache
|
||||
.get(&monitor)
|
||||
.and_then(|e| e.proxy.as_ref().map(|p| p.image.clone()))
|
||||
{
|
||||
return image;
|
||||
}
|
||||
// Both monitors render through the oakrender ticket arena (falling
|
||||
// back to the synthetic pattern when rendering is unavailable): the
|
||||
@@ -4990,6 +5194,15 @@ impl AppEngine for RealEngine {
|
||||
// The app-side transforms read the process global; the workers pick
|
||||
// the new settings up with the next graph upload.
|
||||
oak_render::color::set_pipeline_color_settings(working, spec);
|
||||
// The workers derive their pipeline colors from the uploaded project
|
||||
// snapshot; a settings change alone does not bump the undo-stack
|
||||
// revision (the manager dedups re-uploads on it), so push an explicit
|
||||
// resync — the workers re-deserialize the rewritten snapshot, adopt
|
||||
// the new working space/output spec and drop their cached frames.
|
||||
if let Some(m) = RenderManager::global() {
|
||||
let revision = oak_undo::global::index().unwrap_or(0).max(0) as u64;
|
||||
let _ = m.resync_graph_snapshot(&project, revision);
|
||||
}
|
||||
// The pipeline colorspace changed: every cached frame (CPU image,
|
||||
// GPU texture, preview slot) was rendered under the old space.
|
||||
super::displaycolor::invalidate();
|
||||
@@ -8483,9 +8696,8 @@ mod tests {
|
||||
#[test]
|
||||
fn audio_prefetch_orders_and_serves_chunks() {
|
||||
let mut st = AudioPrefetch::new();
|
||||
assert!(!st.covers(0), "uninitialized prefetch covers nothing");
|
||||
assert!(st.needs_reset(0), "uninitialized prefetch needs a reset");
|
||||
st.reset(0, 10);
|
||||
assert!(!st.covers(0), "nothing submitted yet");
|
||||
st.next_submit = 40; // pretend 4 chunks were submitted (0..40)
|
||||
|
||||
// Out-of-order arrivals (different workers) are reordered.
|
||||
@@ -8497,16 +8709,25 @@ mod tests {
|
||||
vec![0, 20, 30],
|
||||
"sorted by start ts"
|
||||
);
|
||||
// The chunk at the playhead is served.
|
||||
let got = st.pop_at(0).expect("chunk 0 buffered");
|
||||
assert_eq!(got.sample_rate, 48000);
|
||||
assert_eq!(st.buffered.len(), 2);
|
||||
// A not-yet-rendered chunk reports nothing.
|
||||
assert!(st.pop_at(10).is_none());
|
||||
// Stale arrivals (a seek raced the render) are dropped.
|
||||
// drain_ready serves everything rendered, in ts order — on-time
|
||||
// and future chunks alike (the device queue is the cushion).
|
||||
let served = st.drain_ready(0);
|
||||
assert_eq!(served.len(), 3);
|
||||
assert_eq!(served[0].sample_rate, 48000);
|
||||
assert_eq!(st.buffered.len(), 0);
|
||||
assert_eq!(st.last_consumed, Some(30));
|
||||
// Stale arrivals (a seek raced the render) are dropped by insert.
|
||||
st.insert(-10, audio_chunk(-10).1);
|
||||
st.insert(100, audio_chunk(100).1);
|
||||
assert_eq!(st.buffered.len(), 2, "stale chunks dropped");
|
||||
assert_eq!(st.buffered.len(), 0, "stale chunks dropped");
|
||||
// A chunk that arrives after its playback time passed is dropped,
|
||||
// not played late (A/V sync beats completeness).
|
||||
st.reset(0, 1);
|
||||
st.next_submit = 10;
|
||||
st.insert(3, audio_chunk(3).1);
|
||||
let served = st.drain_ready(5);
|
||||
assert!(served.is_empty(), "late chunk dropped");
|
||||
assert_eq!(st.last_consumed, Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -8515,20 +8736,14 @@ mod tests {
|
||||
st.reset(0, 10);
|
||||
st.next_submit = 40;
|
||||
st.insert(10, audio_chunk(10).1);
|
||||
// A seek far ahead: the playhead is outside [front_ts, next_submit).
|
||||
assert!(!st.covers(200));
|
||||
let _ = st.drain_ready(10); // served cursor now 20
|
||||
// A seek far ahead (past the submitted window) resets.
|
||||
assert!(st.needs_reset(200));
|
||||
// A backward seek past the serve-ahead lead resets too.
|
||||
assert!(st.needs_reset(0), "playhead jumped back behind the served cursor");
|
||||
st.reset(200, 10);
|
||||
assert_eq!(st.buffered.len(), 0, "old chunks dropped");
|
||||
assert!(st.pop_at(200).is_none());
|
||||
// A backward seek (the playhead behind the buffer front after it
|
||||
// advanced) is a reset too.
|
||||
st.reset(0, 10);
|
||||
st.next_submit = 40;
|
||||
st.insert(10, audio_chunk(10).1);
|
||||
let _ = st.pop_at(10); // front_ts now 20
|
||||
assert!(!st.covers(5), "chunk 5 is behind the front");
|
||||
st.reset(5, 10);
|
||||
assert_eq!(st.buffered.len(), 0);
|
||||
assert!(st.drain_ready(200).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -8541,6 +8756,119 @@ mod tests {
|
||||
assert_eq!(st.buffered.len(), 1, "duplicates dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_prefetch_does_not_repeat_a_stalled_playhead() {
|
||||
let mut st = AudioPrefetch::new();
|
||||
st.reset(0, 1);
|
||||
st.next_submit = 7; // pretend chunks 0..7 were submitted
|
||||
st.insert(0, audio_chunk(0).1);
|
||||
st.insert(1, audio_chunk(1).1);
|
||||
// Serving runs ahead: both rendered chunks are pushed at once.
|
||||
assert_eq!(st.drain_ready(0).len(), 2);
|
||||
assert_eq!(st.last_consumed, Some(1));
|
||||
// The next ticks (60 Hz vs 25 fps) still show frame 0: a stalled
|
||||
// playhead is neither a reset nor a re-push.
|
||||
assert!(!st.needs_reset(0), "a stalled playhead is not a reset");
|
||||
assert!(st.drain_ready(0).is_empty(), "nothing new to serve");
|
||||
// Small audio-master corrections stay inside the tolerance.
|
||||
assert!(!st.needs_reset(-2));
|
||||
// A genuine seek (far ahead, or a loop wrap back to 0 after the
|
||||
// served cursor advanced) still resets.
|
||||
assert!(st.needs_reset(200), "a seek is a reset");
|
||||
st.reset(200, 1);
|
||||
assert_eq!(st.last_consumed, None, "a reset forgets the old playhead");
|
||||
// After serving the new frame the guard re-arms.
|
||||
st.next_submit = 207;
|
||||
st.insert(200, audio_chunk(200).1);
|
||||
assert_eq!(st.drain_ready(200).len(), 1);
|
||||
assert!(!st.needs_reset(200), "a stall at 200 is not a reset either");
|
||||
}
|
||||
|
||||
// ---- M12 P1a audio-mastered playhead ----------------------------------
|
||||
|
||||
/// The audio-clock → frame conversion: an anchor recorded late (the
|
||||
/// first chunk reaches the device ~50-150 ms after play()) must land
|
||||
/// the playhead on the frame actually audible, and drift/underruns must
|
||||
/// let it catch up.
|
||||
#[test]
|
||||
fn audio_target_frame_anchors_and_converts() {
|
||||
let rate = FrameRate::new(25, 1);
|
||||
// Startup offset: the anchor was taken 0.1 s after play(); by
|
||||
// audio-clock 1.1 s exactly 1.0 s has played → 25 frames past the
|
||||
// anchor frame.
|
||||
assert_eq!(audio_target_frame(0, 0.1, 1.1, rate), 25);
|
||||
// Catch-up: the device already consumed audio before the anchor.
|
||||
assert_eq!(audio_target_frame(100, 2.0, 4.0, rate), 150);
|
||||
// Fractional seconds round to the nearest frame.
|
||||
assert_eq!(audio_target_frame(0, 0.0, 0.02, rate), 1, "0.5 s rounds up");
|
||||
assert_eq!(audio_target_frame(0, 0.0, 0.018, rate), 0, "0.45 s rounds down");
|
||||
// NTSC 29.97: num/den stays a rational, not rounded to 30.
|
||||
let ntsc = FrameRate::NTSC_2997;
|
||||
assert_eq!(audio_target_frame(0, 0.0, 1.0, ntsc), 30);
|
||||
assert_eq!(audio_target_frame(0, 0.0, 10.0, ntsc), 300);
|
||||
// Large frame counts must not overflow (num/den split before the
|
||||
// multiply).
|
||||
assert_eq!(audio_target_frame(1 << 40, 0.0, 1.0, rate), (1 << 40) + 25);
|
||||
}
|
||||
|
||||
/// The audio-master anchor must not survive without a live output
|
||||
/// stream: the tick drops it and playback falls back to the wall clock
|
||||
/// (the fail-safe that keeps playback alive when the device is closed
|
||||
/// or headless).
|
||||
#[gpui::test]
|
||||
async fn audio_master_anchor_falls_back_to_wall_clock(cx: &mut gpui::TestAppContext) {
|
||||
let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx)));
|
||||
cx.update(|app| {
|
||||
engine.update(app, |engine, cx| {
|
||||
// A stale anchor (as if the first chunk was pushed).
|
||||
engine.audio_playback = Some((0, 0.0));
|
||||
engine.play(Monitor::Program, cx);
|
||||
engine.tick(cx);
|
||||
assert!(
|
||||
engine.audio_playback.is_none(),
|
||||
"no output stream → anchor dropped, wall clock drives"
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// A paused playhead miss (seek while paused, worker still rendering)
|
||||
/// must serve the last displayed frame instead of blocking the UI on a
|
||||
/// synchronous render — the "freeze on pause" regression. The engine
|
||||
/// has no project, so any render path would fail; returning the cached
|
||||
/// proxy proves the non-blocking fallback.
|
||||
#[gpui::test]
|
||||
async fn paused_playhead_miss_serves_last_displayed_frame(cx: &mut gpui::TestAppContext) {
|
||||
let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx)));
|
||||
let (width, height, samples) = synthetic_frame_samples(Frame(0));
|
||||
let scope = analyze_f32_rgba(width, height, &samples);
|
||||
let image = Arc::new(f32_rgba_to_bgra_image(width, height, &samples));
|
||||
cx.update(|app| {
|
||||
engine.update(app, |engine, cx| {
|
||||
// The cache already displayed a frame (the proxy).
|
||||
engine
|
||||
.cpu_frame_cache
|
||||
.lock()
|
||||
.unwrap()
|
||||
.entry(Monitor::Program)
|
||||
.or_default()
|
||||
.proxy = Some(ProxyEntry {
|
||||
frame: 0,
|
||||
image: image.clone(),
|
||||
scope,
|
||||
});
|
||||
// Seek the (paused) playhead to a frame that is NOT cached:
|
||||
// the display must fall back to the last displayed frame.
|
||||
engine.request_frame(Monitor::Program, Frame(24), cx);
|
||||
});
|
||||
});
|
||||
let got = cx.read(|app| engine.read(app).cpu_frame(Monitor::Program, app));
|
||||
assert!(
|
||||
Arc::ptr_eq(&image, &got),
|
||||
"a paused miss serves the last displayed frame"
|
||||
);
|
||||
}
|
||||
|
||||
// ---- sequence management (engine facade) ------------------------------
|
||||
|
||||
/// The engine's create-folder / create-sequence-with-params commands
|
||||
|
||||
@@ -897,12 +897,13 @@ pub fn render_audio_range(
|
||||
}
|
||||
|
||||
/// Submit one audio-chunk render for the playback prefetch (M15 S3; M16
|
||||
/// S2: rendered inline on the calling tick — the manager's audio dispatch
|
||||
/// is the inline dispatcher, so this is a synchronous mix into
|
||||
/// `TicketPayload::Audio`; the shm-slot branch stays for worker-pool
|
||||
/// backends). The completion sends `(start_ts, samples)` on `tx`. Render
|
||||
/// errors send silence of the expected length so the playback buffer stays
|
||||
/// aligned (the real-time path must never stall the UI thread on a worker).
|
||||
/// S3: the chunk is queued on the dedicated audio-render thread — see
|
||||
/// [`audio_thread`](super::audio_thread) — so the render never runs on the
|
||||
/// UI tick; a slow decode no longer freezes the UI or delays the next
|
||||
/// chunk (the old inline-dispatch path underran the audio device). The
|
||||
/// completion sends `(start_ts, samples)` on `tx`. Render errors send
|
||||
/// silence of the expected length so the playback buffer stays aligned
|
||||
/// (the real-time path must never stall the UI thread on a worker).
|
||||
pub fn submit_audio_chunk(
|
||||
p: &ProjectRef,
|
||||
seq: NodeId,
|
||||
@@ -918,12 +919,7 @@ pub fn submit_audio_chunk(
|
||||
let montage = audio_montage(p, seq, range);
|
||||
let sample_rate = 48000;
|
||||
let channel_layout = 0x3u64;
|
||||
let channels = channel_layout.count_ones().max(1) as i32;
|
||||
let m = RenderManager::global().ok_or_else(|| "render manager is not initialized".to_string())?;
|
||||
// Completion-only (fire-and-forget): use the non-polling submit so
|
||||
// the arena reaps the entry once the completion lands (the polling
|
||||
// variant would pin it forever — nobody calls result() on this path).
|
||||
m.tickets.submit_audio(
|
||||
super::audio_thread::submit(
|
||||
AudioTicketParams {
|
||||
viewer: seq.identity(),
|
||||
range,
|
||||
@@ -931,40 +927,9 @@ pub fn submit_audio_chunk(
|
||||
channel_layout,
|
||||
montage,
|
||||
},
|
||||
Box::new(move |result| {
|
||||
let data = match result {
|
||||
Ok(TicketPayload::Audio(samples)) => RenderedAudio {
|
||||
data: samples.samples,
|
||||
sample_rate: samples.sample_rate,
|
||||
channel_count: samples.channel_count,
|
||||
},
|
||||
Ok(TicketPayload::ShmAudio(audio)) => {
|
||||
let samples = audio.to_audio_samples();
|
||||
if let Some(m) = RenderManager::global() {
|
||||
m.release_audio_frame(&audio);
|
||||
}
|
||||
RenderedAudio {
|
||||
data: samples.samples,
|
||||
sample_rate: samples.sample_rate,
|
||||
channel_count: samples.channel_count,
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Silence of the expected length keeps the output buffer
|
||||
// aligned (an underrun here just plays zeros).
|
||||
let seconds = len_ts as f64 * tb.0 as f64 / tb.1 as f64;
|
||||
let frames = (seconds * sample_rate as f64).round() as usize;
|
||||
RenderedAudio {
|
||||
data: vec![0.0; frames * channels as usize],
|
||||
sample_rate,
|
||||
channel_count: channels,
|
||||
}
|
||||
}
|
||||
};
|
||||
let _ = tx.send((start_ts, data));
|
||||
}),
|
||||
);
|
||||
Ok(())
|
||||
start_ts,
|
||||
tx,
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -193,6 +193,15 @@ impl ManagerInner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read and reset the output underrun counter (frames the callback had
|
||||
/// to zero-fill because the buffer was empty). The playback engine
|
||||
/// polls this per tick and resyncs after an underrun: the zero-fill
|
||||
/// advanced the playback clock past the queued content, so without a
|
||||
/// resync everything afterwards plays late.
|
||||
pub fn take_output_underrun_frames(&self) -> i64 {
|
||||
self.output_buffer.take_underrun_frames()
|
||||
}
|
||||
|
||||
/// Stop the output stream.
|
||||
///
|
||||
/// `// CPP-PARITY: src/audio/src/audiomanager.cpp:229` (`stop_output` aborts
|
||||
|
||||
@@ -131,6 +131,9 @@ impl PortAudioOutput {
|
||||
}
|
||||
}
|
||||
sink_cb.add_output_frames((total / channels_usize) as i64);
|
||||
// Account the zero-filled tail separately so the engine can
|
||||
// resync after an underrun instead of drifting out of sync.
|
||||
sink_cb.add_underrun_frames((total / channels_usize - frames_got) as i64);
|
||||
};
|
||||
let err_callback = |err: cpal::Error| {
|
||||
eprintln!("output stream error: {err}");
|
||||
|
||||
@@ -35,6 +35,12 @@ pub struct PreviewAudioDevice {
|
||||
/// Frames consumed by the output callback (playback clock, includes
|
||||
/// underrun zero-fill).
|
||||
output_frames_consumed: AtomicI64,
|
||||
/// Frames the output callback had to zero-fill (buffer was empty).
|
||||
/// Kept separate from the playback clock so the engine can detect
|
||||
/// underruns and resync — zero-fill advances the playback clock past
|
||||
/// the queued content, so everything queued afterwards would play
|
||||
/// late (the classic audio-behind-video drift).
|
||||
underrun_frames: AtomicI64,
|
||||
}
|
||||
|
||||
struct PreviewAudioDeviceInner {
|
||||
@@ -62,6 +68,7 @@ impl PreviewAudioDevice {
|
||||
notify_callback: None,
|
||||
}),
|
||||
output_frames_consumed: AtomicI64::new(0),
|
||||
underrun_frames: AtomicI64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,6 +183,19 @@ impl PreviewAudioDevice {
|
||||
.fetch_add(frame_count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Account for frames the output callback zero-filled (underrun).
|
||||
pub fn add_underrun_frames(&self, frame_count: i64) {
|
||||
if frame_count > 0 {
|
||||
self.underrun_frames
|
||||
.fetch_add(frame_count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Read and reset the underrun counter (the engine polls it per tick).
|
||||
pub fn take_underrun_frames(&self) -> i64 {
|
||||
self.underrun_frames.swap(0, Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Frames consumed by the output callback.
|
||||
pub fn output_frames_consumed(&self) -> i64 {
|
||||
self.output_frames_consumed.load(Ordering::Relaxed)
|
||||
|
||||
+157
-83
@@ -250,6 +250,15 @@ impl FFmpegDecoder {
|
||||
.and_then(|s| s.hw_device.map(crate::hwdecode::device_type_name))
|
||||
.map(|name| name.to_string())
|
||||
}
|
||||
|
||||
/// Test-only: how many `seek()` calls the open session has performed.
|
||||
/// Contiguous audio chunks must not increase this (they continue the
|
||||
/// decode without re-seeking); a non-contiguous chunk must.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn audio_seek_count(&self) -> u64 {
|
||||
let state = self.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
state.as_ref().map(|s| s.audio_seeks).unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FFmpegDecoder {
|
||||
@@ -481,6 +490,10 @@ struct DecoderState {
|
||||
eof: bool,
|
||||
video: Option<VideoDecodeState>,
|
||||
audio: Option<AudioDecodeState>,
|
||||
/// Test-only counter of `seek()` calls (proves contiguous audio chunks
|
||||
/// skip the seek).
|
||||
#[cfg(test)]
|
||||
audio_seeks: u64,
|
||||
}
|
||||
|
||||
enum DecoderInner {
|
||||
@@ -523,6 +536,14 @@ struct ScalingCache {
|
||||
/// Audio decode state: a resampler keyed by (rate, layout).
|
||||
struct AudioDecodeState {
|
||||
resampler: Option<(u32, u64, AudioResampler)>,
|
||||
/// The first sample of the next contiguous chunk: the sample index
|
||||
/// (at the current `sample_rate`) one past the last decoded chunk.
|
||||
/// Set on every successful `retrieve_audio_to`; cleared on every seek
|
||||
/// (any seek resets the decoder and resampler, breaking continuity).
|
||||
/// When the next chunk's start sample equals this, the decode can
|
||||
/// continue without re-seeking — skipping the per-chunk decoder flush
|
||||
/// and resampler reset that cause boundary artifacts (pops/clicks).
|
||||
contiguous_end_sample: Option<i64>,
|
||||
}
|
||||
|
||||
/// A swresample conversion context.
|
||||
@@ -641,7 +662,10 @@ impl DecoderState {
|
||||
None
|
||||
};
|
||||
let audio = if matches!(medium, MediaType::Audio) {
|
||||
Some(AudioDecodeState { resampler: None })
|
||||
Some(AudioDecodeState {
|
||||
resampler: None,
|
||||
contiguous_end_sample: None,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -661,6 +685,8 @@ impl DecoderState {
|
||||
eof: false,
|
||||
video,
|
||||
audio,
|
||||
#[cfg(test)]
|
||||
audio_seeks: 0,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -784,7 +810,16 @@ impl DecoderState {
|
||||
DecoderInner::Video(d) => d.flush(),
|
||||
DecoderInner::Audio(d) => d.flush(),
|
||||
}
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.audio_seeks += 1;
|
||||
}
|
||||
self.eof = false;
|
||||
// Any seek breaks audio continuity (decoder flush + resampler
|
||||
// reset): the next audio chunk must not skip its own seek.
|
||||
if let Some(a) = &mut self.audio {
|
||||
a.contiguous_end_sample = None;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1137,88 +1172,71 @@ impl DecoderState {
|
||||
|
||||
dest.fill(0.0);
|
||||
|
||||
// Seek to just before the range start.
|
||||
let start_ts =
|
||||
oak_rational(self.stream_time_base).time_to_timestamp(Rational::from_double(start_sec));
|
||||
self.seek(start_ts)?;
|
||||
|
||||
// Take the cached resampler out (or create one) so the decode loop
|
||||
// below can borrow `self` freely; it is put back before returning.
|
||||
let src_layout = channel_layout_from_mask(self.input_channel_layout_mask);
|
||||
let mut resampler = match self.audio.as_mut().expect("audio session").resampler.take() {
|
||||
Some((rate, layout, rs)) if rate == sample_rate as u32 && layout == channel_layout => {
|
||||
rs
|
||||
}
|
||||
_ => AudioResampler::get(
|
||||
self.input_sample_format,
|
||||
src_layout,
|
||||
self.input_sample_rate,
|
||||
Sample::F32(SampleType::Packed),
|
||||
dst_layout,
|
||||
sample_rate as u32,
|
||||
)?,
|
||||
};
|
||||
let stream_time_base = self.stream_time_base;
|
||||
|
||||
let mut next_sample: Option<i64> = None;
|
||||
let dest_frames = (dest.len() / dst_channels) as i64;
|
||||
|
||||
while let Some(frame) = self.next_frame()? {
|
||||
let DecodedFrame::Audio(audio) = frame else {
|
||||
continue;
|
||||
};
|
||||
let converted = resample_to_interleaved_f32(&mut resampler, &audio)?;
|
||||
if converted.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let chunk_samples = (converted.len() / dst_channels) as i64;
|
||||
let frame_start = match audio.pts() {
|
||||
Some(pts) => {
|
||||
let secs = oak_rational(stream_time_base)
|
||||
.timestamp_to_time(pts)
|
||||
.to_f64();
|
||||
(secs * sample_rate as f64).round() as i64
|
||||
}
|
||||
None => next_sample.unwrap_or(start_sample),
|
||||
};
|
||||
let offset = frame_start - start_sample;
|
||||
if offset < dest_frames && offset + chunk_samples > 0 {
|
||||
let copy_start = offset.max(0) as usize;
|
||||
let copy_end = (offset + chunk_samples).min(dest_frames).max(0) as usize;
|
||||
if copy_end > copy_start {
|
||||
let src_off = (copy_start as i64 - offset) as usize * dst_channels;
|
||||
let n = (copy_end - copy_start) * dst_channels;
|
||||
let dst_off = copy_start * dst_channels;
|
||||
if dst_off + n <= dest.len() {
|
||||
dest[dst_off..dst_off + n]
|
||||
.copy_from_slice(&converted[src_off..src_off + n]);
|
||||
}
|
||||
}
|
||||
}
|
||||
next_sample = Some(frame_start + chunk_samples);
|
||||
if offset + chunk_samples >= dest_frames {
|
||||
break;
|
||||
}
|
||||
// Continuity fast path: when this chunk's start sample equals the
|
||||
// sample one past the previous chunk's end, the decoder and
|
||||
// resampler are still positioned exactly there — continue without
|
||||
// seeking. Skipping the per-chunk seek avoids the decoder flush and
|
||||
// resampler reset whose boundary artifacts (silence gaps, phase
|
||||
// resets) were audible as pops/clicks at chunk edges. The caller's
|
||||
// anchored sample grid (`round(out·rate) − round(in·rate)`) makes
|
||||
// the next chunk's start exactly the previous chunk's recorded end,
|
||||
// so exact comparison is correct.
|
||||
let contiguous = self
|
||||
.audio
|
||||
.as_ref()
|
||||
.map(|a| a.contiguous_end_sample == Some(start_sample))
|
||||
.unwrap_or(false);
|
||||
if !contiguous {
|
||||
// Seek to just before the range start.
|
||||
let start_ts =
|
||||
oak_rational(self.stream_time_base).time_to_timestamp(Rational::from_double(start_sec));
|
||||
self.seek(start_ts)?;
|
||||
}
|
||||
|
||||
// Put the resampler back into the cache for the next call.
|
||||
self.audio.as_mut().expect("audio session").resampler =
|
||||
Some((sample_rate as u32, channel_layout, resampler));
|
||||
// The fill+decode+flush body runs in a closure so the success path
|
||||
// can record the chunk's end sample (continuity) and an error path
|
||||
// can clear it (the decoder/resampler position is unknown after a
|
||||
// mid-chunk failure).
|
||||
let read = (|| -> crate::error::Result<RetrieveAudioStatus> {
|
||||
// Take the cached resampler out (or create one) so the decode loop
|
||||
// below can borrow `self` freely; it is put back before returning.
|
||||
let src_layout = channel_layout_from_mask(self.input_channel_layout_mask);
|
||||
let mut resampler = match self.audio.as_mut().expect("audio session").resampler.take() {
|
||||
Some((rate, layout, rs)) if rate == sample_rate as u32 && layout == channel_layout => {
|
||||
rs
|
||||
}
|
||||
_ => AudioResampler::get(
|
||||
self.input_sample_format,
|
||||
src_layout,
|
||||
self.input_sample_rate,
|
||||
Sample::F32(SampleType::Packed),
|
||||
dst_layout,
|
||||
sample_rate as u32,
|
||||
)?,
|
||||
};
|
||||
let stream_time_base = self.stream_time_base;
|
||||
|
||||
// Flush any samples still buffered in the resampler (rate conversion
|
||||
// tail), appending after the last decoded sample.
|
||||
let resampler = self
|
||||
.audio
|
||||
.as_mut()
|
||||
.expect("audio session")
|
||||
.resampler
|
||||
.as_mut()
|
||||
.map(|r| &mut r.2)
|
||||
.unwrap();
|
||||
if let Some(flush) = flush_resampler_interleaved_f32(resampler)? {
|
||||
if !flush.is_empty() {
|
||||
let chunk_samples = (flush.len() / dst_channels) as i64;
|
||||
let frame_start = next_sample.unwrap_or(start_sample);
|
||||
let mut next_sample: Option<i64> = None;
|
||||
let dest_frames = (dest.len() / dst_channels) as i64;
|
||||
|
||||
while let Some(frame) = self.next_frame()? {
|
||||
let DecodedFrame::Audio(audio) = frame else {
|
||||
continue;
|
||||
};
|
||||
let converted = resample_to_interleaved_f32(&mut resampler, &audio)?;
|
||||
if converted.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let chunk_samples = (converted.len() / dst_channels) as i64;
|
||||
let frame_start = match audio.pts() {
|
||||
Some(pts) => {
|
||||
let secs = oak_rational(stream_time_base)
|
||||
.timestamp_to_time(pts)
|
||||
.to_f64();
|
||||
(secs * sample_rate as f64).round() as i64
|
||||
}
|
||||
None => next_sample.unwrap_or(start_sample),
|
||||
};
|
||||
let offset = frame_start - start_sample;
|
||||
if offset < dest_frames && offset + chunk_samples > 0 {
|
||||
let copy_start = offset.max(0) as usize;
|
||||
@@ -1229,14 +1247,70 @@ impl DecoderState {
|
||||
let dst_off = copy_start * dst_channels;
|
||||
if dst_off + n <= dest.len() {
|
||||
dest[dst_off..dst_off + n]
|
||||
.copy_from_slice(&flush[src_off..src_off + n]);
|
||||
.copy_from_slice(&converted[src_off..src_off + n]);
|
||||
}
|
||||
}
|
||||
}
|
||||
next_sample = Some(frame_start + chunk_samples);
|
||||
if offset + chunk_samples >= dest_frames {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Put the resampler back into the cache for the next call.
|
||||
self.audio.as_mut().expect("audio session").resampler =
|
||||
Some((sample_rate as u32, channel_layout, resampler));
|
||||
|
||||
// Flush any samples still buffered in the resampler (rate conversion
|
||||
// tail), appending after the last decoded sample.
|
||||
let resampler = self
|
||||
.audio
|
||||
.as_mut()
|
||||
.expect("audio session")
|
||||
.resampler
|
||||
.as_mut()
|
||||
.map(|r| &mut r.2)
|
||||
.unwrap();
|
||||
if let Some(flush) = flush_resampler_interleaved_f32(resampler)? {
|
||||
if !flush.is_empty() {
|
||||
let chunk_samples = (flush.len() / dst_channels) as i64;
|
||||
let frame_start = next_sample.unwrap_or(start_sample);
|
||||
let offset = frame_start - start_sample;
|
||||
if offset < dest_frames && offset + chunk_samples > 0 {
|
||||
let copy_start = offset.max(0) as usize;
|
||||
let copy_end = (offset + chunk_samples).min(dest_frames).max(0) as usize;
|
||||
if copy_end > copy_start {
|
||||
let src_off = (copy_start as i64 - offset) as usize * dst_channels;
|
||||
let n = (copy_end - copy_start) * dst_channels;
|
||||
let dst_off = copy_start * dst_channels;
|
||||
if dst_off + n <= dest.len() {
|
||||
dest[dst_off..dst_off + n]
|
||||
.copy_from_slice(&flush[src_off..src_off + n]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(RetrieveAudioStatus::Success)
|
||||
// Record the end sample: the next chunk starting exactly here
|
||||
// continues without a seek.
|
||||
self.audio.as_mut().expect("audio session").contiguous_end_sample =
|
||||
Some(start_sample + dest_frames);
|
||||
|
||||
Ok(RetrieveAudioStatus::Success)
|
||||
})();
|
||||
|
||||
match read {
|
||||
Ok(status) => Ok(status),
|
||||
Err(e) => {
|
||||
// A failed chunk leaves the decoder/resampler position
|
||||
// unknown: force a fresh seek on the next chunk.
|
||||
if let Some(a) = &mut self.audio {
|
||||
a.contiguous_end_sample = None;
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Conform the open stream's audio into per-channel PCM files, mirroring
|
||||
|
||||
@@ -57,17 +57,61 @@ pub static HW_TRANSFERS: std::sync::atomic::AtomicU64 = std::sync::atomic::Atomi
|
||||
/// 0 = force software). Default ON by user mandate.
|
||||
pub const CONFIG_KEY_HARDWARE_DECODING: &str = "HardwareDecoding";
|
||||
|
||||
/// Process-wide negative cache of failed device types (a `static`, not a
|
||||
/// `const` — a const array is inlined per use site, so writes from
|
||||
/// `mark_device_unavailable` would never be visible to `device_unavailable`).
|
||||
/// Creating a hardware device context (`av_hwdevice_ctx_create`) is
|
||||
/// expensive when the driver is missing and emits FFmpeg's
|
||||
/// `[VAAPI @ ...] Failed to initialise VAAPI connection` spam every time —
|
||||
/// on a box without libva that is once per decoder open, per process, with
|
||||
/// the full log line. The cache marks a device type unavailable after its
|
||||
/// first failed creation, so every later open skips it (and its log noise)
|
||||
/// entirely. The slot count is generous: `AVHWDeviceType` values are small
|
||||
/// non-negative enumerants (< 32), 64 covers them all.
|
||||
static UNAVAILABLE_DEVICES: [std::sync::atomic::AtomicBool; 64] =
|
||||
[const { std::sync::atomic::AtomicBool::new(false) }; 64];
|
||||
|
||||
/// Test-only counter of `open_hw_accel` device-context creation attempts
|
||||
/// (incremented at the top of the function, before any FFmpeg call). Lets
|
||||
/// tests prove the negative cache short-circuits before FFmpeg is
|
||||
/// involved.
|
||||
#[cfg(test)]
|
||||
static CREATE_ATTEMPTS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
|
||||
/// Whether `device_type` is known unavailable — a device-context
|
||||
/// creation failed once earlier in this process.
|
||||
pub fn device_unavailable(device_type: sys::AVHWDeviceType) -> bool {
|
||||
UNAVAILABLE_DEVICES
|
||||
.get(device_type as usize)
|
||||
.map(|f| f.load(std::sync::atomic::Ordering::Relaxed))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Record that `device_type`'s device context failed to create (sticky
|
||||
/// for the process; the caller falls back to the next candidate).
|
||||
pub fn mark_device_unavailable(device_type: sys::AVHWDeviceType) {
|
||||
if let Some(f) = UNAVAILABLE_DEVICES.get(device_type as usize) {
|
||||
f.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether hardware decoding is preferred (the config switch). Default
|
||||
/// ON by user mandate; only an explicit `"false"` turns it off (the
|
||||
/// string accessor, same convention as the app's config helpers — the
|
||||
/// store's typed `get_bool` only parses pre-typed Bool entries).
|
||||
/// store's typed `get_bool` only parses pre-typed Bool entries). The
|
||||
/// `OAK_HWACCEL` environment variable overrides both: `0` force-disables
|
||||
/// hardware decode entirely (a diagnostic escape hatch on machines where
|
||||
/// probing the device is slow or noisy).
|
||||
pub fn hardware_decoding_enabled() -> bool {
|
||||
match oak_common::configstore::ConfigStore::instance()
|
||||
.get(None, CONFIG_KEY_HARDWARE_DECODING)
|
||||
{
|
||||
Ok(value) => value != "false",
|
||||
Err(_) => true,
|
||||
}
|
||||
if let Ok(v) = std::env::var("OAK_HWACCEL") {
|
||||
return v != "0";
|
||||
}
|
||||
match oak_common::configstore::ConfigStore::instance()
|
||||
.get(None, CONFIG_KEY_HARDWARE_DECODING)
|
||||
{
|
||||
Ok(value) => value != "false",
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// The hardware device types to try, most preferred first. On a machine
|
||||
@@ -116,6 +160,18 @@ pub fn open_hw_accel(
|
||||
codec: ffmpeg::Codec,
|
||||
device_type: sys::AVHWDeviceType,
|
||||
) -> Option<(ffmpeg::codec::decoder::Opened, sys::AVHWDeviceType)> {
|
||||
// Negative cache: a device type whose context creation failed once
|
||||
// (no driver, headless box) is never tried again — creation is slow
|
||||
// and logs `[VAAPI @ ...] Failed to initialise VAAPI connection` per
|
||||
// attempt. Checked before any FFmpeg call so marked types cost
|
||||
// nothing.
|
||||
if device_unavailable(device_type) {
|
||||
return None;
|
||||
}
|
||||
#[cfg(test)]
|
||||
{
|
||||
CREATE_ATTEMPTS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
let mut context = ffmpeg::codec::Context::from_parameters(params.clone()).ok()?;
|
||||
let mut device: *mut sys::AVBufferRef = std::ptr::null_mut();
|
||||
// SAFETY: `device` is a valid out-pointer; on success it owns the
|
||||
@@ -130,6 +186,9 @@ pub fn open_hw_accel(
|
||||
)
|
||||
};
|
||||
if rc < 0 || device.is_null() {
|
||||
// The device is unavailable (missing driver / no hardware): mark
|
||||
// it so later opens skip the attempt and its log noise.
|
||||
mark_device_unavailable(device_type);
|
||||
return None;
|
||||
}
|
||||
// SAFETY: `hw_device_ctx` takes ownership of the reference; the codec
|
||||
@@ -241,4 +300,47 @@ mod tests {
|
||||
);
|
||||
assert!(opened.is_some(), "VideoToolbox hwaccel must open for H.264");
|
||||
}
|
||||
|
||||
/// A device type marked unavailable is short-circuited before any
|
||||
/// FFmpeg call: `open_hw_accel` returns None without attempting device
|
||||
/// creation (which is what used to log
|
||||
/// `[VAAPI @ ...] Failed to initialise VAAPI connection` every time on
|
||||
/// boxes without the driver).
|
||||
#[test]
|
||||
fn negative_cache_skips_marked_device() {
|
||||
// VDPAU is not a candidate on any supported platform, so marking
|
||||
// it cannot disturb the platform tests in this process (e.g. the
|
||||
// macOS VideoToolbox test above).
|
||||
let dev = sys::AVHWDeviceType::AV_HWDEVICE_TYPE_VDPAU;
|
||||
mark_device_unavailable(dev);
|
||||
assert!(device_unavailable(dev));
|
||||
|
||||
let input = ffmpeg::format::input(&"../oak-app/tests/demo.mp4".to_string())
|
||||
.expect("open demo.mp4");
|
||||
let fstream = input.stream(0).expect("stream 0");
|
||||
let params = fstream.parameters();
|
||||
let codec = ffmpeg::decoder::find(params.id()).expect("software h264 codec");
|
||||
|
||||
CREATE_ATTEMPTS.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
let opened = open_hw_accel(¶ms, codec, dev);
|
||||
assert!(opened.is_none(), "marked device must not open");
|
||||
assert_eq!(
|
||||
CREATE_ATTEMPTS.load(std::sync::atomic::Ordering::Relaxed),
|
||||
0,
|
||||
"marked device must be skipped before any creation attempt"
|
||||
);
|
||||
|
||||
// Restore so the (non-candidate) type stays clean for other tests.
|
||||
UNAVAILABLE_DEVICES[dev as usize].store(false, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// A device type never tried before is not cached as unavailable
|
||||
/// (only a failed creation marks it). DXVA2 is not a candidate on any
|
||||
/// supported platform (Windows uses D3D11VA/CUDA), so it is never
|
||||
/// touched by other tests in this process.
|
||||
#[test]
|
||||
fn unmarked_device_type_is_not_unavailable() {
|
||||
let dev = sys::AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2;
|
||||
assert!(!device_unavailable(dev));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,6 +179,85 @@ fn audio_decode_is_non_empty() {
|
||||
assert!(peak > 0.0, "decoded audio is all silence");
|
||||
}
|
||||
|
||||
/// Contiguous audio chunks (each chunk's start is exactly where the
|
||||
/// previous chunk ended) must continue the decode without re-seeking:
|
||||
/// the seek count stays flat, and the concatenated chunks must match a
|
||||
/// one-shot decode of the same span. A non-contiguous chunk re-seeks.
|
||||
/// This is the regression test for the boundary pops/clicks caused by the
|
||||
/// per-chunk `av_seek_frame` + decoder flush + resampler reset.
|
||||
#[test]
|
||||
fn contiguous_audio_chunks_skip_seek_and_match_oneshot() {
|
||||
let d = FFmpegDecoder::new();
|
||||
let s = CodecStream::with_block(demo_path().to_string_lossy().into_owned(), 1, None);
|
||||
d.open(&s).expect("open audio stream");
|
||||
|
||||
// Chunk [0s, 1s): opens the session (one seek inside retrieve).
|
||||
let mut c1 = vec![0f32; 48000 * 2];
|
||||
d.retrieve_audio(
|
||||
&mut c1,
|
||||
&TimeRange::new(Rational::new(0, 1), Rational::new(1, 1)),
|
||||
48000,
|
||||
0x3,
|
||||
)
|
||||
.expect("chunk [0,1)");
|
||||
assert!(c1.iter().any(|&v| v != 0.0), "chunk [0,1) is all silence");
|
||||
let seeks_after_first = d.audio_seek_count();
|
||||
|
||||
// Chunk [1s, 2s): contiguous with the previous one — must not seek.
|
||||
let mut c2 = vec![0f32; 48000 * 2];
|
||||
d.retrieve_audio(
|
||||
&mut c2,
|
||||
&TimeRange::new(Rational::new(1, 1), Rational::new(2, 1)),
|
||||
48000,
|
||||
0x3,
|
||||
)
|
||||
.expect("chunk [1,2)");
|
||||
assert!(
|
||||
d.audio_seek_count() == seeks_after_first,
|
||||
"contiguous chunk must skip the seek ({} -> {})",
|
||||
seeks_after_first,
|
||||
d.audio_seek_count()
|
||||
);
|
||||
assert!(c2.iter().any(|&v| v != 0.0), "chunk [1,2) is all silence");
|
||||
|
||||
// Chunk [3s, 4s): not contiguous — must seek again.
|
||||
let mut c4 = vec![0f32; 48000 * 2];
|
||||
d.retrieve_audio(
|
||||
&mut c4,
|
||||
&TimeRange::new(Rational::new(3, 1), Rational::new(4, 1)),
|
||||
48000,
|
||||
0x3,
|
||||
)
|
||||
.expect("chunk [3,4)");
|
||||
assert!(
|
||||
d.audio_seek_count() > seeks_after_first,
|
||||
"non-contiguous chunk must seek"
|
||||
);
|
||||
|
||||
// Concatenating [0,1)+[1,2) must equal a one-shot decode of [0,2):
|
||||
// both read the same decoder continuously from sample 0, so the
|
||||
// samples must match exactly (the second chunk only differs in that it
|
||||
// skipped its seek — no flush/resampler reset was involved).
|
||||
let mut combined = c1;
|
||||
combined.extend_from_slice(&c2);
|
||||
let mut oneshot = vec![0f32; 48000 * 2 * 2];
|
||||
d.retrieve_audio(
|
||||
&mut oneshot,
|
||||
&TimeRange::new(Rational::new(0, 1), Rational::new(2, 1)),
|
||||
48000,
|
||||
0x3,
|
||||
)
|
||||
.expect("oneshot [0,2)");
|
||||
let mut max_diff = 0.0f32;
|
||||
for (a, b) in combined.iter().zip(oneshot.iter()) {
|
||||
max_diff = max_diff.max((a - b).abs());
|
||||
}
|
||||
assert!(
|
||||
max_diff < 0.01,
|
||||
"contiguous chunks diverge from one-shot decode (max diff {max_diff})"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_h264_roundtrip_to_tmp() {
|
||||
let out = std::env::temp_dir().join(format!("oakcodec_roundtrip_{}.mp4", std::process::id()));
|
||||
|
||||
+314
-170
@@ -31,6 +31,15 @@
|
||||
//! chromatic adaptation between differing white points (D65 sources → the
|
||||
//! ACES D60-ish white of AP1). Unit tests pin the results against the
|
||||
//! published ACES transform values.
|
||||
//!
|
||||
//! ## Performance
|
||||
//!
|
||||
//! The per-pixel transforms run over full frames on hot paths (decode,
|
||||
//! the app-side output node). libm `powf` costs ~15 ns per channel —
|
||||
//! ~375 ms per 1080p frame for a matrix+OETF pass — so the transfer
|
||||
//! functions are evaluated through 4096-entry LUTs with linear
|
||||
//! interpolation (max error ≈ 2e-5, far below 10-bit quantization) and
|
||||
//! large frames are split across scoped threads by row band.
|
||||
|
||||
/// A 3x3 row-major matrix.
|
||||
pub type Mat3 = [[f32; 3]; 3];
|
||||
@@ -476,6 +485,138 @@ pub fn hlg_oetf(v: f32) -> f32 {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fast transfer evaluation (LUT) and parallel per-pixel transforms
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// LUT intervals over the [0, 1] domain (plus the 1.0 endpoint).
|
||||
const LUT_N: usize = 4096;
|
||||
|
||||
/// Declare one lazily-built transfer LUT over [0, 1].
|
||||
macro_rules! tf_lut {
|
||||
($name:ident, $f:expr) => {
|
||||
static $name: std::sync::LazyLock<[f32; LUT_N + 1]> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
let f: fn(f32) -> f32 = $f;
|
||||
let mut t = [0.0f32; LUT_N + 1];
|
||||
for (i, e) in t.iter_mut().enumerate() {
|
||||
*e = f(i as f32 / LUT_N as f32);
|
||||
}
|
||||
t
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
tf_lut!(SRGB_EOTF_LUT, srgb_eotf);
|
||||
tf_lut!(SRGB_OETF_LUT, srgb_oetf);
|
||||
tf_lut!(GAMMA22_EOTF_LUT, |v| gamma_eotf(v, 2.2));
|
||||
tf_lut!(GAMMA22_OETF_LUT, |v| gamma_oetf(v, 2.2));
|
||||
tf_lut!(GAMMA28_EOTF_LUT, |v| gamma_eotf(v, 2.8));
|
||||
tf_lut!(PQ_EOTF_LUT, pq_eotf);
|
||||
tf_lut!(PQ_OETF_LUT, pq_oetf);
|
||||
tf_lut!(HLG_EOTF_LUT, hlg_eotf);
|
||||
tf_lut!(HLG_OETF_LUT, hlg_oetf);
|
||||
|
||||
/// Evaluate a transfer function through its [0, 1] LUT (linear
|
||||
/// interpolation; max error ≈ 2e-5 — far below 10-bit quantization).
|
||||
/// Negatives mirror like the exact functions; values above 1.0 fall back
|
||||
/// to the exact function (HDR super-whites are rare and already clamped
|
||||
/// by the output node).
|
||||
#[inline]
|
||||
fn lut_tf(lut: &[f32; LUT_N + 1], exact: fn(f32) -> f32, v: f32) -> f32 {
|
||||
let (sign, a) = if v < 0.0 { (-1.0, -v) } else { (1.0, v) };
|
||||
let out = if a >= 1.0 {
|
||||
exact(a)
|
||||
} else {
|
||||
let x = a * LUT_N as f32;
|
||||
let i = (x as usize).min(LUT_N - 1);
|
||||
let frac = x - i as f32;
|
||||
lut[i] + frac * (lut[i + 1] - lut[i])
|
||||
};
|
||||
sign * out
|
||||
}
|
||||
|
||||
/// Fast [`srgb_eotf`]: the linear toe is exact, the power part is LUT'd.
|
||||
#[inline]
|
||||
fn srgb_eotf_fast(v: f32) -> f32 {
|
||||
if v.abs() <= 0.04045 {
|
||||
v / 12.92
|
||||
} else {
|
||||
lut_tf(&SRGB_EOTF_LUT, srgb_eotf, v)
|
||||
}
|
||||
}
|
||||
|
||||
/// Fast [`srgb_oetf`]: the linear toe is exact, the power part is LUT'd.
|
||||
#[inline]
|
||||
fn srgb_oetf_fast(v: f32) -> f32 {
|
||||
if v.abs() <= 0.0031308 {
|
||||
12.92 * v
|
||||
} else {
|
||||
lut_tf(&SRGB_OETF_LUT, srgb_oetf, v)
|
||||
}
|
||||
}
|
||||
|
||||
/// The fast decode-side linearizer for `transfer`.
|
||||
fn decode_transfer_fn(transfer: SourceTransfer) -> fn(f32) -> f32 {
|
||||
match transfer {
|
||||
SourceTransfer::SdrGamma | SourceTransfer::Unknown => srgb_eotf_fast,
|
||||
SourceTransfer::Gamma22 => |v| lut_tf(&GAMMA22_EOTF_LUT, |x| gamma_eotf(x, 2.2), v),
|
||||
SourceTransfer::Gamma28 => |v| lut_tf(&GAMMA28_EOTF_LUT, |x| gamma_eotf(x, 2.8), v),
|
||||
SourceTransfer::Pq => |v| lut_tf(&PQ_EOTF_LUT, pq_eotf, v),
|
||||
SourceTransfer::Hlg => |v| lut_tf(&HLG_EOTF_LUT, hlg_eotf, v),
|
||||
SourceTransfer::Linear => |v| v,
|
||||
}
|
||||
}
|
||||
|
||||
/// The fast output-side encoder for `transfer`.
|
||||
fn output_transfer_fn(transfer: OutputTransfer) -> fn(f32) -> f32 {
|
||||
match transfer {
|
||||
OutputTransfer::Srgb => srgb_oetf_fast,
|
||||
OutputTransfer::Gamma22 => |v| lut_tf(&GAMMA22_OETF_LUT, |x| gamma_oetf(x, 2.2), v),
|
||||
OutputTransfer::Pq => |v| lut_tf(&PQ_OETF_LUT, pq_oetf, v),
|
||||
OutputTransfer::Hlg => |v| lut_tf(&HLG_OETF_LUT, hlg_oetf, v),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split `samples` (whole pixels = 4 floats) into row bands and run `f`
|
||||
/// on each band, on scoped threads when the frame is big enough to be
|
||||
/// worth the spawn overhead (a single 1080p frame is ~8 Mpx; the per-
|
||||
/// pixel work below is memory- and ALU-bound, so bands scale).
|
||||
fn par_pixels_f32(samples: &mut [f32], f: impl Fn(&mut [f32]) + Sync) {
|
||||
par_bands(samples, 4, f);
|
||||
}
|
||||
|
||||
/// The `&mut [u8]` analog of [`par_pixels_f32`] (16 bytes per pixel).
|
||||
fn par_pixels_bytes(bytes: &mut [u8], f: impl Fn(&mut [u8]) + Sync) {
|
||||
par_bands(bytes, 16, f);
|
||||
}
|
||||
|
||||
fn par_bands<T: Send>(buf: &mut [T], align: usize, f: impl Fn(&mut [T]) + Sync) {
|
||||
let len = buf.len();
|
||||
let bands = if len / align >= (1 << 19) {
|
||||
std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(1)
|
||||
.min(8)
|
||||
} else {
|
||||
1
|
||||
};
|
||||
if bands <= 1 {
|
||||
f(buf);
|
||||
return;
|
||||
}
|
||||
let band_len = (len / bands).div_ceil(align) * align;
|
||||
let f = &f;
|
||||
std::thread::scope(|s| {
|
||||
let mut rest = buf;
|
||||
while !rest.is_empty() {
|
||||
let (band, tail) = rest.split_at_mut(band_len.min(rest.len()));
|
||||
s.spawn(move || f(band));
|
||||
rest = tail;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Source decode characterization
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -497,11 +638,13 @@ pub enum SourcePrimaries {
|
||||
/// The transfer characteristic of a decoded source.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum SourceTransfer {
|
||||
/// The sRGB piecewise EOTF (IEC 61966-2-1), applied at decode.
|
||||
/// The sRGB piecewise EOTF (IEC 61966-2-1), applied at decode. Also
|
||||
/// used for BT.709 / BT.2020 SDR video: the BT.709 camera OETF inverse
|
||||
/// is ≈ gamma 2.2 (sRGB EOTF to within measurement noise), while
|
||||
/// decoding with the BT.1886 *display* EOTF (2.4) bakes in the ~1.1
|
||||
/// system gamma that belongs at final display only — SDR sources
|
||||
/// round-tripped through the working space came out visibly dark.
|
||||
SdrGamma,
|
||||
/// BT.1886 display-referred EOTF (pure power 2.4) — the SDR display
|
||||
/// reference for BT.709 / BT.2020 sources.
|
||||
Gamma24,
|
||||
/// Pure power 2.2 gamma.
|
||||
Gamma22,
|
||||
/// Pure power 2.8 gamma.
|
||||
@@ -532,13 +675,13 @@ pub fn source_primaries_from_av(color_primaries: i32) -> SourcePrimaries {
|
||||
/// (AVCOL_TRC_* numbering, H.273 ISO codes.)
|
||||
pub fn source_transfer_from_av(color_trc: i32) -> SourceTransfer {
|
||||
match color_trc {
|
||||
1 => SourceTransfer::Gamma24, // BT.709
|
||||
1 => SourceTransfer::SdrGamma, // BT.709 (see the enum docs: OETF inverse ≈ 2.2)
|
||||
4 => SourceTransfer::Gamma22, // gamma 2.2
|
||||
5 => SourceTransfer::Gamma28, // gamma 2.8
|
||||
6 => SourceTransfer::Gamma24, // SMPTE 170M
|
||||
6 => SourceTransfer::SdrGamma, // SMPTE 170M
|
||||
13 => SourceTransfer::SdrGamma, // sRGB (its EOTF is applied at decode)
|
||||
14 => SourceTransfer::Gamma24, // BT.2020 10-bit
|
||||
15 => SourceTransfer::Gamma24, // BT.2020 12-bit
|
||||
14 => SourceTransfer::SdrGamma, // BT.2020 10-bit
|
||||
15 => SourceTransfer::SdrGamma, // BT.2020 12-bit
|
||||
16 => SourceTransfer::Pq, // SMPTE ST 2084
|
||||
18 => SourceTransfer::Hlg, // ARIB STD-B67
|
||||
8 => SourceTransfer::Linear, // linear
|
||||
@@ -563,55 +706,23 @@ impl SourcePrimaries {
|
||||
|
||||
/// Decode-direction transform: gamma-encoded source RGB (in the source's
|
||||
/// own primaries) → ACEScg linear. `samples` is an F32 RGBA buffer,
|
||||
/// transformed in place.
|
||||
/// transformed in place. Linearize and the primaries matrix are fused
|
||||
/// into a single pass (LUT'd transfer, row-band parallel).
|
||||
pub fn decode_to_acescg(
|
||||
samples: &mut [f32],
|
||||
primaries: SourcePrimaries,
|
||||
transfer: SourceTransfer,
|
||||
) {
|
||||
// Stage 1: linearize in the source's own primaries.
|
||||
match transfer {
|
||||
SourceTransfer::SdrGamma | SourceTransfer::Unknown => {
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = srgb_eotf(px[c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
SourceTransfer::Gamma24 | SourceTransfer::Gamma22 | SourceTransfer::Gamma28 => {
|
||||
let gamma = match transfer {
|
||||
SourceTransfer::Gamma24 => 2.4,
|
||||
SourceTransfer::Gamma22 => 2.2,
|
||||
_ => 2.8,
|
||||
};
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = gamma_eotf(px[c], gamma);
|
||||
}
|
||||
}
|
||||
}
|
||||
SourceTransfer::Pq => {
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = pq_eotf(px[c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
SourceTransfer::Hlg => {
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = hlg_eotf(px[c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
SourceTransfer::Linear => {}
|
||||
}
|
||||
|
||||
// Stage 2: source primaries → AP1 (with chromatic adaptation).
|
||||
let linearize = decode_transfer_fn(transfer);
|
||||
let matrix = rgb_to_rgb_matrix(primaries.primaries(), PRIMARIES_AP1);
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
apply_mat(matrix, px);
|
||||
}
|
||||
par_pixels_f32(samples, move |band| {
|
||||
for px in band.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = linearize(px[c]);
|
||||
}
|
||||
apply_mat(matrix, px);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Decode-direction transform on a little-endian F32 RGBA byte buffer
|
||||
@@ -626,33 +737,26 @@ pub fn decode_to_acescg_bytes(
|
||||
if bytes.len() < pixels * 16 {
|
||||
return;
|
||||
}
|
||||
let linearize: fn(f32) -> f32 = match transfer {
|
||||
SourceTransfer::SdrGamma | SourceTransfer::Unknown => srgb_eotf,
|
||||
SourceTransfer::Gamma24 => |v| gamma_eotf(v, 2.4),
|
||||
SourceTransfer::Gamma22 => |v| gamma_eotf(v, 2.2),
|
||||
SourceTransfer::Gamma28 => |v| gamma_eotf(v, 2.8),
|
||||
SourceTransfer::Pq => pq_eotf,
|
||||
SourceTransfer::Hlg => hlg_eotf,
|
||||
SourceTransfer::Linear => |v| v,
|
||||
};
|
||||
let linearize = decode_transfer_fn(transfer);
|
||||
let matrix = rgb_to_rgb_matrix(primaries.primaries(), PRIMARIES_AP1);
|
||||
for i in 0..pixels {
|
||||
let off = i * 16;
|
||||
let mut px = [
|
||||
f32::from_le_bytes(bytes[off..off + 4].try_into().unwrap()),
|
||||
f32::from_le_bytes(bytes[off + 4..off + 8].try_into().unwrap()),
|
||||
f32::from_le_bytes(bytes[off + 8..off + 12].try_into().unwrap()),
|
||||
f32::from_le_bytes(bytes[off + 12..off + 16].try_into().unwrap()),
|
||||
];
|
||||
for c in 0..3 {
|
||||
px[c] = linearize(px[c]);
|
||||
par_pixels_bytes(bytes, move |band| {
|
||||
for px in band.chunks_exact_mut(16) {
|
||||
let mut v = [
|
||||
f32::from_le_bytes(px[0..4].try_into().unwrap()),
|
||||
f32::from_le_bytes(px[4..8].try_into().unwrap()),
|
||||
f32::from_le_bytes(px[8..12].try_into().unwrap()),
|
||||
f32::from_le_bytes(px[12..16].try_into().unwrap()),
|
||||
];
|
||||
for c in 0..3 {
|
||||
v[c] = linearize(v[c]);
|
||||
}
|
||||
apply_mat(matrix, &mut v);
|
||||
px[0..4].copy_from_slice(&v[0].to_le_bytes());
|
||||
px[4..8].copy_from_slice(&v[1].to_le_bytes());
|
||||
px[8..12].copy_from_slice(&v[2].to_le_bytes());
|
||||
px[12..16].copy_from_slice(&v[3].to_le_bytes());
|
||||
}
|
||||
apply_mat(matrix, &mut px);
|
||||
bytes[off..off + 4].copy_from_slice(&px[0].to_le_bytes());
|
||||
bytes[off + 4..off + 8].copy_from_slice(&px[1].to_le_bytes());
|
||||
bytes[off + 8..off + 12].copy_from_slice(&px[2].to_le_bytes());
|
||||
bytes[off + 12..off + 16].copy_from_slice(&px[3].to_le_bytes());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Output-direction transform: ACEScg linear → the output colorspace
|
||||
@@ -662,16 +766,19 @@ pub fn decode_to_acescg_bytes(
|
||||
/// AP1 → target-gamut matrix, the RGB channels are clamped to [0, 1] before
|
||||
/// the transfer encoding, so out-of-gamut values are clipped rather than
|
||||
/// wrapped. This applies to HDR targets too (PQ/HLG code 1.0 = 10 000 nits);
|
||||
/// alpha is untouched.
|
||||
/// alpha is untouched. Matrix, clamp and encoding are fused into a single
|
||||
/// pass (LUT'd transfer, row-band parallel).
|
||||
pub fn acescg_to_output(samples: &mut [f32], spec: OutputColorSpec) {
|
||||
let matrix = rgb_to_rgb_matrix(PRIMARIES_AP1, spec.gamut.primaries());
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
apply_mat(matrix, px);
|
||||
for c in 0..3 {
|
||||
px[c] = px[c].clamp(0.0, 1.0);
|
||||
let encode = output_transfer_fn(spec.transfer);
|
||||
par_pixels_f32(samples, move |band| {
|
||||
for px in band.chunks_exact_mut(4) {
|
||||
apply_mat(matrix, px);
|
||||
for c in 0..3 {
|
||||
px[c] = encode(px[c].clamp(0.0, 1.0));
|
||||
}
|
||||
}
|
||||
}
|
||||
apply_transfer_oetf(samples, spec.transfer);
|
||||
});
|
||||
}
|
||||
|
||||
/// Output-direction transform on a little-endian F32 RGBA byte buffer
|
||||
@@ -683,67 +790,38 @@ pub fn acescg_to_output_bytes(bytes: &mut [u8], pixels: usize, spec: OutputColor
|
||||
return;
|
||||
}
|
||||
let matrix = rgb_to_rgb_matrix(PRIMARIES_AP1, spec.gamut.primaries());
|
||||
let encode: fn(f32) -> f32 = match spec.transfer {
|
||||
OutputTransfer::Srgb => srgb_oetf,
|
||||
OutputTransfer::Gamma22 => |v| gamma_oetf(v, 2.2),
|
||||
OutputTransfer::Pq => pq_oetf,
|
||||
OutputTransfer::Hlg => hlg_oetf,
|
||||
};
|
||||
for i in 0..pixels {
|
||||
let off = i * 16;
|
||||
let mut px = [
|
||||
f32::from_le_bytes(bytes[off..off + 4].try_into().unwrap()),
|
||||
f32::from_le_bytes(bytes[off + 4..off + 8].try_into().unwrap()),
|
||||
f32::from_le_bytes(bytes[off + 8..off + 12].try_into().unwrap()),
|
||||
f32::from_le_bytes(bytes[off + 12..off + 16].try_into().unwrap()),
|
||||
];
|
||||
apply_mat(matrix, &mut px);
|
||||
for c in 0..3 {
|
||||
px[c] = px[c].clamp(0.0, 1.0);
|
||||
let encode = output_transfer_fn(spec.transfer);
|
||||
par_pixels_bytes(bytes, move |band| {
|
||||
for px in band.chunks_exact_mut(16) {
|
||||
let mut v = [
|
||||
f32::from_le_bytes(px[0..4].try_into().unwrap()),
|
||||
f32::from_le_bytes(px[4..8].try_into().unwrap()),
|
||||
f32::from_le_bytes(px[8..12].try_into().unwrap()),
|
||||
f32::from_le_bytes(px[12..16].try_into().unwrap()),
|
||||
];
|
||||
apply_mat(matrix, &mut v);
|
||||
for c in 0..3 {
|
||||
v[c] = encode(v[c].clamp(0.0, 1.0));
|
||||
}
|
||||
px[0..4].copy_from_slice(&v[0].to_le_bytes());
|
||||
px[4..8].copy_from_slice(&v[1].to_le_bytes());
|
||||
px[8..12].copy_from_slice(&v[2].to_le_bytes());
|
||||
px[12..16].copy_from_slice(&v[3].to_le_bytes());
|
||||
}
|
||||
for c in 0..3 {
|
||||
px[c] = encode(px[c]);
|
||||
}
|
||||
bytes[off..off + 4].copy_from_slice(&px[0].to_le_bytes());
|
||||
bytes[off + 4..off + 8].copy_from_slice(&px[1].to_le_bytes());
|
||||
bytes[off + 8..off + 12].copy_from_slice(&px[2].to_le_bytes());
|
||||
bytes[off + 12..off + 16].copy_from_slice(&px[3].to_le_bytes());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Apply just the transfer encoding of `transfer` (linear → code). In place
|
||||
/// on the RGB channels.
|
||||
/// on the RGB channels (LUT'd, row-band parallel).
|
||||
pub fn apply_transfer_oetf(samples: &mut [f32], transfer: OutputTransfer) {
|
||||
match transfer {
|
||||
OutputTransfer::Srgb => {
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = srgb_oetf(px[c]);
|
||||
}
|
||||
let encode = output_transfer_fn(transfer);
|
||||
par_pixels_f32(samples, move |band| {
|
||||
for px in band.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = encode(px[c]);
|
||||
}
|
||||
}
|
||||
OutputTransfer::Gamma22 => {
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = gamma_oetf(px[c], 2.2);
|
||||
}
|
||||
}
|
||||
}
|
||||
OutputTransfer::Pq => {
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = pq_oetf(px[c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
OutputTransfer::Hlg => {
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = hlg_oetf(px[c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// The presentation transform: working space → the display target colorspace
|
||||
@@ -772,21 +850,23 @@ pub fn working_to_display_target(
|
||||
/// Hlg → HLG), then convert RGB → XYZ through the gamut's own matrix. All
|
||||
/// output gamuts share the D65 white point, so no chromatic adaptation is
|
||||
/// needed; `rgb_to_xyz_matrix` already normalizes the white point to unit
|
||||
/// luminance.
|
||||
/// luminance. Linearize and matrix are fused into one LUT'd parallel pass.
|
||||
pub fn output_spec_to_xyz_d65(samples: &mut [f32], spec: OutputColorSpec) {
|
||||
let linearize: fn(f32) -> f32 = match spec.transfer {
|
||||
OutputTransfer::Srgb => srgb_eotf,
|
||||
OutputTransfer::Gamma22 => |v| gamma_eotf(v, 2.2),
|
||||
OutputTransfer::Pq => pq_eotf,
|
||||
OutputTransfer::Hlg => hlg_eotf,
|
||||
};
|
||||
let linearize = decode_transfer_fn(match spec.transfer {
|
||||
OutputTransfer::Srgb => SourceTransfer::SdrGamma,
|
||||
OutputTransfer::Gamma22 => SourceTransfer::Gamma22,
|
||||
OutputTransfer::Pq => SourceTransfer::Pq,
|
||||
OutputTransfer::Hlg => SourceTransfer::Hlg,
|
||||
});
|
||||
let matrix = rgb_to_xyz_matrix(spec.gamut.primaries());
|
||||
for px in samples.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = linearize(px[c]);
|
||||
par_pixels_f32(samples, move |band| {
|
||||
for px in band.chunks_exact_mut(4) {
|
||||
for c in 0..3 {
|
||||
px[c] = linearize(px[c]);
|
||||
}
|
||||
apply_mat(matrix, px);
|
||||
}
|
||||
apply_mat(matrix, px);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -1050,13 +1130,13 @@ mod tests {
|
||||
assert_eq!(source_primaries_from_av(9), SourcePrimaries::Bt2020);
|
||||
assert_eq!(source_primaries_from_av(12), SourcePrimaries::DisplayP3);
|
||||
assert_eq!(source_primaries_from_av(2), SourcePrimaries::Unknown);
|
||||
assert_eq!(source_transfer_from_av(1), SourceTransfer::Gamma24);
|
||||
assert_eq!(source_transfer_from_av(1), SourceTransfer::SdrGamma);
|
||||
assert_eq!(source_transfer_from_av(4), SourceTransfer::Gamma22);
|
||||
assert_eq!(source_transfer_from_av(5), SourceTransfer::Gamma28);
|
||||
assert_eq!(source_transfer_from_av(6), SourceTransfer::Gamma24);
|
||||
assert_eq!(source_transfer_from_av(6), SourceTransfer::SdrGamma);
|
||||
assert_eq!(source_transfer_from_av(13), SourceTransfer::SdrGamma);
|
||||
assert_eq!(source_transfer_from_av(14), SourceTransfer::Gamma24);
|
||||
assert_eq!(source_transfer_from_av(15), SourceTransfer::Gamma24);
|
||||
assert_eq!(source_transfer_from_av(14), SourceTransfer::SdrGamma);
|
||||
assert_eq!(source_transfer_from_av(15), SourceTransfer::SdrGamma);
|
||||
assert_eq!(source_transfer_from_av(16), SourceTransfer::Pq);
|
||||
assert_eq!(source_transfer_from_av(18), SourceTransfer::Hlg);
|
||||
assert_eq!(source_transfer_from_av(8), SourceTransfer::Linear);
|
||||
@@ -1121,29 +1201,93 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_gamma24_dispatch() {
|
||||
// BT.1886 decode: code 0.5 linearizes via pure power 2.4 (≈ 0.1895,
|
||||
// not sRGB's piecewise ≈ 0.2140), then the BT.709 → AP1 matrix.
|
||||
let mut samples = [0.5f32, 0.5, 0.5, 1.0];
|
||||
decode_to_acescg(&mut samples, SourcePrimaries::Bt709, SourceTransfer::Gamma24);
|
||||
let m = rgb_to_rgb_matrix(PRIMARIES_SRGB, PRIMARIES_AP1);
|
||||
let expected = mat_vec(m, [gamma_eotf(0.5, 2.4); 3]);
|
||||
for i in 0..3 {
|
||||
assert!(
|
||||
approx(samples[i], expected[i], 1e-6),
|
||||
"Gamma24 decode channel {i}: {} vs {}",
|
||||
samples[i],
|
||||
expected[i]
|
||||
);
|
||||
fn lut_fast_transfer_matches_exact() {
|
||||
// The LUT'd fast paths track the exact functions over the whole
|
||||
// [0, 1] domain (and mirror negatives); far below 10-bit quanta.
|
||||
let cases: &[(&std::sync::LazyLock<[f32; LUT_N + 1]>, fn(f32) -> f32, &str)] = &[
|
||||
(&SRGB_EOTF_LUT, srgb_eotf, "srgb_eotf"),
|
||||
(&SRGB_OETF_LUT, srgb_oetf, "srgb_oetf"),
|
||||
(&GAMMA22_EOTF_LUT, |v| gamma_eotf(v, 2.2), "gamma22_eotf"),
|
||||
(&GAMMA22_OETF_LUT, |v| gamma_oetf(v, 2.2), "gamma22_oetf"),
|
||||
(&GAMMA28_EOTF_LUT, |v| gamma_eotf(v, 2.8), "gamma28_eotf"),
|
||||
(&PQ_EOTF_LUT, pq_eotf, "pq_eotf"),
|
||||
(&PQ_OETF_LUT, pq_oetf, "pq_oetf"),
|
||||
(&HLG_EOTF_LUT, hlg_eotf, "hlg_eotf"),
|
||||
(&HLG_OETF_LUT, hlg_oetf, "hlg_oetf"),
|
||||
];
|
||||
for &(lut, exact, name) in cases {
|
||||
for k in 0..=1000 {
|
||||
let v = k as f32 / 1000.0;
|
||||
for v in [v, -v] {
|
||||
let fast = lut_tf(lut, exact, v);
|
||||
let want = exact(v);
|
||||
assert!(
|
||||
(fast - want).abs() < 2e-4,
|
||||
"{name}({v}): fast {fast} vs exact {want}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// ... and the bytes variant picks the same curve.
|
||||
// Above 1.0 the exact function is used.
|
||||
assert_eq!(lut_tf(&SRGB_OETF_LUT, srgb_oetf, 2.0), srgb_oetf(2.0));
|
||||
// The sRGB wrappers keep their linear toes exact.
|
||||
assert_eq!(srgb_eotf_fast(0.02), 0.02 / 12.92);
|
||||
assert_eq!(srgb_oetf_fast(0.001), 12.92 * 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parallel_transform_matches_scalar_reference() {
|
||||
// A frame above the band threshold exercises the scoped-thread
|
||||
// path; it must agree with the scalar math pixel for pixel.
|
||||
let pixels = 1 << 20;
|
||||
let mut samples: Vec<f32> = (0..pixels * 4)
|
||||
.map(|i: usize| ((i.wrapping_mul(2654435761)) % 1000) as f32 / 999.0)
|
||||
.collect();
|
||||
let reference: Vec<f32> = samples
|
||||
.chunks_exact(4)
|
||||
.flat_map(|px| {
|
||||
let mut v = [srgb_eotf(px[0]), srgb_eotf(px[1]), srgb_eotf(px[2]), px[3]];
|
||||
apply_mat(rgb_to_rgb_matrix(PRIMARIES_SRGB, PRIMARIES_AP1), &mut v);
|
||||
v
|
||||
})
|
||||
.collect();
|
||||
decode_to_acescg(&mut samples, SourcePrimaries::Bt709, SourceTransfer::SdrGamma);
|
||||
for (i, (&got, &want)) in samples.iter().zip(reference.iter()).enumerate() {
|
||||
assert!(approx(got, want, 1e-4), "pixel {i}: {got} vs {want}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bt709_decode_output_round_trip_is_identity() {
|
||||
// SDR video (BT.709 / BT.2020 NCL tags) decodes with the sRGB EOTF
|
||||
// (OETF inverse ≈ 2.2 — see the SourceTransfer docs), so an SDR
|
||||
// frame round-tripped through the ACEScg working space and back to
|
||||
// sRGB output must come back unchanged. Decoding with the BT.1886
|
||||
// display EOTF (2.4) instead made this round trip visibly dark
|
||||
// (the ~1.1 system gamma belongs at final display only).
|
||||
for code in [0.05f32, 0.18, 0.5, 0.75, 1.0] {
|
||||
let mut samples = [code, code, code, 1.0];
|
||||
decode_to_acescg(&mut samples, SourcePrimaries::Bt709, SourceTransfer::SdrGamma);
|
||||
acescg_to_output(&mut samples, OutputColorSpec::default());
|
||||
for i in 0..3 {
|
||||
assert!(
|
||||
approx(samples[i], code, 1e-4),
|
||||
"BT.709 round trip code {code} channel {i}: {}",
|
||||
samples[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
// ... and the bytes variant picks the same curve (sRGB piecewise,
|
||||
// not a pure power): code 0.5 → srgb_eotf(0.5) ≈ 0.2140.
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes[0..4].copy_from_slice(&0.5f32.to_le_bytes());
|
||||
bytes[4..8].copy_from_slice(&0.5f32.to_le_bytes());
|
||||
bytes[8..12].copy_from_slice(&0.5f32.to_le_bytes());
|
||||
decode_to_acescg_bytes(&mut bytes, 1, SourcePrimaries::Bt709, SourceTransfer::Gamma24);
|
||||
decode_to_acescg_bytes(&mut bytes, 1, SourcePrimaries::Bt709, SourceTransfer::SdrGamma);
|
||||
let r = f32::from_le_bytes(bytes[0..4].try_into().unwrap());
|
||||
assert!(approx(r, expected[0], 1e-6), "bytes Gamma24 R: {r}");
|
||||
let m = rgb_to_rgb_matrix(PRIMARIES_SRGB, PRIMARIES_AP1);
|
||||
let expected = mat_vec(m, [srgb_eotf(0.5); 3]);
|
||||
assert!(approx(r, expected[0], 1e-6), "bytes SdrGamma R: {r}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1198,7 +1198,18 @@ fn audio_layout(params: &crate::ticket::AudioTicketParams) -> Result<(i32, u64,
|
||||
if seconds <= 0.0 || seconds > 3600.0 {
|
||||
return Err(Error::Invalid);
|
||||
}
|
||||
let total_frames = (seconds * rate as f64).round() as usize;
|
||||
|
||||
// Anchor each chunk to the absolute sample grid (`round(out·rate) -
|
||||
// round(in·rate)`) instead of rounding the duration: at fractional
|
||||
// frame rates (29.97 fps → 1601.6 samples/frame) a duration-round
|
||||
// would emit 1602 samples for every chunk and accumulate ~12 extra
|
||||
// samples per second, slowly desyncing audio from video. Per-chunk
|
||||
// anchoring keeps the total exact and matches the decode side
|
||||
// (`FFmpegDecoder::retrieve_audio_to` fills `round(out·rate) -
|
||||
// round(in·rate)` samples).
|
||||
let total_frames = ((params.range.out().to_f64() * rate as f64).round()
|
||||
- (params.range.in_().to_f64() * rate as f64).round())
|
||||
.max(0.0) as usize;
|
||||
Ok((rate, params.channel_layout, channels, total_frames))
|
||||
}
|
||||
|
||||
@@ -1230,8 +1241,11 @@ fn mix_audio_montage(
|
||||
if out_time <= in_time {
|
||||
continue;
|
||||
}
|
||||
let start_frame = ((in_time - params.range.in_()).to_f64() * rate as f64) as usize;
|
||||
let end_frame = ((out_time - params.range.in_()).to_f64() * rate as f64) as usize;
|
||||
// Round to the absolute sample grid like the decode side (which
|
||||
// anchors at `round(media_start·rate)`): truncation would shift a
|
||||
// clip's mix by up to one sample per chunk.
|
||||
let start_frame = ((in_time - params.range.in_()).to_f64() * rate as f64).round() as usize;
|
||||
let end_frame = ((out_time - params.range.in_()).to_f64() * rate as f64).round() as usize;
|
||||
if start_frame >= total_frames {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -232,6 +232,33 @@ impl RenderManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// M16 S1 graph mode: force the workers to re-load the current project
|
||||
/// snapshot even when the undo-stack revision is unchanged. The
|
||||
/// color-settings dialog writes project settings directly (no undo
|
||||
/// command), so the revision-based dedup in [`set_graph_snapshot`] would
|
||||
/// never re-send the snapshot — workers would keep rendering under the
|
||||
/// colors they loaded at project-load time. The snapshot file is
|
||||
/// rewritten (see [`GraphSnapshotStore::acquire_rewrite`]) and
|
||||
/// `load_graph` re-broadcast to every live worker (the dispatcher's
|
||||
/// re-send has no dedup).
|
||||
pub fn resync_graph_snapshot(
|
||||
&self,
|
||||
project: &std::sync::Mutex<oak_node::project::Project>,
|
||||
revision: u64,
|
||||
) -> Result<()> {
|
||||
if self.stopping.load(Ordering::Acquire) {
|
||||
return Ok(()); // teardown: no re-arm after the drain
|
||||
}
|
||||
let path = self.snapshots.acquire_rewrite(project, revision)?;
|
||||
if let Some(old) = lock(&self.current_snapshot).replace(path.clone()) {
|
||||
self.snapshots.release(&old);
|
||||
}
|
||||
self.dispatch.set_graph_snapshot(Some(path));
|
||||
let uuid = lock(project).uuid.clone();
|
||||
*lock(&self.current_key) = Some((uuid, revision));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// M16 S1 graph mode: drop the current snapshot (project closed). The
|
||||
/// protocol has no clear message, so alive workers keep their loaded
|
||||
/// graph; new/restarted workers no longer load it and the snapshot file
|
||||
|
||||
@@ -352,6 +352,51 @@ impl GraphSnapshotStore {
|
||||
Ok(path_str)
|
||||
}
|
||||
|
||||
/// Like [`acquire`], but always rewrites the snapshot file even when the
|
||||
/// (uuid, revision) key is already present. Used to force the workers to
|
||||
/// re-deserialize a project whose settings changed without an undo-stack
|
||||
/// revision bump (color-settings resync): the dedup in [`acquire`] would
|
||||
/// otherwise hand the workers the stale file and `load_graph` would never
|
||||
/// re-broadcast. Atomic staging and reference counting are identical to
|
||||
/// [`acquire`].
|
||||
pub fn acquire_rewrite(
|
||||
&self,
|
||||
project: &std::sync::Mutex<oak_node::project::Project>,
|
||||
revision: u64,
|
||||
) -> Result<String> {
|
||||
let (uuid, xml) = {
|
||||
let g = lock(project);
|
||||
let xml = oak_node::serializer::save(&g)
|
||||
.map_err(|e| Error::Failed(format!("save graph snapshot: {e}")))?;
|
||||
(g.uuid.clone(), xml)
|
||||
};
|
||||
let path = self.dir.join(format!("graph-{uuid}-{revision}.xml"));
|
||||
let path_str = path.to_string_lossy().into_owned();
|
||||
// Atomic staging: temp file + rename (see [`acquire`]).
|
||||
let tmp = self
|
||||
.dir
|
||||
.join(format!("graph-{uuid}-{revision}.{}.tmp", std::process::id()));
|
||||
std::fs::write(&tmp, &xml)
|
||||
.map_err(|e| Error::Failed(format!("write snapshot temp: {e}")))?;
|
||||
if let Err(e) = std::fs::rename(&tmp, &path) {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(Error::Failed(format!("rename snapshot: {e}")));
|
||||
}
|
||||
let mut entries = lock(&self.entries);
|
||||
if let Some(entry) = entries.get_mut(&path_str) {
|
||||
entry.refs += 1;
|
||||
} else {
|
||||
entries.insert(
|
||||
path_str.clone(),
|
||||
SnapshotEntry {
|
||||
refs: 1,
|
||||
cached: false,
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(path_str)
|
||||
}
|
||||
|
||||
/// Drop one reference. The FILE IS NOT UNLINKED: a worker may still
|
||||
/// hold the path for a late `load_graph` (M16 S1), and per-project
|
||||
/// filenames mean a stale snapshot is never confused with a live one.
|
||||
@@ -608,4 +653,42 @@ mod tests {
|
||||
"cleanup removes the snapshot directory"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_rewrite_forces_file_rewrite_on_same_key() {
|
||||
let store = GraphSnapshotStore::new();
|
||||
let project = oak_node::project::Project::new();
|
||||
// First snapshot: the default project (working space ACEScg).
|
||||
let p1 = store.acquire(&project, 1).unwrap();
|
||||
let before = std::fs::read_to_string(&p1).unwrap();
|
||||
assert!(
|
||||
std::path::Path::new(&p1).exists(),
|
||||
"snapshot file written"
|
||||
);
|
||||
assert_eq!(store.refs(&p1), 1);
|
||||
|
||||
// A settings change with no revision bump: plain acquire must NOT
|
||||
// rewrite the file (the dedup the resync exists to bypass)...
|
||||
{
|
||||
let mut guard = project.lock().unwrap_or_else(|e| e.into_inner());
|
||||
guard.set_working_color_space(oak_common::colormath::WorkingColorSpace::SrgbLegacy);
|
||||
}
|
||||
let p2 = store.acquire(&project, 1).unwrap();
|
||||
assert_eq!(p1, p2, "same (uuid, revision) key reuses the file");
|
||||
assert_eq!(std::fs::read_to_string(&p1).unwrap(), before, "acquire never rewrites");
|
||||
store.release(&p2);
|
||||
|
||||
// ...while acquire_rewrite rewrites it in place at the same key.
|
||||
let p3 = store.acquire_rewrite(&project, 1).unwrap();
|
||||
assert_eq!(p1, p3, "rewrite keeps the same path token");
|
||||
let after = std::fs::read_to_string(&p1).unwrap();
|
||||
assert_ne!(after, before, "rewrite replaces the snapshot content");
|
||||
assert!(
|
||||
after.contains("srgb_legacy"),
|
||||
"rewritten snapshot carries the new setting: {after}"
|
||||
);
|
||||
store.release(&p3);
|
||||
store.release(&p1);
|
||||
store.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1047,8 +1047,40 @@ impl WorkerSession {
|
||||
})
|
||||
}
|
||||
|
||||
/// Refresh the process-global pipeline color settings from the loaded
|
||||
/// project snapshot (the source of truth for what this worker renders).
|
||||
/// Returns true when the settings changed — the caller must drop the
|
||||
/// frame cache, whose F32 bytes were produced under the old settings
|
||||
/// (M16 S2). The resync keeps the global current via `load_graph`; this
|
||||
/// covers tickets in flight before that IPC lands, and mirrors
|
||||
/// [`handle_load_graph`]'s adopt step.
|
||||
fn sync_pipeline_color_from_graph(&mut self) -> bool {
|
||||
let Some(graph) = &self.graph else { return false; };
|
||||
let Some(project) = &graph.project else { return false; };
|
||||
let (working, output) = {
|
||||
let guard = project.lock().unwrap_or_else(|e| e.into_inner());
|
||||
(guard.working_color_space(), guard.output_color_spec())
|
||||
};
|
||||
if oak_render::color::pipeline_working_space() == working
|
||||
&& oak_render::color::pipeline_output_spec() == output
|
||||
{
|
||||
return false;
|
||||
}
|
||||
oak_render::color::set_pipeline_color_settings(working, output);
|
||||
true
|
||||
}
|
||||
|
||||
/// Render `spec` into the slot's data block and fill the slot meta.
|
||||
fn render_spec_pixels(&mut self, spec: &BatchTicketSpec, pool: &FrameSlotPool) -> Result<(), String> {
|
||||
// Derive the pipeline colors from the loaded project on every render:
|
||||
// eval's decode linearization and the output node below read the
|
||||
// process global, which the resync (load_graph) keeps current — but a
|
||||
// ticket in flight before that IPC lands must not render under stale
|
||||
// colors. A change invalidates the F32 frame cache: cached bytes were
|
||||
// produced under the previous settings (M16 S2).
|
||||
if self.sync_pipeline_color_from_graph() {
|
||||
self.frame_cache.clear();
|
||||
}
|
||||
let w = spec.width;
|
||||
let h = spec.height;
|
||||
if w <= 0 || h <= 0 {
|
||||
@@ -1806,6 +1838,130 @@ mod tests {
|
||||
let _ = std::fs::remove_file(&bad);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_graph_adopts_pipeline_colors_from_snapshot() {
|
||||
use oak_common::colormath::{OutputColorSpec, WorkingColorSpace};
|
||||
// Reset the process global to something different from the snapshot's
|
||||
// settings so the adopt step is observable.
|
||||
oak_render::color::set_pipeline_color_settings(
|
||||
WorkingColorSpace::SrgbLegacy,
|
||||
OutputColorSpec::default(),
|
||||
);
|
||||
let project = oak_node::project::Project::new();
|
||||
{
|
||||
let mut guard = project.lock().unwrap_or_else(|e| e.into_inner());
|
||||
guard.set_working_color_space(WorkingColorSpace::AcesCg);
|
||||
}
|
||||
let xml = oak_node::serializer::save(&project.lock().unwrap_or_else(|e| e.into_inner()))
|
||||
.expect("serialize project");
|
||||
let path = std::env::temp_dir().join("oak_worker_main_test_acescg.ove");
|
||||
std::fs::write(&path, &xml).unwrap();
|
||||
|
||||
let mut s = WorkerSession::create("none").unwrap();
|
||||
let resp = s.handle_line(
|
||||
&json!({ "type": "load_graph", "path": path.display().to_string() }).to_string(),
|
||||
);
|
||||
assert!(resp.is_none(), "unexpected error: {resp:?}");
|
||||
assert_eq!(
|
||||
oak_render::color::pipeline_working_space(),
|
||||
WorkingColorSpace::AcesCg,
|
||||
"load_graph must adopt the snapshot's working space"
|
||||
);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
// Restore the default global so parallel tests are not disturbed.
|
||||
oak_render::color::set_pipeline_color_settings(
|
||||
WorkingColorSpace::default(),
|
||||
OutputColorSpec::default(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_pipeline_color_from_graph_restores_stale_global() {
|
||||
use oak_common::colormath::{OutputColorSpec, WorkingColorSpace};
|
||||
let project = oak_node::project::Project::new();
|
||||
{
|
||||
let mut guard = project.lock().unwrap_or_else(|e| e.into_inner());
|
||||
guard.set_working_color_space(WorkingColorSpace::AcesCg);
|
||||
}
|
||||
let xml = oak_node::serializer::save(&project.lock().unwrap_or_else(|e| e.into_inner()))
|
||||
.expect("serialize project");
|
||||
let path = std::env::temp_dir().join("oak_worker_main_test_sync.ove");
|
||||
std::fs::write(&path, &xml).unwrap();
|
||||
let mut s = WorkerSession::create("none").unwrap();
|
||||
let resp = s.handle_line(
|
||||
&json!({ "type": "load_graph", "path": path.display().to_string() }).to_string(),
|
||||
);
|
||||
assert!(resp.is_none(), "unexpected error: {resp:?}");
|
||||
|
||||
// Simulate the app committing new project settings: the process
|
||||
// global flips immediately, but the resync (load_graph re-broadcast)
|
||||
// reaches this worker later. A render ticket in that window must not
|
||||
// run under the stale colors.
|
||||
oak_render::color::set_pipeline_color_settings(
|
||||
WorkingColorSpace::SrgbLegacy,
|
||||
oak_render::color::pipeline_output_spec(),
|
||||
);
|
||||
assert!(
|
||||
s.sync_pipeline_color_from_graph(),
|
||||
"stale global must be refreshed from the loaded project"
|
||||
);
|
||||
assert_eq!(
|
||||
oak_render::color::pipeline_working_space(),
|
||||
WorkingColorSpace::AcesCg,
|
||||
"sync must restore the snapshot's working space"
|
||||
);
|
||||
assert!(
|
||||
!s.sync_pipeline_color_from_graph(),
|
||||
"no change means no frame-cache invalidation"
|
||||
);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
oak_render::color::set_pipeline_color_settings(
|
||||
WorkingColorSpace::default(),
|
||||
OutputColorSpec::default(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acescg_pipeline_round_trip_preserves_srgb_encoded_midgray() {
|
||||
// The ACEScg overexposure bug: F32 bytes produced by the legacy sRGB
|
||||
// pass-through (already sRGB-encoded) are re-encoded by the ACEScg
|
||||
// output transform, applying the sRGB OETF a second time — 0.5
|
||||
// becomes srgb_oetf(0.5) ≈ 0.735. The correct pipeline linearizes
|
||||
// the code (sRGB EOTF) into ACEScg and then encodes for output, so
|
||||
// the mid-gray comes back near 0.5.
|
||||
use oak_common::colormath::{
|
||||
acescg_to_output_bytes, decode_to_acescg_bytes, srgb_oetf, OutputColorSpec, OutputGamut,
|
||||
OutputTransfer, SourcePrimaries, SourceTransfer, WorkingColorSpace,
|
||||
};
|
||||
let spec = OutputColorSpec {
|
||||
gamut: OutputGamut::Srgb,
|
||||
transfer: OutputTransfer::Srgb,
|
||||
};
|
||||
|
||||
// Buggy path: treating the already-encoded code as ACEScg linear.
|
||||
let mut double_encoded = [0.5f32, 0.5, 0.5, 1.0];
|
||||
oak_common::colormath::working_to_display_target(
|
||||
&mut double_encoded,
|
||||
WorkingColorSpace::AcesCg,
|
||||
spec,
|
||||
);
|
||||
assert!(
|
||||
(double_encoded[0] - srgb_oetf(0.5)).abs() < 1e-4,
|
||||
"mis-encoding 0.5 as ACEScg-linear must yield srgb_oetf(0.5) ≈ 0.735, got {}",
|
||||
double_encoded[0]
|
||||
);
|
||||
|
||||
// Correct path: decode (sRGB EOTF) → ACEScg → output encode.
|
||||
let mut bytes = [0.5f32, 0.5, 0.5, 1.0].map(f32::to_le_bytes).concat();
|
||||
decode_to_acescg_bytes(&mut bytes, 1, SourcePrimaries::Bt709, SourceTransfer::SdrGamma);
|
||||
acescg_to_output_bytes(&mut bytes, 1, spec);
|
||||
let out = f32::from_le_bytes(bytes[0..4].try_into().unwrap());
|
||||
assert!(
|
||||
(out - 0.5).abs() < 1e-3,
|
||||
"correct pipeline must preserve mid-gray, got {out}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_frame_without_pool_reports_error_with_ticket() {
|
||||
let mut s = WorkerSession::create("none").unwrap();
|
||||
|
||||
Reference in New Issue
Block a user