// Oak Video Editor - Non-Linear Video Editor // Copyright (C) 2026 Oak Team // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . //! The real-time resampler/format converter (`olive::AudioProcessor`). //! //! Drives an in-process FFmpeg audio filter graph (abuffer → atempo chain → //! aformat → abuffersink) via ffmpeg-next; the C++ build went through the //! `fb_audio_graph_*`/`fb_frame_*` symbols of libffmpeg_bridge, which only //! existed to absorb FFmpeg API churn. The conversion output is always //! planar 32-bit float //! (`OAKAUDIO_PROCESSOR_OUTPUT_FORMAT == SampleFormat::F32Planar == 4`). use std::ptr; use std::sync::Mutex; use ffmpeg::format::sample::Type as SampleType; use ffmpeg::format::Sample; use ffmpeg::{ChannelLayout, Error as FfmpegError}; use ffmpeg_next as ffmpeg; use crate::error::{Error, Result}; use crate::params::{AudioParams, SampleFormat}; /// An audio processor; created closed, configured with /// [`open`](Processor::open). pub struct Processor { inner: Mutex, } /// Resampler state behind the handle's mutex. struct ProcessorInner { /// Live filter graph (`None` = closed). graph: Option, /// Scratch output frame reused for every pull. out_frame: ffmpeg::frame::Audio, /// Input spec recorded at `open`. from: AudioParams, /// Output spec recorded at `open`. to: AudioParams, } // SAFETY: the filter graph's raw pointers are only dereferenced through the // FFmpeg API while the processor's mutex is held, so all access is // serialized. unsafe impl Send for ProcessorInner {} impl Default for ProcessorInner { fn default() -> Self { ProcessorInner { graph: None, out_frame: ffmpeg::frame::Audio::empty(), from: AudioParams { sample_rate: 0, channel_layout: 0, format: SampleFormat::Invalid, }, to: AudioParams { sample_rate: 0, channel_layout: 0, format: SampleFormat::Invalid, }, } } } /// Map an oakcore [`SampleFormat`] to the equivalent ffmpeg [`Sample`]. /// Replaces `FFmpegUtils::get_ffmpeg_sample_format` crossing the oakcommon /// C ABI (`// CPP-PARITY: src/common/src/ffmpegutils.cpp:83`). fn to_ffmpeg_sample_format(fmt: SampleFormat) -> Sample { match fmt { SampleFormat::U8Planar => Sample::U8(SampleType::Planar), SampleFormat::S16Planar => Sample::I16(SampleType::Planar), SampleFormat::S32Planar => Sample::I32(SampleType::Planar), SampleFormat::S64Planar => Sample::I64(SampleType::Planar), SampleFormat::F32Planar => Sample::F32(SampleType::Planar), SampleFormat::F64Planar => Sample::F64(SampleType::Planar), SampleFormat::U8 => Sample::U8(SampleType::Packed), SampleFormat::S16 => Sample::I16(SampleType::Packed), SampleFormat::S32 => Sample::I32(SampleType::Packed), SampleFormat::S64 => Sample::I64(SampleType::Packed), SampleFormat::F32 => Sample::F32(SampleType::Packed), SampleFormat::F64 => Sample::F64(SampleType::Packed), SampleFormat::Invalid => Sample::None, } } /// Rebuild an ffmpeg [`ChannelLayout`] from a channel mask (0 = unknown → /// stereo fallback). Same construction as oakcodec's /// `channel_layout_from_mask`. fn channel_layout_from_mask(mask: u64) -> ChannelLayout { if mask == 0 { return ChannelLayout::default(2); } let channels = mask.count_ones() as i32; ChannelLayout(ffmpeg::ffi::AVChannelLayout { order: ffmpeg::ffi::AVChannelOrder::AV_CHANNEL_ORDER_NATIVE, nb_channels: channels, u: ffmpeg::ffi::AVChannelLayout__bindgen_ty_1 { mask }, opaque: ptr::null_mut(), }) } /// `// CPP-PARITY: src/audio/src/audioprocessor.cpp:50` — ensure a usable /// channel layout mask: 0 (unknown) falls back to a default layout derived /// from the channel count, itself defaulting to stereo. fn fix_channel_layout(params: AudioParams) -> AudioParams { let mut result = params; if params.channel_layout == 0 { let mut channels = params.channel_count(); if channels <= 0 { channels = 2; } result.channel_layout = ChannelLayout::default(channels).bits(); } result } /// Build the conversion graph: abuffer → atempo chain → aformat (fltp at the /// output rate/layout) → abuffersink. `atempo` accepts factors in /// [0.5, 100], so out-of-range tempos are chained /// (`// CPP-PARITY: ffmpeg_bridge.cpp` `fb_audio_graph_create`). fn build_graph(from: &AudioParams, to: &AudioParams, speed: f64) -> Result { let in_format = to_ffmpeg_sample_format(from.format); if in_format == Sample::None { return Err(Box::new(Error::Failed("invalid input sample format".to_string()))); } let abuffer = ffmpeg::filter::find("abuffer") .ok_or_else(|| Error::Failed("abuffer filter not found".to_string()))?; let abuffersink = ffmpeg::filter::find("abuffersink") .ok_or_else(|| Error::Failed("abuffersink filter not found".to_string()))?; let mut graph = ffmpeg::filter::Graph::new(); let in_args = format!( "time_base=1/{rate}:sample_rate={rate}:sample_fmt={fmt}:channel_layout=0x{layout:x}", rate = from.sample_rate, fmt = in_format.name(), layout = from.channel_layout, ); graph .add(&abuffer, "in", &in_args) .map_err(|e| Error::Failed(format!("failed to add abuffer: {e}")))?; graph .add(&abuffersink, "out", "") .map_err(|e| Error::Failed(format!("failed to add abuffersink: {e}")))?; // Chain atempo for out-of-range factors, then force the output format // (planar f32 at the requested rate/layout) with aformat. let mut spec = String::new(); let mut tempo = speed; while tempo > 100.0 { spec.push_str("atempo=100.0,"); tempo /= 100.0; } while tempo < 0.5 { spec.push_str("atempo=0.5,"); tempo /= 0.5; } if tempo != 1.0 { spec.push_str(&format!("atempo={tempo},")); } spec.push_str(&format!( "aformat=sample_fmts=fltp:sample_rates={}:channel_layouts=0x{:x}", to.sample_rate, to.channel_layout, )); graph .output("in", 0) .and_then(|p| p.input("out", 0)) .and_then(|p| p.parse(&spec)) .map_err(|e| Error::Failed(format!("failed to parse filter spec: {e}")))?; graph .validate() .map_err(|e| Error::Failed(format!("failed to validate filter graph: {e}")))?; Ok(graph) } /// Whether a pull error just means "no output available right now" (needs /// more input, or the drained end after a flush). fn is_drain(e: &FfmpegError) -> bool { matches!(e, FfmpegError::Eof) || matches!(e, FfmpegError::Other { errno } if *errno == ffmpeg::error::EAGAIN) } impl Processor { /// Create a closed processor. pub fn init() -> Processor { Processor { inner: Mutex::new(ProcessorInner::default()), } } /// Open the resampling/format-conversion graph. `out_format` is accepted /// for interface completeness but the conversion output is always planar /// f32. /// /// `// CPP-PARITY: src/audio/c_api/processor.cpp:43` (validation order: /// already-open state, invalid rates/speed, forced output format) and /// `src/audio/src/audioprocessor.cpp:82` (graph creation). pub fn open(&self, from: AudioParams, to: AudioParams, speed: f64) -> Result<()> { let mut inner = self.inner.lock().unwrap(); if inner.graph.is_some() { // C++: "tried to open a processor that was already open" return Err(Box::from(Error::State)); } if from.sample_rate <= 0 || to.sample_rate <= 0 || speed <= 0.0 { return Err(Box::from(Error::Invalid)); } // The C ABI delivers planar float output only; force the output format // stage to f32p (OAKAUDIO_PROCESSOR_OUTPUT_FORMAT == 4). if to.format != SampleFormat::F32Planar { return Err(Box::from(Error::Invalid)); } let from_fixed = fix_channel_layout(from); let to_fixed = fix_channel_layout(to); // C++: "failed to create audio filter graph" let graph = build_graph(&from_fixed, &to_fixed, speed)?; inner.graph = Some(graph); inner.out_frame = ffmpeg::frame::Audio::empty(); inner.from = from_fixed; inner.to = to_fixed; Ok(()) } /// Close the graph (safe when closed). pub fn close(&self) -> Result<()> { let mut inner = self.inner.lock().unwrap(); inner.graph = None; inner.out_frame = ffmpeg::frame::Audio::empty(); Ok(()) } /// 1 when open, 0 when closed. pub fn is_open(&self) -> Result { let inner = self.inner.lock().unwrap(); Ok(inner.graph.is_some()) } /// Push planar float input and pull converted output. Returns the number /// of output frames written. /// /// `// CPP-PARITY: src/audio/c_api/processor.cpp:91` (validation, state /// check, null `out_planar` short-circuit) and /// `src/audio/src/audioprocessor.cpp:141` (push/pull loop, byte counting). pub fn convert( &self, in_planar: *const *const f32, in_frame_count: i32, out_planar: *const *mut f32, out_capacity_frames: i32, ) -> Result { let mut guard = self.inner.lock().unwrap(); let inner = &mut *guard; if inner.graph.is_none() { return Err(Box::from(Error::State)); } if in_frame_count < 0 || out_capacity_frames < 0 || (in_frame_count > 0 && in_planar.is_null()) { return Err(Box::from(Error::Invalid)); } let channels = inner.to.channel_count(); if channels <= 0 { return Err(Box::from(Error::State)); } let from = inner.from; let graph = inner.graph.as_mut().unwrap(); let out_frame = &mut inner.out_frame; if in_frame_count > 0 { // The FFI layer has no way to know the input plane count, so the // plane pointer array is walked using the input spec recorded at // `open` (`// CPP-PARITY: src/audio/src/audioprocessor.cpp:141`). let nb = in_frame_count as usize; let in_channels = from.channel_count().max(0) as usize; let layout = channel_layout_from_mask(from.channel_layout); let mut frame = ffmpeg::frame::Audio::new(to_ffmpeg_sample_format(from.format), nb, layout); frame.set_rate(from.sample_rate as u32); let planar = from.format.is_planar(); // `plane_mut::` requires the exact sample type of the frame // format, so the copy dispatches on the recorded input format. macro_rules! fill { ($t:ty) => {{ if planar { for ch in 0..in_channels { // SAFETY: `in_planar` is non-null here and the FFI // contract guarantees at least `from.channel_count()` // entries, each pointing at `nb` samples of the // recorded input format. let src = unsafe { *in_planar.add(ch) } as *const $t; let dst = frame.plane_mut::<$t>(ch); unsafe { ptr::copy_nonoverlapping(src, dst.as_mut_ptr(), nb) }; } } else { // Packed input: a single plane at `in_planar[0]`. // SAFETY: see above; the plane holds `nb * channels` // samples. let src = unsafe { *in_planar } as *const $t; let dst = frame.plane_mut::<$t>(0); unsafe { ptr::copy_nonoverlapping(src, dst.as_mut_ptr(), nb * in_channels) }; } }}; } match from.format { SampleFormat::U8Planar | SampleFormat::U8 => fill!(u8), SampleFormat::S16Planar | SampleFormat::S16 => fill!(i16), SampleFormat::S32Planar | SampleFormat::S32 => fill!(i32), SampleFormat::S64Planar | SampleFormat::S64 => { // ffmpeg-next's typed plane API has no `i64` impl; copy the // 8-byte samples through the raw plane pointers. if planar { for ch in 0..in_channels { // SAFETY: same contract as above; the plane is // `nb * 8` bytes. let src = unsafe { *in_planar.add(ch) } as *const u8; let dst = unsafe { *(*frame.as_mut_ptr()).extended_data.add(ch) }; unsafe { ptr::copy_nonoverlapping(src, dst, nb * 8) }; } } else { // SAFETY: same contract as above; the plane is // `nb * channels * 8` bytes. let src = unsafe { *in_planar } as *const u8; let dst = unsafe { *(*frame.as_mut_ptr()).extended_data }; unsafe { ptr::copy_nonoverlapping(src, dst, nb * in_channels * 8) }; } } SampleFormat::F32Planar | SampleFormat::F32 => fill!(f32), SampleFormat::F64Planar | SampleFormat::F64 => fill!(f64), SampleFormat::Invalid => return Err(Box::new(Error::State)), } if let Err(e) = graph.get("in").unwrap().source().add(&frame) { return Err(Box::new(Error::Failed(format!( "failed to add frame to buffersrc: {e}" )))); } } // C++: `out_planar ? &buf : nullptr` — with no destination, the input is // pushed but nothing is pulled. if out_planar.is_null() { return Ok(0); } let mut total: i64 = 0; loop { let pulled = graph.get("out").unwrap().sink().frame(out_frame); match pulled { Ok(()) => {} Err(e) if is_drain(&e) => break, Err(e) => { return Err(Box::from(Error::Failed(format!( "failed to pull from buffersink: {e}" )))) } } let nb = out_frame.samples() as i32; if nb > 0 && total < i64::from(out_capacity_frames) { let to_copy = (i64::from(out_capacity_frames) - total).min(i64::from(nb)) as i32; for ch in 0..channels { // SAFETY: the FFI contract guarantees at least `channels` // entries in `out_planar` (NULL entries are skipped). let dst = unsafe { *out_planar.add(ch as usize) }; if dst.is_null() { continue; } // Output is planar f32 (enforced by open()); each plane is // `to_copy` float samples. let src = out_frame.plane::(ch as usize); unsafe { ptr::copy_nonoverlapping(src.as_ptr(), dst, to_copy as usize); } } } total += i64::from(nb); } Ok(total.min(i64::from(out_capacity_frames)) as i32) } /// Signal end-of-input to the graph (flushes internal delay). /// /// `// CPP-PARITY: src/audio/c_api/processor.cpp:137` (state check) /// and `src/audio/src/audioprocessor.cpp:210` (flush has no failure path; a /// negative push return is logged only). pub fn flush(&self) -> Result<()> { let mut inner = self.inner.lock().unwrap(); let Some(graph) = inner.graph.as_mut() else { return Err(Box::from(Error::State)); }; let _ = graph.get("in").unwrap().source().flush(); Ok(()) } } /// Format of the conversion output (always planar f32). pub const OUTPUT_FORMAT: SampleFormat = SampleFormat::F32Planar;