diff --git a/crates/oak-app/tests/tone48k.mp4 b/crates/oak-app/tests/tone48k.mp4 new file mode 100644 index 000000000..bd7e5c4e4 Binary files /dev/null and b/crates/oak-app/tests/tone48k.mp4 differ diff --git a/crates/oak-codec/src/ffmpeg.rs b/crates/oak-codec/src/ffmpeg.rs index d99c18f95..66c182eba 100644 --- a/crates/oak-codec/src/ffmpeg.rs +++ b/crates/oak-codec/src/ffmpeg.rs @@ -544,6 +544,19 @@ struct AudioDecodeState { /// continue without re-seeking — skipping the per-chunk decoder flush /// and resampler reset that cause boundary artifacts (pops/clicks). contiguous_end_sample: Option, + /// Tail of the last decoded codec frame that fell PAST the previous + /// chunk's end, interleaved in the chunk's destination format; + /// `carry_start` is the output-sample index `carry[0]` belongs to and + /// `carry_format` the `(sample_rate, channel_layout)` it was rendered + /// for. The decoder consumes whole codec frames, so without this + /// carry the overflow (up to one codec frame — 1024 samples for AAC) + /// would be lost and every chunk whose end does not align with the + /// codec frame grid would start with a hole (a periodic stutter). + /// Served at the start of the next contiguous chunk; cleared on + /// seek, on format change, and on chunk failure. + carry: Vec, + carry_start: i64, + carry_format: (u32, u64), } /// A swresample conversion context. @@ -665,6 +678,9 @@ impl DecoderState { Some(AudioDecodeState { resampler: None, contiguous_end_sample: None, + carry: Vec::new(), + carry_start: 0, + carry_format: (0, 0), }) } else { None @@ -816,9 +832,15 @@ impl DecoderState { } self.eof = false; // Any seek breaks audio continuity (decoder flush + resampler - // reset): the next audio chunk must not skip its own seek. + // reset): the next audio chunk must not skip its own seek, the + // carried overflow belongs to the pre-seek position, and the + // cached resampler's internal state is stale — drop it so a + // post-seek decode is identical to a fresh one (reusing it + // measurably shifted the output). if let Some(a) = &mut self.audio { a.contiguous_end_sample = None; + a.carry.clear(); + a.resampler = None; } Ok(()) } @@ -1193,6 +1215,30 @@ impl DecoderState { self.seek(start_ts)?; } + // Serve the overflow tail carried over from the previous chunk: + // it belongs exactly at this chunk's start (the anchored grid + // makes positions exact). A stale or format-mismatched carry is + // discarded (a pre-seek carry was already cleared by seek()). + let mut carried_frames: i64 = 0; + { + let a = self.audio.as_mut().expect("audio session"); + let valid = contiguous + && a.carry_format == (sample_rate as u32, channel_layout) + && a.carry_start == start_sample; + if valid && !a.carry.is_empty() { + let dest_frames = dest.len() / dst_channels; + let n_frames = (a.carry.len() / dst_channels).min(dest_frames); + dest[..n_frames * dst_channels].copy_from_slice(&a.carry[..n_frames * dst_channels]); + carried_frames = n_frames as i64; + // A chunk shorter than the carry (degenerate grid) keeps + // the remainder for the next chunk. + a.carry.drain(..n_frames * dst_channels); + a.carry_start += n_frames as i64; + } else { + a.carry.clear(); + } + } + // The fill+decode+flush body runs in a closure so the success path // can record the chunk's end sample (continuity) and an error path // can clear it (the decoder/resampler position is unknown after a @@ -1216,8 +1262,18 @@ impl DecoderState { }; let stream_time_base = self.stream_time_base; - let mut next_sample: Option = None; + let mut next_sample: Option = if carried_frames > 0 { + Some(start_sample + carried_frames) + } else { + None + }; let dest_frames = (dest.len() / dst_channels) as i64; + // Overflow past this chunk's end (see AudioDecodeState::carry): + // the decoder consumes whole codec frames, so the tail of the + // last frame crossing the chunk end would otherwise be lost + // and the next chunk would start with a hole. + let mut new_carry: Vec = Vec::new(); + let mut new_carry_start: i64 = start_sample + dest_frames; while let Some(frame) = self.next_frame()? { let DecodedFrame::Audio(audio) = frame else { @@ -1251,6 +1307,22 @@ impl DecoderState { } } } + if offset + chunk_samples > dest_frames { + let inside = (dest_frames - offset).clamp(0, chunk_samples) as usize; + let pos = frame_start + inside as i64; + if !new_carry.is_empty() + && pos != new_carry_start + (new_carry.len() / dst_channels) as i64 + { + // Non-adjacent overflow (should not happen): keep + // only the latest contiguous run. + new_carry.clear(); + new_carry_start = pos; + } + if new_carry.is_empty() { + new_carry_start = pos; + } + new_carry.extend_from_slice(&converted[inside * dst_channels..]); + } next_sample = Some(frame_start + chunk_samples); if offset + chunk_samples >= dest_frames { break; @@ -1289,13 +1361,33 @@ impl DecoderState { } } } + if offset + chunk_samples > dest_frames { + let inside = (dest_frames - offset).clamp(0, chunk_samples) as usize; + let pos = frame_start + inside as i64; + if !new_carry.is_empty() + && pos != new_carry_start + (new_carry.len() / dst_channels) as i64 + { + new_carry.clear(); + new_carry_start = pos; + } + if new_carry.is_empty() { + new_carry_start = pos; + } + new_carry.extend_from_slice(&flush[inside * dst_channels..]); + } } } // Record the end sample: the next chunk starting exactly here - // continues without a seek. - self.audio.as_mut().expect("audio session").contiguous_end_sample = - Some(start_sample + dest_frames); + // continues without a seek — and is served the carried overflow + // first (see the chunk entry above). + { + let a = self.audio.as_mut().expect("audio session"); + a.carry = new_carry; + a.carry_start = new_carry_start; + a.carry_format = (sample_rate as u32, channel_layout); + a.contiguous_end_sample = Some(start_sample + dest_frames); + } Ok(RetrieveAudioStatus::Success) })(); @@ -1307,6 +1399,7 @@ impl DecoderState { // unknown: force a fresh seek on the next chunk. if let Some(a) = &mut self.audio { a.contiguous_end_sample = None; + a.carry.clear(); } Err(e) } diff --git a/crates/oak-codec/src/realmedia_tests.rs b/crates/oak-codec/src/realmedia_tests.rs index e9c8e76a0..a7400b5eb 100644 --- a/crates/oak-codec/src/realmedia_tests.rs +++ b/crates/oak-codec/src/realmedia_tests.rs @@ -258,6 +258,86 @@ fn contiguous_audio_chunks_skip_seek_and_match_oneshot() { ); } +/// Playback-shaped chunks (one 25 fps frame = 1920 samples at 48 kHz) tile +/// the AAC 1024-sample grid badly: the codec frame crossing each chunk end +/// leaves a 128..896-sample tail past the chunk, and because the decoder +/// has already consumed that frame the next chunk used to start with a +/// hole of exactly that size (7 of 8 chunks — the periodic stutter). The +/// overflow carry (`AudioDecodeState::carry`) must make chunked decoding +/// sample-exact against a one-shot decode of the same span. The fixture is +/// a continuous 440 Hz tone (demo.mp4's audio is digital silence and +/// cannot expose the holes). +#[test] +fn playback_sized_chunks_match_oneshot_sample_exact() { + let d = FFmpegDecoder::new(); + let tone = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oak-app/tests/tone48k.mp4"); + let s = CodecStream::with_block(tone.to_string_lossy().into_owned(), 0, None); + d.open(&s).expect("open tone audio stream"); + + const CHUNK: usize = 1920; // one 25 fps frame at 48 kHz + const CHUNKS: usize = 175; // 7 seconds (the tone is 8 s) + let mut chunked = Vec::with_capacity(CHUNK * CHUNKS * 2); + for i in 0..CHUNKS as i64 { + let mut dest = vec![0f32; CHUNK * 2]; + d.retrieve_audio( + &mut dest, + &TimeRange::new(Rational::new(i, 25), Rational::new(i + 1, 25)), + 48000, + 0x3, + ) + .expect("playback-sized chunk"); + chunked.extend_from_slice(&dest); + } + + // Guard against a vacuous pass on silent media: the span must + // contain real content. + let peak = chunked.iter().fold(0.0f32, |a, &v| a.max(v.abs())); + assert!(peak > 0.05, "tone span is unexpectedly quiet"); + + // A separate fresh decoder: both paths decode the span straight from + // a seek to 0, so the only tolerated difference is the chunking + // itself (none — the carry makes them sample-exact). NOTE: decoding + // on the SAME decoder after a seek is NOT bit-identical to a fresh + // decode (libavcodec/demuxer priming state survives avcodec_flush; + // ~1e-2 wobble, inaudible) — a separate pre-existing quirk, not what + // this test guards. + let d2 = FFmpegDecoder::new(); + let s2 = CodecStream::with_block(tone.to_string_lossy().into_owned(), 0, None); + d2.open(&s2).expect("open tone audio stream (oneshot)"); + let mut oneshot = vec![0f32; CHUNK * CHUNKS * 2]; + d2.retrieve_audio( + &mut oneshot, + &TimeRange::new(Rational::new(0, 1), Rational::new(7, 1)), + 48000, + 0x3, + ) + .expect("oneshot [0,7)"); + + let mut max_diff = 0.0f32; + let mut worst: Vec<(usize, f32)> = Vec::new(); + for (i, (a, b)) in chunked.iter().zip(oneshot.iter()).enumerate() { + let d = (a - b).abs(); + max_diff = max_diff.max(d); + if d > 1e-6 { + worst.push((i, d)); + } + } + if !worst.is_empty() { + worst.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); + let positions: Vec = worst + .iter() + .take(20) + .map(|(i, d)| format!("{}(chunk {},+{},{:.4})", i, i / (CHUNK * 2), (i / 2) % CHUNK, d)) + .collect(); + eprintln!("diffs>1e-6: {} total; worst: {}", worst.len(), positions.join(" ")); + } + assert!( + max_diff < 1e-6, + "playback-sized chunks diverge from one-shot decode (max diff {max_diff}); \ + the chunk-end overflow carry is dropping samples" + ); +} + #[test] fn encode_h264_roundtrip_to_tmp() { let out = std::env::temp_dir().join(format!("oakcodec_roundtrip_{}.mp4", std::process::id())); diff --git a/crates/oak-render/examples/render_audio_wav.rs b/crates/oak-render/examples/render_audio_wav.rs new file mode 100644 index 000000000..1549363c4 --- /dev/null +++ b/crates/oak-render/examples/render_audio_wav.rs @@ -0,0 +1,92 @@ +// Temporary diagnostic: render a media file's audio exactly like the +// playback prefetch does (one sequence frame per chunk through +// `eval::render_audio_samples`) and write the result as a PCM s16 WAV, +// so render-side artifacts can be analyzed offline. +// +// Usage: render_audio_wav [fps] + +use oak_core::{Rational, TimeRange}; +use oak_render::eval::render_audio_samples; +use oak_render::ticket::{AudioTicketParams, MontageClip, TicketPayload}; + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() < 6 { + eprintln!("usage: render_audio_wav [fps]"); + std::process::exit(64); + } + let media = &args[1]; + let stream_index: i32 = args[2].parse().unwrap(); + let start_sec: f64 = args[3].parse().unwrap(); + let end_sec: f64 = args[4].parse().unwrap(); + let out_path = &args[5]; + let fps: i64 = if args.len() > 6 { args[6].parse().unwrap() } else { 25 }; + + let sample_rate = 48000i32; + let channel_layout = 0x3u64; // stereo + let channels = 2usize; + + let start_frame = (start_sec * fps as f64).round() as i64; + let end_frame = (end_sec * fps as f64).round() as i64; + + let montage = vec![MontageClip { + filename: media.clone(), + stream_index, + in_time: Rational::new(start_frame, fps), + out_time: Rational::new(end_frame, fps), + media_in: Rational::new(0, 1), + gain: 1.0, + effects: Vec::new(), + }]; + + let mut pcm: Vec = Vec::new(); + for f in start_frame..end_frame { + let range = TimeRange::new(Rational::new(f, fps), Rational::new(f + 1, fps)); + let params = AudioTicketParams { + viewer: 1, + range, + sample_rate, + channel_layout, + montage: montage.clone(), + }; + match render_audio_samples(¶ms) { + Ok(TicketPayload::Audio(samples)) => { + for s in &samples.samples { + pcm.push((s.clamp(-1.0, 1.0) * 32767.0) as i16); + } + } + other => { + eprintln!("frame {f}: render failed: {:?}", other.is_err()); + let frames = ((sample_rate as f64) / fps as f64).round() as usize; + pcm.extend(std::iter::repeat(0i16).take(frames * channels)); + } + } + } + + // Minimal 16-bit PCM WAV. + let data_len = (pcm.len() * 2) as u32; + let mut w = Vec::with_capacity(44 + data_len as usize); + w.extend_from_slice(b"RIFF"); + w.extend_from_slice(&(36 + data_len).to_le_bytes()); + w.extend_from_slice(b"WAVEfmt "); + w.extend_from_slice(&16u32.to_le_bytes()); + w.extend_from_slice(&1u16.to_le_bytes()); // PCM + w.extend_from_slice(&(channels as u16).to_le_bytes()); + w.extend_from_slice(&(sample_rate as u32).to_le_bytes()); + w.extend_from_slice(&(sample_rate as u32 * channels as u32 * 2).to_le_bytes()); + w.extend_from_slice(&(channels as u16 * 2).to_le_bytes()); + w.extend_from_slice(&16u16.to_le_bytes()); + w.extend_from_slice(b"data"); + w.extend_from_slice(&data_len.to_le_bytes()); + for s in &pcm { + w.extend_from_slice(&s.to_le_bytes()); + } + std::fs::write(out_path, &w).expect("write wav"); + println!( + "wrote {} samples ({} ch @ {} Hz) to {}", + pcm.len() / channels, + channels, + sample_rate, + out_path + ); +}