audio: master-clock playback timing, output clock compensation, buffer config, interpolated speed

- playback timer uses the audio output device as its master clock: the
  PortAudio callback counts consumed frames (including underrun
  zero-fill) so video cannot drift away from what is heard; wall clock
  remains as fallback when no clocked output is running
- output clock compensates for device output latency; new Preferences >
  Audio buffer size setting (0 = auto)
- SampleBuffer::speed() now uses linear interpolation instead of
  nearest-neighbor sampling
- regression tests: audio-clock driven timer (fwd/rev/speed), wall
  clock fallback, interpolation correctness
This commit is contained in:
2026-07-19 21:44:34 +08:00
parent 0c02ff0d77
commit a7ddc0f114
16 changed files with 266 additions and 8 deletions
+32 -2
View File
@@ -71,6 +71,10 @@ int output_callback(const void *input, void *output, unsigned long frame_count,
max_read - read_count);
}
// Count all frames leaving the device (including zero-filled underrun
// frames) so this can serve as the playback master clock
device->add_output_frames(frame_count);
return paContinue;
}
@@ -106,10 +110,14 @@ bool AudioManager::push_to_output(const AudioParams &params,
PaStreamParameters p = get_port_audio_params(params, output_device_);
// 0 = let PortAudio choose the buffer size
const unsigned long frames_per_buffer =
OAK_CONFIG("AudioOutputBufferSize").toUInt();
PaError r = Pa_OpenStream(&output_stream_, nullptr, &p,
output_params_.sample_rate(),
paFramesPerBufferUnspecified, paNoFlag,
output_callback, output_buffer_);
frames_per_buffer, paNoFlag, output_callback,
output_buffer_);
if (r != paNoError) {
// Unhandled error
//qCritical() << "Failed to open output stream:" << Pa_GetErrorText(r);
@@ -141,6 +149,28 @@ void AudioManager::clear_buffered_output()
output_buffer_->clear();
}
double AudioManager::seconds() const
{
if (!output_stream_ || !Pa_IsStreamActive(output_stream_)) {
return -1.0;
}
double seconds = double(output_buffer_->output_frames_consumed()) /
double(output_params_.sample_rate());
// Compensate for output latency so the clock reflects what is audible
if (const PaStreamInfo *info = Pa_GetStreamInfo(output_stream_)) {
seconds -= info->outputLatency;
}
return qMax(0.0, seconds);
}
void AudioManager::reset_output_clock()
{
output_buffer_->reset_output_frames();
}
PaSampleFormat AudioManager::get_port_audio_sample_format(SampleFormat fmt)
{
switch (fmt) {
+15 -1
View File
@@ -30,6 +30,7 @@
#include "audiovisualwaveform.h"
#include "audio/audioprocessor.h"
#include "common/define.h"
#include "common/playbackaudioclock.h"
#include "codec/ffmpeg/ffmpegencoder.h"
#include "render/audioplaybackcache.h"
#include "render/previewaudiodevice.h"
@@ -43,7 +44,7 @@ namespace olive
* Wraps around a QAudioOutput and AudioHybridDevice, connecting them together and exposing audio functionality to
* the rest of the system.
*/
class AudioManager : public QObject {
class AudioManager : public QObject, public PlaybackAudioClock {
Q_OBJECT
public:
static void create_instance();
@@ -60,6 +61,19 @@ public:
void stop_output();
/**
* @brief Seconds of audio consumed by the output device since the last reset
*
* Compensated for output latency so it represents what is actually
* audible. Returns a negative value when no output stream is running.
*/
virtual double seconds() const override;
/**
* @brief Restarts the output clock at zero for a new playback run
*/
void reset_output_clock();
PaDeviceIndex get_output_device() const
{
return output_device_;
+1
View File
@@ -43,6 +43,7 @@ target_sources(libolive-editor PRIVATE
oiioutils.cpp
oiioutils.h
otioutils.h
playbackaudioclock.h
qtutils.cpp
qtutils.h
range.h
+45
View File
@@ -0,0 +1,45 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef OAK_PLAYBACKAUDIOCLOCK_H
#define OAK_PLAYBACKAUDIOCLOCK_H
namespace olive
{
/**
* @brief Source of an audio output clock for playback timing
*/
class PlaybackAudioClock {
public:
virtual ~PlaybackAudioClock() = default;
/**
* @brief Seconds of audio consumed by the output device
*
* Must return a negative value when no clocked output is running, in
* which case the caller should fall back to the wall clock.
*/
virtual double seconds() const = 0;
};
}
#endif // OAK_PLAYBACKAUDIOCLOCK_H
+4
View File
@@ -210,6 +210,10 @@ void Config::set_defaults()
QStringLiteral("AudioOutputSampleFormat"), NodeValue::k_text,
QString::fromStdString(SampleFormat(SampleFormat::s16).to_string()));
// Output buffer size in frames, 0 = let PortAudio/the device decide
set_entry_internal(QStringLiteral("AudioOutputBufferSize"), NodeValue::k_int,
0);
set_entry_internal(QStringLiteral("AudioRecordingFormat"), NodeValue::k_int,
ExportFormat::k_format_wav);
set_entry_internal(QStringLiteral("AudioRecordingCodec"), NodeValue::k_int,
@@ -124,6 +124,20 @@ PreferencesAudioTab::PreferencesAudioTab()
.toStdString()));
output_param_layout->addWidget(output_fmt_combo_, output_row,
1);
output_row++;
output_param_layout->addWidget(
new QLabel(tr("Buffer Size:")), output_row, 0);
output_buffer_size_ = new QSpinBox();
output_buffer_size_->setRange(0, 65536);
output_buffer_size_->setSpecialValueText(tr("Auto"));
output_buffer_size_->setSuffix(tr(" frames"));
output_buffer_size_->setValue(
OAK_CONFIG("AudioOutputBufferSize").toInt());
output_param_layout->addWidget(output_buffer_size_, output_row,
1);
}
}
@@ -222,6 +236,7 @@ void PreferencesAudioTab::accept(MultiUndoCommand *command)
QVariant::fromValue(output_ch_layout_combo_->get_channel_layout());
OAK_CONFIG("AudioOutputSampleFormat") = QString::fromStdString(
output_fmt_combo_->get_sample_format().to_string());
OAK_CONFIG("AudioOutputBufferSize") = output_buffer_size_->value();
OAK_CONFIG("AudioRecordingFormat") = record_format_combo_->get_format();
OAK_CONFIG("AudioRecordingCodec") = record_options_->get_codec();
@@ -25,6 +25,7 @@
#include <QComboBox>
#include <QPushButton>
#include <QCheckBox>
#include <QSpinBox>
#include "dialog/configbase/configdialogbase.h"
#include "dialog/export/exportaudiotab.h"
@@ -68,6 +69,11 @@ private:
ChannelLayoutComboBox *output_ch_layout_combo_;
SampleFormatComboBox *output_fmt_combo_;
/**
* @brief UI widget for the output buffer size in frames (0 = auto)
*/
QSpinBox *output_buffer_size_;
ExportFormatComboBox *record_format_combo_;
ExportAudioTab *record_options_;
+1
View File
@@ -87,6 +87,7 @@ void PreviewAudioDevice::clear()
buffer_.clear();
bytes_read_ = 0;
output_frames_consumed_.store(0);
}
}
+25
View File
@@ -24,6 +24,8 @@
#include <olive/core/render/audioparams.h>
#include <atomic>
#include "previewautocacher.h"
namespace olive
@@ -66,6 +68,27 @@ public:
void clear();
/**
* @brief Frames consumed by the audio output callback
*
* Counted in the callback itself so underrun (zero-filled) frames are
* included, making the value usable as a playback clock.
*/
void add_output_frames(qint64 frame_count)
{
output_frames_consumed_.fetch_add(frame_count);
}
qint64 output_frames_consumed() const
{
return output_frames_consumed_.load();
}
void reset_output_frames()
{
output_frames_consumed_.store(0);
}
signals:
void notify();
@@ -79,6 +102,8 @@ private:
qint64 notify_interval_;
qint64 bytes_read_;
std::atomic<qint64> output_frames_consumed_{0};
};
}
+4
View File
@@ -1384,6 +1384,10 @@ void ViewerWidget::finish_play_preprocess()
int64_t playback_start_time = get_timestamp();
// Restart the audio output clock for this playback run; the playback
// timer uses it as its master clock
AudioManager::instance()->reset_output_clock();
// Start audio waveform playback
if (!prequeued_audio_.isEmpty()) {
QString error;
+3 -1
View File
@@ -35,6 +35,7 @@
#include <QScreen>
#include <QTextEdit>
#include "audio/audiomanager.h"
#include "common/define.h"
#include "common/html.h"
#include "common/qtutils.h"
@@ -1632,7 +1633,8 @@ void ViewerDisplayWidget::play(const int64_t &start_timestamp,
playback_timebase_ = timebase;
playback_speed_ = playback_speed;
timer_.start(start_timestamp, playback_speed, timebase.to_double());
timer_.start(start_timestamp, playback_speed, timebase.to_double(),
AudioManager::instance());
if (start_updating) {
connect(this, &ViewerDisplayWidget::frame_swapped, this,
+21 -1
View File
@@ -28,16 +28,36 @@ namespace olive
void ViewerPlaybackTimer::start(const int64_t &start_timestamp,
const int &playback_speed,
const double &timebase)
const double &timebase,
const PlaybackAudioClock *audio_clock)
{
timer_.start();
start_timestamp_ = start_timestamp;
playback_speed_ = playback_speed;
timebase_ = timebase * 1000;
audio_clock_ = audio_clock;
}
int64_t ViewerPlaybackTimer::get_timestamp_now() const
{
// The audio output clock is the master clock when available: sound card
// consumption is what the viewer must stay in sync with, and unlike the
// wall clock it cannot drift away from what is actually heard
if (audio_clock_) {
const double audio_seconds = audio_clock_->seconds();
if (audio_seconds >= 0.0) {
// At speeds other than 1x the audio tempo is scaled, so one
// output second corresponds to |speed| timeline seconds. The
// result is already in timeline frames, so it is applied
// signed rather than multiplied by the speed again.
const double timeline_ms =
audio_seconds * 1000.0 * qAbs(playback_speed_);
const int64_t frames_since_start = qFloor(timeline_ms / timebase_);
return start_timestamp_ +
frames_since_start * (playback_speed_ < 0 ? -1 : 1);
}
}
int64_t real_time = timer_.elapsed();
int64_t frames_since_start =
+5 -1
View File
@@ -26,6 +26,7 @@
#include <QElapsedTimer>
#include "common/define.h"
#include "common/playbackaudioclock.h"
namespace olive
{
@@ -33,7 +34,8 @@ namespace olive
class ViewerPlaybackTimer {
public:
void start(const int64_t &start_timestamp, const int &playback_speed,
const double &timebase);
const double &timebase,
const PlaybackAudioClock *audio_clock = nullptr);
int64_t get_timestamp_now() const;
@@ -44,6 +46,8 @@ private:
int playback_speed_;
double timebase_;
const PlaybackAudioClock *audio_clock_ = nullptr;
};
}
+11 -2
View File
@@ -149,6 +149,8 @@ void SampleBuffer::speed(double speed)
return;
}
const size_t input_sample_count = sample_count_per_channel_;
sample_count_per_channel_ =
std::llround(static_cast<double>(sample_count_per_channel_) / speed);
@@ -160,10 +162,17 @@ void SampleBuffer::speed(double speed)
}
for (size_t i = 0; i < sample_count_per_channel_; i++) {
size_t input_index = std::floor(static_cast<double>(i) * speed);
// Linear interpolation between the two nearest input samples,
// rather than nearest-neighbor sampling which aliases audibly
const double input_position = static_cast<double>(i) * speed;
const size_t input_index = static_cast<size_t>(input_position);
const double fraction = input_position - input_index;
const size_t next_index =
std::min(input_index + 1, input_sample_count - 1);
for (int j = 0; j < audio_params_.channel_count(); j++) {
output_data[j][i] = data_[j][input_index];
output_data[j][i] = data_[j][input_index] * (1.0 - fraction) +
data_[j][next_index] * fraction;
}
}
+23
View File
@@ -251,6 +251,29 @@ TEST(CoreSampleBuffer, Speed)
EXPECT_FLOAT_EQ(b.data(0)[1], 0.3f);
}
TEST(CoreSampleBuffer, SpeedInterpolatesBetweenSamples)
{
AudioParams params = make_params();
SampleBuffer b(params, 4);
b.data(0)[0] = 0.0f;
b.data(0)[1] = 1.0f;
b.data(0)[2] = 0.0f;
b.data(0)[3] = -1.0f;
// 0.5x doubles the length; odd output samples sit exactly between two
// input samples and must be interpolated, not nearest-neighbor picked
b.speed(0.5);
ASSERT_EQ(b.sample_count(), 8u);
EXPECT_FLOAT_EQ(b.data(0)[0], 0.0f);
EXPECT_FLOAT_EQ(b.data(0)[1], 0.5f);
EXPECT_FLOAT_EQ(b.data(0)[2], 1.0f);
EXPECT_FLOAT_EQ(b.data(0)[3], 0.5f);
EXPECT_FLOAT_EQ(b.data(0)[4], 0.0f);
EXPECT_FLOAT_EQ(b.data(0)[5], -0.5f);
EXPECT_FLOAT_EQ(b.data(0)[6], -1.0f);
EXPECT_FLOAT_EQ(b.data(0)[7], -1.0f); // clamped at the last sample
}
TEST(CoreSampleBuffer, UnallocatedOperationsNoCrash)
{
SampleBuffer b;
+55
View File
@@ -145,6 +145,61 @@ TEST(ViewerSmokeTimer, ZeroSpeed)
EXPECT_EQ(ts1, ts2);
}
class FakeAudioClock : public PlaybackAudioClock {
public:
virtual double seconds() const override
{
return seconds_;
}
double seconds_ = -1.0;
};
TEST(ViewerSmokeTimer, AudioClockDrivesTimestamp)
{
FakeAudioClock clock;
ViewerPlaybackTimer timer;
// Start at timestamp 100, 1x speed, 32fps (exact in floating point)
timer.start(100, 1, 1.0 / 32.0, &clock);
// One second of consumed audio = 32 frames, regardless of wall time
clock.seconds_ = 1.0;
EXPECT_EQ(timer.get_timestamp_now(), 132);
// The audio clock fully overrides the wall clock (no sleep involved)
clock.seconds_ = 0.5;
EXPECT_EQ(timer.get_timestamp_now(), 116);
}
TEST(ViewerSmokeTimer, AudioClockScalesWithPlaybackSpeed)
{
FakeAudioClock clock;
ViewerPlaybackTimer timer;
// At 2x speed one output second is two timeline seconds
timer.start(0, 2, 1.0 / 32.0, &clock);
clock.seconds_ = 0.5;
EXPECT_EQ(timer.get_timestamp_now(), 32);
// In reverse the playhead moves backward at |speed|
timer.start(100, -2, 1.0 / 32.0, &clock);
clock.seconds_ = 1.0;
EXPECT_EQ(timer.get_timestamp_now(), 36);
}
TEST(ViewerSmokeTimer, InvalidAudioClockFallsBackToWallClock)
{
FakeAudioClock clock; // seconds() returns -1: no clocked output running
ViewerPlaybackTimer timer;
timer.start(0, 1, 1.0 / 24.0, &clock);
EXPECT_GE(timer.get_timestamp_now(), 0);
QThread::msleep(50); // 50ms > one 24fps frame period
EXPECT_GT(timer.get_timestamp_now(), 0);
}
// ============================================================================
// Smoke Test: ViewerQueue
// ============================================================================