Implement audio sync foundations

This commit is contained in:
2026-07-13 10:19:29 +08:00
parent 4d52267c4a
commit 08b1c6d562
21 changed files with 1094 additions and 18 deletions
+6
View File
@@ -16,6 +16,12 @@
set(OLIVE_SOURCES
${OLIVE_SOURCES}
audio/audiolevelmeter.cpp
audio/audiolevelmeter.h
audio/audiosynchronizer.cpp
audio/audiosynchronizer.h
audio/audiowaveformsync.cpp
audio/audiowaveformsync.h
audio/audiomanager.cpp
audio/audiomanager.h
audio/audioprocessor.cpp
+103
View File
@@ -0,0 +1,103 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "audiolevelmeter.h"
#include <algorithm>
#include <cmath>
#include "common/decibel.h"
namespace olive
{
AudioLevelMeter::Stats
AudioLevelMeter::AnalyzeSampleBuffer(const core::SampleBuffer &samples)
{
Stats stats;
const int channel_count = samples.channel_count();
const size_t sample_count = samples.sample_count();
stats.channels.resize(channel_count);
if (!channel_count || !sample_count) {
return stats;
}
double total_square = 0.0;
size_t total_samples = 0;
for (int channel = 0; channel < channel_count; channel++) {
const float *channel_data = samples.data(channel);
double peak = 0.0;
double square_sum = 0.0;
for (size_t sample = 0; sample < sample_count; sample++) {
const double value = channel_data[sample];
const double abs_value = std::abs(value);
peak = std::max(peak, abs_value);
square_sum += value * value;
}
const double mean_square = square_sum / static_cast<double>(sample_count);
const double rms = std::sqrt(mean_square);
ChannelStats channel_stats;
channel_stats.peak_linear = peak;
channel_stats.peak_db = LinearToDb(peak);
channel_stats.rms_linear = rms;
channel_stats.rms_db = LinearToDb(rms);
channel_stats.vu_db = channel_stats.rms_db;
stats.channels[channel] = channel_stats;
stats.max_peak_linear = std::max(stats.max_peak_linear, peak);
total_square += square_sum;
total_samples += sample_count;
}
stats.silence = qFuzzyIsNull(stats.max_peak_linear);
stats.integrated_lufs = PowerToLufs(
total_square / static_cast<double>(total_samples));
return stats;
}
double AudioLevelMeter::LinearToDb(double linear)
{
if (linear <= 0.0) {
return Decibel::MINIMUM;
}
return Decibel::fromLinear(linear);
}
double AudioLevelMeter::PowerToLufs(double mean_square)
{
if (mean_square <= 0.0) {
return Decibel::MINIMUM;
}
// BS.1770 loudness uses K-weighted mean square. This first pass stores the
// compatible unit and can be extended with K-weighting without changing UI.
return -0.691 + 10.0 * std::log10(mean_square);
}
}
+57
View File
@@ -0,0 +1,57 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef AUDIOLEVELMETER_H
#define AUDIOLEVELMETER_H
#include <QVector>
#include "olive/core/render/samplebuffer.h"
namespace olive
{
class AudioLevelMeter {
public:
struct ChannelStats {
double peak_linear = 0.0;
double peak_db = -200.0;
double rms_linear = 0.0;
double rms_db = -200.0;
double vu_db = -200.0;
};
struct Stats {
QVector<ChannelStats> channels;
double max_peak_linear = 0.0;
double integrated_lufs = -200.0;
bool silence = true;
};
static Stats AnalyzeSampleBuffer(const core::SampleBuffer &samples);
private:
static double LinearToDb(double linear);
static double PowerToLufs(double mean_square);
};
}
#endif // AUDIOLEVELMETER_H
+66
View File
@@ -0,0 +1,66 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "audiosynchronizer.h"
namespace olive
{
AudioSynchronizer::Placement AudioSynchronizer::PlaceBySourceTime(
const SourceClip &reference, const SourceClip &candidate,
const core::rational &reference_timeline_in)
{
Placement placement;
if (!reference.has_source_start_time ||
!candidate.has_source_start_time ||
reference.source_start_time.isNaN() ||
candidate.source_start_time.isNaN()) {
return placement;
}
const core::rational reference_head_source =
reference.source_start_time + reference.media_in;
const core::rational candidate_head_source =
candidate.source_start_time + candidate.media_in;
placement.timeline_in =
reference_timeline_in + candidate_head_source - reference_head_source;
placement.valid = !placement.timeline_in.isNaN();
return placement;
}
AudioSynchronizer::Placement AudioSynchronizer::PlaceByWaveformOffset(
const core::rational &reference_timeline_in,
int64_t candidate_offset_samples, int sample_rate)
{
Placement placement;
if (sample_rate <= 0) {
return placement;
}
placement.timeline_in =
reference_timeline_in +
core::rational::fromDouble(static_cast<double>(candidate_offset_samples) /
static_cast<double>(sample_rate));
placement.valid = !placement.timeline_in.isNaN();
return placement;
}
}
+55
View File
@@ -0,0 +1,55 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef AUDIOSYNCHRONIZER_H
#define AUDIOSYNCHRONIZER_H
#include <cstdint>
#include "olive/core/util/rational.h"
namespace olive
{
class AudioSynchronizer {
public:
struct SourceClip {
core::rational source_start_time;
core::rational media_in;
bool has_source_start_time = false;
};
struct Placement {
core::rational timeline_in;
bool valid = false;
};
static Placement PlaceBySourceTime(const SourceClip &reference,
const SourceClip &candidate,
const core::rational &reference_timeline_in);
static Placement PlaceByWaveformOffset(
const core::rational &reference_timeline_in,
int64_t candidate_offset_samples, int sample_rate);
};
}
#endif // AUDIOSYNCHRONIZER_H
+152
View File
@@ -0,0 +1,152 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "audiowaveformsync.h"
#include <algorithm>
#include <cmath>
namespace olive
{
QVector<double>
AudioWaveformSync::ExtractRmsEnvelope(const core::SampleBuffer &samples,
size_t window_samples)
{
QVector<double> envelope;
const int channel_count = samples.channel_count();
const size_t sample_count = samples.sample_count();
if (!channel_count || !sample_count || !window_samples) {
return envelope;
}
const size_t window_count =
(sample_count + window_samples - 1) / window_samples;
envelope.resize(static_cast<int>(window_count));
for (size_t window = 0; window < window_count; window++) {
const size_t start = window * window_samples;
const size_t end = std::min(start + window_samples, sample_count);
double square_sum = 0.0;
size_t total = 0;
for (int channel = 0; channel < channel_count; channel++) {
const float *data = samples.data(channel);
for (size_t sample = start; sample < end; sample++) {
const double value = data[sample];
square_sum += value * value;
total++;
}
}
envelope[static_cast<int>(window)] =
total ? std::sqrt(square_sum / static_cast<double>(total)) : 0.0;
}
return envelope;
}
AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateOffset(
const core::SampleBuffer &reference, const core::SampleBuffer &candidate,
size_t window_samples, int64_t max_offset_samples)
{
if (!window_samples) {
return OffsetResult();
}
const QVector<double> reference_envelope =
ExtractRmsEnvelope(reference, window_samples);
const QVector<double> candidate_envelope =
ExtractRmsEnvelope(candidate, window_samples);
const int64_t max_offset_windows =
max_offset_samples / static_cast<int64_t>(window_samples);
return EstimateEnvelopeOffset(reference_envelope, candidate_envelope,
window_samples, max_offset_windows);
}
AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateEnvelopeOffset(
const QVector<double> &reference, const QVector<double> &candidate,
size_t window_samples, int64_t max_offset_windows)
{
OffsetResult result;
if (reference.isEmpty() || candidate.isEmpty() || !window_samples) {
return result;
}
double best_score = -2.0;
int64_t best_lag = 0;
for (int64_t lag = -max_offset_windows; lag <= max_offset_windows; lag++) {
const int reference_start = static_cast<int>(std::max<int64_t>(0, -lag));
const int candidate_start = static_cast<int>(std::max<int64_t>(0, lag));
const int overlap = std::min(reference.size() - reference_start,
candidate.size() - candidate_start);
if (overlap < 2) {
continue;
}
double reference_mean = 0.0;
double candidate_mean = 0.0;
for (int i = 0; i < overlap; i++) {
reference_mean += reference.at(reference_start + i);
candidate_mean += candidate.at(candidate_start + i);
}
reference_mean /= static_cast<double>(overlap);
candidate_mean /= static_cast<double>(overlap);
double numerator = 0.0;
double reference_energy = 0.0;
double candidate_energy = 0.0;
for (int i = 0; i < overlap; i++) {
const double reference_value =
reference.at(reference_start + i) - reference_mean;
const double candidate_value =
candidate.at(candidate_start + i) - candidate_mean;
numerator += reference_value * candidate_value;
reference_energy += reference_value * reference_value;
candidate_energy += candidate_value * candidate_value;
}
if (qFuzzyIsNull(reference_energy) || qFuzzyIsNull(candidate_energy)) {
continue;
}
const double score =
numerator / std::sqrt(reference_energy * candidate_energy);
if (score > best_score) {
best_score = score;
best_lag = lag;
}
}
if (best_score > -2.0) {
result.valid = true;
result.confidence = std::max(0.0, best_score);
result.offset_samples =
best_lag * static_cast<int64_t>(window_samples);
}
return result;
}
}
+58
View File
@@ -0,0 +1,58 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef AUDIOWAVEFORMSYNC_H
#define AUDIOWAVEFORMSYNC_H
#include <cstdint>
#include <QVector>
#include "olive/core/render/samplebuffer.h"
namespace olive
{
class AudioWaveformSync {
public:
struct OffsetResult {
int64_t offset_samples = 0;
double confidence = 0.0;
bool valid = false;
};
static QVector<double>
ExtractRmsEnvelope(const core::SampleBuffer &samples, size_t window_samples);
static OffsetResult EstimateOffset(const core::SampleBuffer &reference,
const core::SampleBuffer &candidate,
size_t window_samples,
int64_t max_offset_samples);
private:
static OffsetResult EstimateEnvelopeOffset(const QVector<double> &reference,
const QVector<double> &candidate,
size_t window_samples,
int64_t max_offset_windows);
};
}
#endif // AUDIOWAVEFORMSYNC_H
+2
View File
@@ -33,5 +33,7 @@ set(OLIVE_SOURCES
codec/frame.h
codec/planarfiledevice.cpp
codec/planarfiledevice.h
codec/timecodemetadata.cpp
codec/timecodemetadata.h
PARENT_SCOPE
)
+43
View File
@@ -81,6 +81,7 @@ extern "C" {
#include <QThread>
#include "codec/planarfiledevice.h"
#include "codec/timecodemetadata.h"
#include "common/ffmpegutils.h"
#include "common/filefunctions.h"
#include "render/renderer.h"
@@ -128,6 +129,36 @@ void DiscardSubtitleStreams(AVFormatContext *ctx)
}
}
TimecodeMetadata::SourceTime ExtractSourceStartTime(
AVDictionary *metadata, const rational &timebase, int sample_rate)
{
if (!metadata) {
return TimecodeMetadata::SourceTime();
}
if (AVDictionaryEntry *entry =
av_dict_get(metadata, "timecode", nullptr, AV_DICT_IGNORE_SUFFIX)) {
TimecodeMetadata::SourceTime parsed =
TimecodeMetadata::FromTimecodeString(
QString::fromUtf8(entry->value), timebase);
if (parsed.valid) {
return parsed;
}
}
if (AVDictionaryEntry *entry = av_dict_get(
metadata, "time_reference", nullptr, AV_DICT_IGNORE_SUFFIX)) {
TimecodeMetadata::SourceTime parsed =
TimecodeMetadata::FromBwfTimeReference(
QString::fromUtf8(entry->value), sample_rate);
if (parsed.valid) {
return parsed;
}
}
return TimecodeMetadata::SourceTime();
}
} // namespace
FFmpegDecoder::FFmpegDecoder()
@@ -530,6 +561,9 @@ FootageDescription FFmpegDecoder::Probe(const QString &filename,
avformat_find_stream_info(fmt_ctx, nullptr);
int64_t footage_duration = fmt_ctx->duration;
TimecodeMetadata::SourceTime source_start_time =
ExtractSourceStartTime(fmt_ctx->metadata, rational(1, AV_TIME_BASE),
0);
bool duration_guessed_from_bitrate =
(fmt_ctx->duration_estimation_method ==
@@ -545,6 +579,11 @@ FootageDescription FFmpegDecoder::Probe(const QString &filename,
for (unsigned int i = 0; i < fmt_ctx->nb_streams; i++) {
// FFmpeg AVStream
AVStream *avstream = fmt_ctx->streams[i];
if (!source_start_time.valid) {
source_start_time = ExtractSourceStartTime(
avstream->metadata, avstream->time_base,
avstream->codecpar->sample_rate);
}
// Find decoder for this stream, if it exists we can proceed
const AVCodec *decoder =
@@ -720,6 +759,10 @@ FootageDescription FFmpegDecoder::Probe(const QString &filename,
}
desc.SetStreamCount(fmt_ctx->nb_streams);
if (source_start_time.valid) {
desc.SetSourceStartTime(source_start_time.time,
source_start_time.source);
}
if (video_streams == 0 && audio_streams > 0 && still_streams > 0) {
// This footage has no video streams, but has audio and image streams. We've probably
+88
View File
@@ -0,0 +1,88 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "timecodemetadata.h"
#include <limits>
#include <numeric>
#include "olive/core/util/timecodefunctions.h"
namespace olive
{
TimecodeMetadata::SourceTime TimecodeMetadata::FromTimecodeString(
const QString &timecode, const core::rational &timebase)
{
SourceTime result;
const QString trimmed = timecode.trimmed();
if (trimmed.isEmpty()) {
return result;
}
bool ok = false;
const core::Timecode::Display display =
trimmed.contains(';') ? core::Timecode::kTimecodeDropFrame
: core::Timecode::kTimecodeNonDropFrame;
result.time = core::Timecode::timecode_to_time(trimmed.toStdString(),
timebase, display, &ok);
result.valid = ok;
if (ok) {
result.source = QStringLiteral("timecode");
}
return result;
}
TimecodeMetadata::SourceTime TimecodeMetadata::FromBwfTimeReference(
const QString &time_reference, int sample_rate)
{
SourceTime result;
if (sample_rate <= 0) {
return result;
}
bool ok = false;
const qulonglong samples = time_reference.trimmed().toULongLong(&ok);
if (!ok) {
return result;
}
qulonglong numerator = samples;
qulonglong denominator = static_cast<qulonglong>(sample_rate);
const qulonglong divisor = std::gcd(numerator, denominator);
numerator /= divisor;
denominator /= divisor;
const qulonglong rational_limit =
static_cast<qulonglong>(std::numeric_limits<int>::max());
if (numerator <= rational_limit && denominator <= rational_limit) {
result.time =
core::rational(static_cast<int>(numerator),
static_cast<int>(denominator));
} else {
result.time = core::rational::fromDouble(
static_cast<double>(samples) / static_cast<double>(sample_rate));
}
result.source = QStringLiteral("bwf_time_reference");
result.valid = true;
return result;
}
}
+48
View File
@@ -0,0 +1,48 @@
/***
Oak - Non-Linear Video Editor
Copyright (C) 2026 Oak Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef TIMECODEMETADATA_H
#define TIMECODEMETADATA_H
#include <QString>
#include "olive/core/util/rational.h"
namespace olive
{
class TimecodeMetadata {
public:
struct SourceTime {
core::rational time;
QString source;
bool valid = false;
};
static SourceTime FromTimecodeString(const QString &timecode,
const core::rational &timebase);
static SourceTime FromBwfTimeReference(const QString &time_reference,
int sample_rate);
};
}
#endif // TIMECODEMETADATA_H
@@ -63,6 +63,26 @@ bool FootageDescription::Load(const QString &filename)
while (XMLReadNextStartElement(&reader)) {
if (reader.name() == QStringLiteral("decoder")) {
decoder_ = reader.readElementText();
} else if (reader.name() ==
QStringLiteral("sourcestarttime")) {
QString source;
{
XMLAttributeLoop((&reader), attr)
{
if (attr.name() == QStringLiteral("source")) {
source = attr.value().toString();
}
}
}
const QStringList split =
reader.readElementText().split('/');
if (split.size() == 2) {
SetSourceStartTime(
rational(split.at(0).toInt(),
split.at(1).toInt()),
source);
}
} else if (reader.name() == QStringLiteral("streams")) {
{
XMLAttributeLoop((&reader), attr)
@@ -133,6 +153,16 @@ bool FootageDescription::Save(const QString &filename) const
writer.writeTextElement(QStringLiteral("decoder"), decoder_);
if (has_source_start_time_) {
writer.writeStartElement(QStringLiteral("sourcestarttime"));
writer.writeAttribute(QStringLiteral("source"),
source_start_time_source_);
writer.writeCharacters(QStringLiteral("%1/%2").arg(
QString::number(source_start_time_.numerator()),
QString::number(source_start_time_.denominator())));
writer.writeEndElement();
}
writer.writeStartElement(QStringLiteral("streams"));
writer.writeAttribute(QStringLiteral("count"),
+32 -1
View File
@@ -22,6 +22,8 @@
#ifndef FOOTAGEDESCRIPTION_H
#define FOOTAGEDESCRIPTION_H
#include <QString>
#include "node/output/track/track.h"
#include "render/subtitleparams.h"
#include "render/videoparams.h"
@@ -34,6 +36,7 @@ public:
FootageDescription(const QString &decoder = QString())
: decoder_(decoder)
, total_stream_count_(0)
, has_source_start_time_(false)
{
}
@@ -131,6 +134,28 @@ public:
total_stream_count_ = s;
}
void SetSourceStartTime(const rational &time, const QString &source)
{
source_start_time_ = time;
source_start_time_source_ = source;
has_source_start_time_ = true;
}
bool HasSourceStartTime() const
{
return has_source_start_time_;
}
const rational &source_start_time() const
{
return source_start_time_;
}
const QString &source_start_time_source() const
{
return source_start_time_source_;
}
bool Load(const QString &filename);
bool Save(const QString &filename) const;
@@ -163,7 +188,7 @@ public:
}
private:
static constexpr unsigned kFootageMetaVersion = 6;
static constexpr unsigned kFootageMetaVersion = 7;
QString decoder_;
@@ -174,6 +199,12 @@ private:
QVector<SubtitleParams> subtitle_streams_;
int total_stream_count_;
rational source_start_time_;
QString source_start_time_source_;
bool has_source_start_time_;
};
}
+6 -4
View File
@@ -25,6 +25,7 @@
#include <QDebug>
#include <QPainter>
#include "audio/audiolevelmeter.h"
#include "audio/audiomanager.h"
#include "common/decibel.h"
#include "common/qtutils.h"
@@ -89,10 +90,11 @@ void AudioMonitor::PushSampleBuffer(const SampleBuffer &d)
QVector<double> v(params_.channel_count(), 0);
AudioVisualWaveform::Sample summed =
AudioVisualWaveform::SumSamples(d, 0, d.sample_count());
AudioVisualWaveformSampleToInternalValues(summed, v);
const AudioLevelMeter::Stats stats =
AudioLevelMeter::AnalyzeSampleBuffer(d);
for (int i = 0; i < v.size() && i < stats.channels.size(); i++) {
v[i] = stats.channels.at(i).peak_linear;
}
// Fill values because they get averaged out for smoothing
values_.fill(v);