Implement audio sync foundations

2026-07-13 10:19:29 +08:00
parent 4d52267c4a
commit 08b1c6d562
21 changed files with 1094 additions and 18 deletions
@@ -16,6 +16,12 @@

 set(OLIVE_SOURCES
  ${OLIVE_SOURCES}
+  audio/audiolevelmeter.cpp
+  audio/audiolevelmeter.h
+  audio/audiosynchronizer.cpp
+  audio/audiosynchronizer.h
+  audio/audiowaveformsync.cpp
+  audio/audiowaveformsync.h
  audio/audiomanager.cpp
  audio/audiomanager.h
  audio/audioprocessor.cpp
@@ -0,0 +1,103 @@
+/***
+
+  Oak - Non-Linear Video Editor
+  Copyright (C) 2026 Oak Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#include "audiolevelmeter.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "common/decibel.h"
+
+namespace olive
+{
+
+AudioLevelMeter::Stats
+AudioLevelMeter::AnalyzeSampleBuffer(const core::SampleBuffer &samples)
+{
+	Stats stats;
+
+	const int channel_count = samples.channel_count();
+	const size_t sample_count = samples.sample_count();
+	stats.channels.resize(channel_count);
+
+	if (!channel_count || !sample_count) {
+		return stats;
+	}
+
+	double total_square = 0.0;
+	size_t total_samples = 0;
+
+	for (int channel = 0; channel < channel_count; channel++) {
+		const float *channel_data = samples.data(channel);
+		double peak = 0.0;
+		double square_sum = 0.0;
+
+		for (size_t sample = 0; sample < sample_count; sample++) {
+			const double value = channel_data[sample];
+			const double abs_value = std::abs(value);
+
+			peak = std::max(peak, abs_value);
+			square_sum += value * value;
+		}
+
+		const double mean_square = square_sum / static_cast<double>(sample_count);
+		const double rms = std::sqrt(mean_square);
+
+		ChannelStats channel_stats;
+		channel_stats.peak_linear = peak;
+		channel_stats.peak_db = LinearToDb(peak);
+		channel_stats.rms_linear = rms;
+		channel_stats.rms_db = LinearToDb(rms);
+		channel_stats.vu_db = channel_stats.rms_db;
+		stats.channels[channel] = channel_stats;
+
+		stats.max_peak_linear = std::max(stats.max_peak_linear, peak);
+		total_square += square_sum;
+		total_samples += sample_count;
+	}
+
+	stats.silence = qFuzzyIsNull(stats.max_peak_linear);
+	stats.integrated_lufs = PowerToLufs(
+		total_square / static_cast<double>(total_samples));
+
+	return stats;
+}
+
+double AudioLevelMeter::LinearToDb(double linear)
+{
+	if (linear <= 0.0) {
+		return Decibel::MINIMUM;
+	}
+
+	return Decibel::fromLinear(linear);
+}
+
+double AudioLevelMeter::PowerToLufs(double mean_square)
+{
+	if (mean_square <= 0.0) {
+		return Decibel::MINIMUM;
+	}
+
+	// BS.1770 loudness uses K-weighted mean square. This first pass stores the
+	// compatible unit and can be extended with K-weighting without changing UI.
+	return -0.691 + 10.0 * std::log10(mean_square);
+}
+
+}
@@ -0,0 +1,57 @@
+/***
+
+  Oak - Non-Linear Video Editor
+  Copyright (C) 2026 Oak Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#ifndef AUDIOLEVELMETER_H
+#define AUDIOLEVELMETER_H
+
+#include <QVector>
+
+#include "olive/core/render/samplebuffer.h"
+
+namespace olive
+{
+
+class AudioLevelMeter {
+public:
+	struct ChannelStats {
+		double peak_linear = 0.0;
+		double peak_db = -200.0;
+		double rms_linear = 0.0;
+		double rms_db = -200.0;
+		double vu_db = -200.0;
+	};
+
+	struct Stats {
+		QVector<ChannelStats> channels;
+		double max_peak_linear = 0.0;
+		double integrated_lufs = -200.0;
+		bool silence = true;
+	};
+
+	static Stats AnalyzeSampleBuffer(const core::SampleBuffer &samples);
+
+private:
+	static double LinearToDb(double linear);
+	static double PowerToLufs(double mean_square);
+};
+
+}
+
+#endif // AUDIOLEVELMETER_H
@@ -0,0 +1,66 @@
+/***
+
+  Oak - Non-Linear Video Editor
+  Copyright (C) 2026 Oak Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#include "audiosynchronizer.h"
+
+namespace olive
+{
+
+AudioSynchronizer::Placement AudioSynchronizer::PlaceBySourceTime(
+	const SourceClip &reference, const SourceClip &candidate,
+	const core::rational &reference_timeline_in)
+{
+	Placement placement;
+	if (!reference.has_source_start_time ||
+		!candidate.has_source_start_time ||
+		reference.source_start_time.isNaN() ||
+		candidate.source_start_time.isNaN()) {
+		return placement;
+	}
+
+	const core::rational reference_head_source =
+		reference.source_start_time + reference.media_in;
+	const core::rational candidate_head_source =
+		candidate.source_start_time + candidate.media_in;
+
+	placement.timeline_in =
+		reference_timeline_in + candidate_head_source - reference_head_source;
+	placement.valid = !placement.timeline_in.isNaN();
+	return placement;
+}
+
+AudioSynchronizer::Placement AudioSynchronizer::PlaceByWaveformOffset(
+	const core::rational &reference_timeline_in,
+	int64_t candidate_offset_samples, int sample_rate)
+{
+	Placement placement;
+	if (sample_rate <= 0) {
+		return placement;
+	}
+
+	placement.timeline_in =
+		reference_timeline_in +
+		core::rational::fromDouble(static_cast<double>(candidate_offset_samples) /
+								   static_cast<double>(sample_rate));
+	placement.valid = !placement.timeline_in.isNaN();
+	return placement;
+}
+
+}
@@ -0,0 +1,55 @@
+/***
+
+  Oak - Non-Linear Video Editor
+  Copyright (C) 2026 Oak Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#ifndef AUDIOSYNCHRONIZER_H
+#define AUDIOSYNCHRONIZER_H
+
+#include <cstdint>
+
+#include "olive/core/util/rational.h"
+
+namespace olive
+{
+
+class AudioSynchronizer {
+public:
+	struct SourceClip {
+		core::rational source_start_time;
+		core::rational media_in;
+		bool has_source_start_time = false;
+	};
+
+	struct Placement {
+		core::rational timeline_in;
+		bool valid = false;
+	};
+
+	static Placement PlaceBySourceTime(const SourceClip &reference,
+									   const SourceClip &candidate,
+									   const core::rational &reference_timeline_in);
+
+	static Placement PlaceByWaveformOffset(
+		const core::rational &reference_timeline_in,
+		int64_t candidate_offset_samples, int sample_rate);
+};
+
+}
+
+#endif // AUDIOSYNCHRONIZER_H
@@ -0,0 +1,152 @@
+/***
+
+  Oak - Non-Linear Video Editor
+  Copyright (C) 2026 Oak Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#include "audiowaveformsync.h"
+
+#include <algorithm>
+#include <cmath>
+
+namespace olive
+{
+
+QVector<double>
+AudioWaveformSync::ExtractRmsEnvelope(const core::SampleBuffer &samples,
+									  size_t window_samples)
+{
+	QVector<double> envelope;
+
+	const int channel_count = samples.channel_count();
+	const size_t sample_count = samples.sample_count();
+	if (!channel_count || !sample_count || !window_samples) {
+		return envelope;
+	}
+
+	const size_t window_count =
+		(sample_count + window_samples - 1) / window_samples;
+	envelope.resize(static_cast<int>(window_count));
+
+	for (size_t window = 0; window < window_count; window++) {
+		const size_t start = window * window_samples;
+		const size_t end = std::min(start + window_samples, sample_count);
+		double square_sum = 0.0;
+		size_t total = 0;
+
+		for (int channel = 0; channel < channel_count; channel++) {
+			const float *data = samples.data(channel);
+			for (size_t sample = start; sample < end; sample++) {
+				const double value = data[sample];
+				square_sum += value * value;
+				total++;
+			}
+		}
+
+		envelope[static_cast<int>(window)] =
+			total ? std::sqrt(square_sum / static_cast<double>(total)) : 0.0;
+	}
+
+	return envelope;
+}
+
+AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateOffset(
+	const core::SampleBuffer &reference, const core::SampleBuffer &candidate,
+	size_t window_samples, int64_t max_offset_samples)
+{
+	if (!window_samples) {
+		return OffsetResult();
+	}
+
+	const QVector<double> reference_envelope =
+		ExtractRmsEnvelope(reference, window_samples);
+	const QVector<double> candidate_envelope =
+		ExtractRmsEnvelope(candidate, window_samples);
+	const int64_t max_offset_windows =
+		max_offset_samples / static_cast<int64_t>(window_samples);
+
+	return EstimateEnvelopeOffset(reference_envelope, candidate_envelope,
+								  window_samples, max_offset_windows);
+}
+
+AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateEnvelopeOffset(
+	const QVector<double> &reference, const QVector<double> &candidate,
+	size_t window_samples, int64_t max_offset_windows)
+{
+	OffsetResult result;
+	if (reference.isEmpty() || candidate.isEmpty() || !window_samples) {
+		return result;
+	}
+
+	double best_score = -2.0;
+	int64_t best_lag = 0;
+
+	for (int64_t lag = -max_offset_windows; lag <= max_offset_windows; lag++) {
+		const int reference_start = static_cast<int>(std::max<int64_t>(0, -lag));
+		const int candidate_start = static_cast<int>(std::max<int64_t>(0, lag));
+		const int overlap = std::min(reference.size() - reference_start,
+									 candidate.size() - candidate_start);
+
+		if (overlap < 2) {
+			continue;
+		}
+
+		double reference_mean = 0.0;
+		double candidate_mean = 0.0;
+		for (int i = 0; i < overlap; i++) {
+			reference_mean += reference.at(reference_start + i);
+			candidate_mean += candidate.at(candidate_start + i);
+		}
+		reference_mean /= static_cast<double>(overlap);
+		candidate_mean /= static_cast<double>(overlap);
+
+		double numerator = 0.0;
+		double reference_energy = 0.0;
+		double candidate_energy = 0.0;
+		for (int i = 0; i < overlap; i++) {
+			const double reference_value =
+				reference.at(reference_start + i) - reference_mean;
+			const double candidate_value =
+				candidate.at(candidate_start + i) - candidate_mean;
+			numerator += reference_value * candidate_value;
+			reference_energy += reference_value * reference_value;
+			candidate_energy += candidate_value * candidate_value;
+		}
+
+		if (qFuzzyIsNull(reference_energy) || qFuzzyIsNull(candidate_energy)) {
+			continue;
+		}
+
+		const double score =
+			numerator / std::sqrt(reference_energy * candidate_energy);
+		if (score > best_score) {
+			best_score = score;
+			best_lag = lag;
+		}
+	}
+
+	if (best_score > -2.0) {
+		result.valid = true;
+		result.confidence = std::max(0.0, best_score);
+		result.offset_samples =
+			best_lag * static_cast<int64_t>(window_samples);
+	}
+
+	return result;
+}
+
+}
@@ -0,0 +1,58 @@
+/***
+
+  Oak - Non-Linear Video Editor
+  Copyright (C) 2026 Oak Team
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+***/
+
+#ifndef AUDIOWAVEFORMSYNC_H
+#define AUDIOWAVEFORMSYNC_H
+
+#include <cstdint>
+
+#include <QVector>
+
+#include "olive/core/render/samplebuffer.h"
+
+namespace olive
+{
+
+class AudioWaveformSync {
+public:
+	struct OffsetResult {
+		int64_t offset_samples = 0;
+		double confidence = 0.0;
+		bool valid = false;
+	};
+
+	static QVector<double>
+	ExtractRmsEnvelope(const core::SampleBuffer &samples, size_t window_samples);
+
+	static OffsetResult EstimateOffset(const core::SampleBuffer &reference,
+									   const core::SampleBuffer &candidate,
+									   size_t window_samples,
+									   int64_t max_offset_samples);
+
+private:
+	static OffsetResult EstimateEnvelopeOffset(const QVector<double> &reference,
+											   const QVector<double> &candidate,
+											   size_t window_samples,
+											   int64_t max_offset_windows);
+};
+
+}
+
+#endif // AUDIOWAVEFORMSYNC_H