Implement audio sync foundations

2026-07-13 10:19:29 +08:00
parent 4d52267c4a
commit 08b1c6d562
21 changed files with 1094 additions and 18 deletions
@@ -3,6 +3,9 @@ add_executable(olive-gtest
  common_current_test.cpp
  common_xmlutils_test.cpp
  config_test.cpp
+  audio_level_meter_test.cpp
+  audio_synchronizer_test.cpp
+  audio_waveform_sync_test.cpp
  color_lut_test.cpp
  node_value_test.cpp
  node_keyframe_test.cpp
@@ -38,6 +41,7 @@ add_executable(olive-gtest
  task_taskmanager_test.cpp
  module_smoke_test.cpp
  shader_resources_test.cpp
+  timecode_metadata_test.cpp
  timebased_widget_test.cpp
  timeline_coordinate_test.cpp
  timeline_workarea_test.cpp
@@ -0,0 +1,80 @@
+#include <gtest/gtest.h>
+
+#include <cmath>
+
+#include "audio/audiolevelmeter.h"
+#include "olive/core/render/audioparams.h"
+#include "olive/core/render/samplebuffer.h"
+#include "olive/core/render/sampleformat.h"
+
+extern "C" {
+#include <libavutil/channel_layout.h>
+}
+
+namespace {
+
+olive::core::AudioParams MakeStereoParams()
+{
+	return olive::core::AudioParams(48000, AV_CH_LAYOUT_STEREO,
+									olive::core::SampleFormat::F32P);
+}
+
+}
+
+TEST(AudioLevelMeter, Silence)
+{
+	olive::core::SampleBuffer samples(MakeStereoParams(), size_t(480));
+	samples.silence();
+
+	const olive::AudioLevelMeter::Stats stats =
+		olive::AudioLevelMeter::AnalyzeSampleBuffer(samples);
+
+	ASSERT_EQ(stats.channels.size(), 2);
+	EXPECT_TRUE(stats.silence);
+	EXPECT_DOUBLE_EQ(stats.max_peak_linear, 0.0);
+	EXPECT_DOUBLE_EQ(stats.channels.at(0).peak_linear, 0.0);
+	EXPECT_DOUBLE_EQ(stats.channels.at(0).rms_linear, 0.0);
+	EXPECT_DOUBLE_EQ(stats.integrated_lufs, -200.0);
+}
+
+TEST(AudioLevelMeter, ConstantSignal)
+{
+	olive::core::SampleBuffer samples(MakeStereoParams(), size_t(480));
+	for (int channel = 0; channel < samples.channel_count(); channel++) {
+		float *data = samples.data(channel);
+		for (size_t i = 0; i < samples.sample_count(); i++) {
+			data[i] = 0.5f;
+		}
+	}
+
+	const olive::AudioLevelMeter::Stats stats =
+		olive::AudioLevelMeter::AnalyzeSampleBuffer(samples);
+
+	ASSERT_EQ(stats.channels.size(), 2);
+	EXPECT_FALSE(stats.silence);
+	EXPECT_DOUBLE_EQ(stats.max_peak_linear, 0.5);
+	EXPECT_NEAR(stats.channels.at(0).peak_db, -6.0206, 0.0001);
+	EXPECT_NEAR(stats.channels.at(0).rms_linear, 0.5, 0.0001);
+	EXPECT_NEAR(stats.channels.at(0).vu_db, -6.0206, 0.0001);
+	EXPECT_NEAR(stats.integrated_lufs, -6.7116, 0.0001);
+}
+
+TEST(AudioLevelMeter, PerChannelPeaksAndRms)
+{
+	olive::core::SampleBuffer samples(MakeStereoParams(), size_t(4));
+	float left[] = { 0.0f, 0.25f, -0.5f, 1.0f };
+	float right[] = { 0.0f, -0.25f, 0.25f, -0.25f };
+	samples.set(0, left, 4);
+	samples.set(1, right, 4);
+
+	const olive::AudioLevelMeter::Stats stats =
+		olive::AudioLevelMeter::AnalyzeSampleBuffer(samples);
+
+	ASSERT_EQ(stats.channels.size(), 2);
+	EXPECT_DOUBLE_EQ(stats.channels.at(0).peak_linear, 1.0);
+	EXPECT_DOUBLE_EQ(stats.channels.at(1).peak_linear, 0.25);
+	EXPECT_NEAR(stats.channels.at(0).rms_linear,
+				std::sqrt((0.25 * 0.25 + 0.5 * 0.5 + 1.0) / 4.0), 0.0001);
+	EXPECT_NEAR(stats.channels.at(1).rms_linear,
+				std::sqrt((0.25 * 0.25 * 3.0) / 4.0), 0.0001);
+}
@@ -0,0 +1,76 @@
+#include <gtest/gtest.h>
+
+#include "audio/audiosynchronizer.h"
+
+TEST(AudioSynchronizer, PlacesCandidateBySourceStartTime)
+{
+	olive::AudioSynchronizer::SourceClip reference;
+	reference.source_start_time = olive::core::rational(100);
+	reference.has_source_start_time = true;
+
+	olive::AudioSynchronizer::SourceClip candidate;
+	candidate.source_start_time = olive::core::rational(112);
+	candidate.has_source_start_time = true;
+
+	const olive::AudioSynchronizer::Placement placement =
+		olive::AudioSynchronizer::PlaceBySourceTime(
+			reference, candidate, olive::core::rational(10));
+
+	ASSERT_TRUE(placement.valid);
+	EXPECT_EQ(placement.timeline_in, olive::core::rational(22));
+}
+
+TEST(AudioSynchronizer, AccountsForMediaInWhenPlacingBySourceTime)
+{
+	olive::AudioSynchronizer::SourceClip reference;
+	reference.source_start_time = olive::core::rational(100);
+	reference.media_in = olive::core::rational(2);
+	reference.has_source_start_time = true;
+
+	olive::AudioSynchronizer::SourceClip candidate;
+	candidate.source_start_time = olive::core::rational(100);
+	candidate.media_in = olive::core::rational(5);
+	candidate.has_source_start_time = true;
+
+	const olive::AudioSynchronizer::Placement placement =
+		olive::AudioSynchronizer::PlaceBySourceTime(
+			reference, candidate, olive::core::rational(20));
+
+	ASSERT_TRUE(placement.valid);
+	EXPECT_EQ(placement.timeline_in, olive::core::rational(23));
+}
+
+TEST(AudioSynchronizer, RejectsMissingSourceStartTime)
+{
+	olive::AudioSynchronizer::SourceClip reference;
+	reference.source_start_time = olive::core::rational(100);
+	reference.has_source_start_time = true;
+
+	olive::AudioSynchronizer::SourceClip candidate;
+
+	const olive::AudioSynchronizer::Placement placement =
+		olive::AudioSynchronizer::PlaceBySourceTime(
+			reference, candidate, olive::core::rational(10));
+
+	EXPECT_FALSE(placement.valid);
+}
+
+TEST(AudioSynchronizer, PlacesCandidateByWaveformOffset)
+{
+	const olive::AudioSynchronizer::Placement placement =
+		olive::AudioSynchronizer::PlaceByWaveformOffset(
+			olive::core::rational(10), 24000, 48000);
+
+	ASSERT_TRUE(placement.valid);
+	EXPECT_EQ(placement.timeline_in, olive::core::rational(21, 2));
+}
+
+TEST(AudioSynchronizer, SupportsCandidateLeadByWaveformOffset)
+{
+	const olive::AudioSynchronizer::Placement placement =
+		olive::AudioSynchronizer::PlaceByWaveformOffset(
+			olive::core::rational(10), -48000, 48000);
+
+	ASSERT_TRUE(placement.valid);
+	EXPECT_EQ(placement.timeline_in, olive::core::rational(9));
+}
@@ -0,0 +1,96 @@
+#include <gtest/gtest.h>
+
+#include "audio/audiowaveformsync.h"
+#include "olive/core/render/audioparams.h"
+#include "olive/core/render/samplebuffer.h"
+#include "olive/core/render/sampleformat.h"
+
+extern "C" {
+#include <libavutil/channel_layout.h>
+}
+
+namespace {
+
+olive::core::AudioParams MakeMonoParams()
+{
+	return olive::core::AudioParams(48000, AV_CH_LAYOUT_MONO,
+									olive::core::SampleFormat::F32P);
+}
+
+olive::core::SampleBuffer MakeBuffer(const QVector<float> &values)
+{
+	olive::core::SampleBuffer samples(MakeMonoParams(),
+									  static_cast<size_t>(values.size()));
+	float *data = samples.data(0);
+	for (int i = 0; i < values.size(); i++) {
+		data[i] = values.at(i);
+	}
+	return samples;
+}
+
+}
+
+TEST(AudioWaveformSync, ExtractsRmsEnvelope)
+{
+	olive::core::SampleBuffer samples =
+		MakeBuffer({ 1.0f, -1.0f, 0.5f, -0.5f, 0.0f, 0.0f });
+
+	const QVector<double> envelope =
+		olive::AudioWaveformSync::ExtractRmsEnvelope(samples, 2);
+
+	ASSERT_EQ(envelope.size(), 3);
+	EXPECT_NEAR(envelope.at(0), 1.0, 0.0001);
+	EXPECT_NEAR(envelope.at(1), 0.5, 0.0001);
+	EXPECT_NEAR(envelope.at(2), 0.0, 0.0001);
+}
+
+TEST(AudioWaveformSync, EstimatesCandidateLag)
+{
+	const QVector<float> reference_values = {
+		0.0f, 0.0f, 0.8f, 0.8f, 0.1f, 0.1f, 0.6f, 0.6f, 0.0f, 0.0f
+	};
+	const QVector<float> candidate_values = {
+		0.0f, 0.0f, 0.0f, 0.0f, 0.8f, 0.8f, 0.1f,
+		0.1f, 0.6f, 0.6f, 0.0f, 0.0f
+	};
+
+	const olive::AudioWaveformSync::OffsetResult result =
+		olive::AudioWaveformSync::EstimateOffset(
+			MakeBuffer(reference_values), MakeBuffer(candidate_values), 2, 8);
+
+	ASSERT_TRUE(result.valid);
+	EXPECT_EQ(result.offset_samples, 2);
+	EXPECT_GT(result.confidence, 0.99);
+}
+
+TEST(AudioWaveformSync, EstimatesCandidateLead)
+{
+	const QVector<float> reference_values = {
+		0.0f, 0.0f, 0.0f, 0.0f, 0.9f, 0.9f, 0.3f,
+		0.3f, 0.7f, 0.7f, 0.0f, 0.0f
+	};
+	const QVector<float> candidate_values = {
+		0.9f, 0.9f, 0.3f, 0.3f, 0.7f, 0.7f, 0.0f, 0.0f
+	};
+
+	const olive::AudioWaveformSync::OffsetResult result =
+		olive::AudioWaveformSync::EstimateOffset(
+			MakeBuffer(reference_values), MakeBuffer(candidate_values), 2, 4);
+
+	ASSERT_TRUE(result.valid);
+	EXPECT_EQ(result.offset_samples, -4);
+	EXPECT_GT(result.confidence, 0.99);
+}
+
+TEST(AudioWaveformSync, RejectsSilence)
+{
+	olive::core::SampleBuffer reference(MakeMonoParams(), size_t(16));
+	olive::core::SampleBuffer candidate(MakeMonoParams(), size_t(16));
+	reference.silence();
+	candidate.silence();
+
+	const olive::AudioWaveformSync::OffsetResult result =
+		olive::AudioWaveformSync::EstimateOffset(reference, candidate, 4, 16);
+
+	EXPECT_FALSE(result.valid);
+}
@@ -0,0 +1,86 @@
+#include <gtest/gtest.h>
+
+#include <QDir>
+#include <QTemporaryDir>
+
+#include "codec/timecodemetadata.h"
+#include "node/project/footage/footagedescription.h"
+
+TEST(TimecodeMetadata, ParsesNonDropFrameTimecode)
+{
+	const olive::TimecodeMetadata::SourceTime parsed =
+		olive::TimecodeMetadata::FromTimecodeString(
+			QStringLiteral("01:02:03:12"), olive::core::rational(1, 24));
+
+	ASSERT_TRUE(parsed.valid);
+	EXPECT_EQ(parsed.source, QStringLiteral("timecode"));
+	EXPECT_EQ(parsed.time, olive::core::rational(1 * 3600 + 2 * 60 + 3, 1) +
+							   olive::core::rational(12, 24));
+}
+
+TEST(TimecodeMetadata, ParsesDropFrameTimecode)
+{
+	const olive::TimecodeMetadata::SourceTime parsed =
+		olive::TimecodeMetadata::FromTimecodeString(
+			QStringLiteral("00:01:00;02"), olive::core::rational(1001, 30000));
+
+	ASSERT_TRUE(parsed.valid);
+	EXPECT_EQ(parsed.source, QStringLiteral("timecode"));
+	EXPECT_GT(parsed.time, olive::core::rational(59));
+	EXPECT_LT(parsed.time, olive::core::rational(61));
+}
+
+TEST(TimecodeMetadata, ParsesBwfTimeReference)
+{
+	const olive::TimecodeMetadata::SourceTime parsed =
+		olive::TimecodeMetadata::FromBwfTimeReference(
+			QStringLiteral("96000"), 48000);
+
+	ASSERT_TRUE(parsed.valid);
+	EXPECT_EQ(parsed.source, QStringLiteral("bwf_time_reference"));
+	EXPECT_EQ(parsed.time, olive::core::rational(2));
+}
+
+TEST(TimecodeMetadata, ParsesLargeBwfTimeReferenceWithoutTruncation)
+{
+	const olive::TimecodeMetadata::SourceTime parsed =
+		olive::TimecodeMetadata::FromBwfTimeReference(
+			QStringLiteral("4294967296"), 48000);
+
+	ASSERT_TRUE(parsed.valid);
+	EXPECT_GT(parsed.time, olive::core::rational(89478));
+	EXPECT_LT(parsed.time, olive::core::rational(89479));
+}
+
+TEST(TimecodeMetadata, RejectsInvalidMetadata)
+{
+	EXPECT_FALSE(olive::TimecodeMetadata::FromTimecodeString(
+					 QString(), olive::core::rational(1, 24))
+					 .valid);
+	EXPECT_FALSE(olive::TimecodeMetadata::FromBwfTimeReference(
+					 QStringLiteral("not-a-number"), 48000)
+					 .valid);
+	EXPECT_FALSE(olive::TimecodeMetadata::FromBwfTimeReference(
+					 QStringLiteral("123"), 0)
+					 .valid);
+}
+
+TEST(TimecodeMetadata, FootageDescriptionCachesSourceStartTime)
+{
+	QTemporaryDir dir;
+	ASSERT_TRUE(dir.isValid());
+	const QString path =
+		QDir(dir.path()).filePath(QStringLiteral("footage-cache.xml"));
+
+	olive::FootageDescription desc(QStringLiteral("ffmpeg"));
+	desc.SetSourceStartTime(olive::core::rational(96000, 48000),
+							QStringLiteral("bwf_time_reference"));
+	ASSERT_TRUE(desc.Save(path));
+
+	olive::FootageDescription loaded;
+	ASSERT_TRUE(loaded.Load(path));
+	ASSERT_TRUE(loaded.HasSourceStartTime());
+	EXPECT_EQ(loaded.source_start_time(), olive::core::rational(2));
+	EXPECT_EQ(loaded.source_start_time_source(),
+			  QStringLiteral("bwf_time_reference"));
+}