From f3389fe6bab04bbc64ddb19e9650f06b87bf4a67 Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Mon, 13 Jul 2026 10:54:12 +0800 Subject: [PATCH] Allow waveform sync with partially cached audio and add regression tests --- app/widget/timelinewidget/CMakeLists.txt | 2 + app/widget/timelinewidget/timelinewidget.cpp | 76 +--------- .../timelinewidgetwaveformsync.cpp | 126 ++++++++++++++++ .../timelinewidgetwaveformsync.h | 84 +++++++++++ tests/gtest/CMakeLists.txt | 1 + tests/gtest/timeline_waveform_sync_test.cpp | 142 ++++++++++++++++++ 6 files changed, 358 insertions(+), 73 deletions(-) create mode 100644 app/widget/timelinewidget/timelinewidgetwaveformsync.cpp create mode 100644 app/widget/timelinewidget/timelinewidgetwaveformsync.h create mode 100644 tests/gtest/timeline_waveform_sync_test.cpp diff --git a/app/widget/timelinewidget/CMakeLists.txt b/app/widget/timelinewidget/CMakeLists.txt index 44f5392c1..d662526ce 100644 --- a/app/widget/timelinewidget/CMakeLists.txt +++ b/app/widget/timelinewidget/CMakeLists.txt @@ -26,5 +26,7 @@ set(OLIVE_SOURCES widget/timelinewidget/timelinewidget.h widget/timelinewidget/timelinewidgetselections.cpp widget/timelinewidget/timelinewidgetselections.h + widget/timelinewidget/timelinewidgetwaveformsync.cpp + widget/timelinewidget/timelinewidgetwaveformsync.h PARENT_SCOPE ) diff --git a/app/widget/timelinewidget/timelinewidget.cpp b/app/widget/timelinewidget/timelinewidget.cpp index 59a1bca1b..74ec9b454 100644 --- a/app/widget/timelinewidget/timelinewidget.cpp +++ b/app/widget/timelinewidget/timelinewidget.cpp @@ -20,6 +20,7 @@ ***/ #include "timelinewidget.h" +#include "timelinewidgetwaveformsync.h" #include #include @@ -76,6 +77,8 @@ namespace olive #define super TimeBasedWidget +using namespace TimelineWaveformSync; + namespace { struct SourceSyncClip { @@ -84,13 +87,6 @@ struct SourceSyncClip { rational source_head; }; -struct WaveformSyncClip { - ClipBlock *clip = nullptr; - const AudioWaveformCache *waveform = nullptr; - TimeRange media_range; - int sample_rate = 0; -}; - bool GetSourceSyncClip(Block *block, SourceSyncClip *out) { ClipBlock *clip = dynamic_cast(block); @@ -124,44 +120,6 @@ QVector GetSelectedSourceSyncClips( return clips; } -bool GetWaveformSyncClip(Block *block, WaveformSyncClip *out) -{ - ClipBlock *clip = dynamic_cast(block); - if (!clip || !clip->waveform()) { - return false; - } - - const TimeRange media_range = clip->media_range(); - if (media_range.length().isNull()) { - return false; - } - - const AudioWaveformCache *waveform = clip->waveform(); - if (waveform->GetParameters().sample_rate() <= 0 || - !waveform->GetInvalidatedRanges(media_range).isEmpty()) { - return false; - } - - out->clip = clip; - out->waveform = waveform; - out->media_range = media_range; - out->sample_rate = waveform->GetParameters().sample_rate(); - return true; -} - -QVector GetSelectedWaveformSyncClips( - const QVector &blocks) -{ - QVector clips; - for (Block *block : blocks) { - WaveformSyncClip sync_clip; - if (GetWaveformSyncClip(block, &sync_clip)) { - clips.append(sync_clip); - } - } - return clips; -} - QVector GetSelectedProxyFootage(const QVector &blocks) { QVector footage; @@ -182,34 +140,6 @@ QVector GetSelectedProxyFootage(const QVector &blocks) return footage; } -QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, - int sample_rate, - size_t window_samples) -{ - QVector envelope; - if (sample_rate <= 0 || !window_samples) { - return envelope; - } - - const rational window_time(static_cast(window_samples), sample_rate); - for (rational t = clip.media_range.in(); t < clip.media_range.out(); - t += window_time) { - const rational length = qMin(window_time, clip.media_range.out() - t); - const AudioVisualWaveform::Sample summary = - clip.waveform->GetSummaryFromTime(t, length); - - double peak = 0.0; - for (const AudioVisualWaveform::SamplePerChannel &channel : summary) { - const double channel_peak = - std::max(std::abs(static_cast(channel.min)), - std::abs(static_cast(channel.max))); - peak = std::max(peak, channel_peak); - } - envelope.append(peak); - } - return envelope; -} - } // namespace TimelineWidget::TimelineWidget(QWidget *parent) diff --git a/app/widget/timelinewidget/timelinewidgetwaveformsync.cpp b/app/widget/timelinewidget/timelinewidgetwaveformsync.cpp new file mode 100644 index 000000000..5e3c040f8 --- /dev/null +++ b/app/widget/timelinewidget/timelinewidgetwaveformsync.cpp @@ -0,0 +1,126 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2022 Olive Team + Modifications Copyright (C) 2025 mikesolar + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#include "timelinewidgetwaveformsync.h" + +#include + +#include "node/block/clip/clip.h" +#include "render/audiowaveformcache.h" + +namespace olive +{ + +namespace TimelineWaveformSync +{ + +bool GetWaveformSyncClip(Block *block, WaveformSyncClip *out) +{ + ClipBlock *clip = dynamic_cast(block); + if (!clip || !clip->waveform()) { + return false; + } + + const TimeRange media_range = clip->media_range(); + if (media_range.length().isNull()) { + return false; + } + + const AudioWaveformCache *waveform = clip->waveform(); + if (waveform->GetParameters().sample_rate() <= 0) { + return false; + } + + // Allow waveform sync as long as at least some portion of the clip's + // media range has been validated. Requiring the entire range to be + // validated makes the menu item stay disabled for long clips and gives + // the appearance that "nothing happens" when the user tries to sync. + const TimeRangeList validated_ranges = + waveform->GetValidatedRanges().Intersects(media_range); + if (validated_ranges.isEmpty()) { + return false; + } + + out->clip = clip; + out->waveform = waveform; + out->media_range = media_range; + out->sample_rate = waveform->GetParameters().sample_rate(); + return true; +} + +QVector GetSelectedWaveformSyncClips( + const QVector &blocks) +{ + QVector clips; + for (Block *block : blocks) { + WaveformSyncClip sync_clip; + if (GetWaveformSyncClip(block, &sync_clip)) { + clips.append(sync_clip); + } + } + return clips; +} + +QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, + int sample_rate, + size_t window_samples) +{ + QVector envelope; + if (sample_rate <= 0 || !window_samples) { + return envelope; + } + + const rational window_time(static_cast(window_samples), sample_rate); + + // Only trust regions that have actually been validated. Unvalidated cache + // returns zero samples, which both drags the correlation score down and + // can produce false peaks if one clip happens to have more cached data + // than another. Using zero placeholders keeps every envelope aligned to + // the same absolute timeline. + const TimeRangeList validated_ranges = + clip.waveform->GetValidatedRanges().Intersects(clip.media_range); + + for (rational t = clip.media_range.in(); t < clip.media_range.out(); + t += window_time) { + const rational length = qMin(window_time, clip.media_range.out() - t); + const TimeRange window(t, t + length); + + double peak = 0.0; + if (validated_ranges.contains(window)) { + const AudioVisualWaveform::Sample summary = + clip.waveform->GetSummaryFromTime(t, length); + + for (const AudioVisualWaveform::SamplePerChannel &channel : summary) { + const double channel_peak = + std::max(std::abs(static_cast(channel.min)), + std::abs(static_cast(channel.max))); + peak = std::max(peak, channel_peak); + } + } + + envelope.append(peak); + } + return envelope; +} + +} // namespace TimelineWaveformSync + +} // namespace olive diff --git a/app/widget/timelinewidget/timelinewidgetwaveformsync.h b/app/widget/timelinewidget/timelinewidgetwaveformsync.h new file mode 100644 index 000000000..05f3c8d5d --- /dev/null +++ b/app/widget/timelinewidget/timelinewidgetwaveformsync.h @@ -0,0 +1,84 @@ +/*** + + Olive - Non-Linear Video Editor + Copyright (C) 2022 Olive Team + Modifications Copyright (C) 2025 mikesolar + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +***/ + +#ifndef TIMELINEWIDGETWAVEFORMSYNC_H +#define TIMELINEWIDGETWAVEFORMSYNC_H + +#include + +#include "node/block/block.h" +#include "olive/core/util/rational.h" +#include "olive/core/util/timerange.h" + +namespace olive +{ + +class AudioWaveformCache; +class ClipBlock; + +/** + * @brief Data required to synchronize a clip using its cached audio waveform. + */ +struct WaveformSyncClip { + ClipBlock *clip = nullptr; + const AudioWaveformCache *waveform = nullptr; + TimeRange media_range; + int sample_rate = 0; +}; + +/** + * @brief Helpers used by TimelineWidget for waveform-based clip synchronization. + * + * Kept in a separate unit so they can be exercised directly by unit tests. + */ +namespace TimelineWaveformSync +{ + +/** + * @brief Fill @p out with waveform-sync metadata for @p block if it is usable. + * + * A clip is considered usable as long as at least a portion of its media range + * has been validated in the waveform cache. Previously the whole range had to + * be validated, which made the context-menu action unavailable for long clips. + */ +bool GetWaveformSyncClip(Block *block, WaveformSyncClip *out); + +/** + * @brief Return all selected blocks that can be synchronized by waveform. + */ +QVector GetSelectedWaveformSyncClips( + const QVector &blocks); + +/** + * @brief Extract a peak envelope from the validated regions of a waveform cache. + * + * Windows that have not been cached yet are filled with zero so that every + * envelope stays aligned to the same absolute timeline. + */ +QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, + int sample_rate, + size_t window_samples); + +} // namespace TimelineWaveformSync + +} // namespace olive + +#endif // TIMELINEWIDGETWAVEFORMSYNC_H diff --git a/tests/gtest/CMakeLists.txt b/tests/gtest/CMakeLists.txt index a4030b827..67b887b41 100644 --- a/tests/gtest/CMakeLists.txt +++ b/tests/gtest/CMakeLists.txt @@ -72,6 +72,7 @@ add_executable(olive-gtest timecode_metadata_test.cpp timebased_widget_test.cpp timeline_coordinate_test.cpp + timeline_waveform_sync_test.cpp timeline_workarea_test.cpp ) diff --git a/tests/gtest/timeline_waveform_sync_test.cpp b/tests/gtest/timeline_waveform_sync_test.cpp new file mode 100644 index 000000000..84b4f086d --- /dev/null +++ b/tests/gtest/timeline_waveform_sync_test.cpp @@ -0,0 +1,142 @@ +/*** + Oak Video Editor - Regression test for TimelineWidget waveform sync + Copyright (C) 2026 Oak Team +***/ + +#include + +#include + +extern "C" { +#include +} + +#include "audio/audiovisualwaveform.h" +#include "node/block/clip/clip.h" +#include "node/node.h" +#include "node/project/footage/footage.h" +#include "olive/core/render/audioparams.h" +#include "olive/core/render/samplebuffer.h" +#include "olive/core/render/sampleformat.h" +#include "render/audiowaveformcache.h" +#include "widget/timelinewidget/timelinewidgetwaveformsync.h" + +using namespace olive; +using namespace olive::core; + +namespace { + +AudioParams MakeMonoParams(int sample_rate) +{ + return AudioParams(sample_rate, static_cast(AV_CH_LAYOUT_MONO), + SampleFormat::F32P); +} + +SampleBuffer MakeMonoBuffer(int sample_rate, float value, int seconds) +{ + SampleBuffer buf(MakeMonoParams(sample_rate), + static_cast(sample_rate * seconds)); + float *data = buf.data(0); + for (size_t i = 0; i < buf.sample_count(); i++) { + data[i] = value; + } + return buf; +} + +void WritePartialWaveform(AudioWaveformCache *cache, int sample_rate) +{ + const AudioParams params = MakeMonoParams(sample_rate); + cache->SetParameters(params); + + // Fill seconds [1,2) with a loud constant signal. + AudioVisualWaveform waveform; + waveform.set_channel_count(1); + SampleBuffer buf = MakeMonoBuffer(sample_rate, 1.0f, 1); + waveform.OverwriteSamples(buf, sample_rate, rational(1)); + + // Tell the cache that only the middle second is valid in a 3-second clip. + cache->WriteWaveform(TimeRange(1, 2), + TimeRangeList({ TimeRange(1, 2) }), + &waveform); +} + +} // namespace + +TEST(TimelineWaveformSync, ExtractEnvelopeUsesOnlyValidatedRanges) +{ + constexpr int kSampleRate = 48000; + constexpr size_t kWindowSamples = kSampleRate / 20; // 50 ms windows + + AudioWaveformCache cache; + WritePartialWaveform(&cache, kSampleRate); + + WaveformSyncClip clip; + clip.waveform = &cache; + clip.media_range = TimeRange(0, 3); + clip.sample_rate = kSampleRate; + + const QVector envelope = + TimelineWaveformSync::ExtractWaveformCacheEnvelope( + clip, kSampleRate, kWindowSamples); + + // 3 seconds at 20 windows per second == 60 windows. + EXPECT_EQ(envelope.size(), 60); + + // Window before the validated region should be silent. + EXPECT_DOUBLE_EQ(envelope.at(0), 0.0); + + // Windows inside the validated region should have a non-zero peak. + bool found_nonzero = false; + for (int i = 20; i < 40; ++i) { + if (envelope.at(i) > 0.0) { + found_nonzero = true; + break; + } + } + EXPECT_TRUE(found_nonzero); + + // Window after the validated region should also be silent. + EXPECT_DOUBLE_EQ(envelope.at(59), 0.0); +} + +TEST(TimelineWaveformSync, PartialCacheIsConsideredReady) +{ + constexpr int kSampleRate = 48000; + + Footage footage; + footage.SetValid(); + + AudioWaveformCache *cache = footage.waveform_cache(); + WritePartialWaveform(cache, kSampleRate); + + ClipBlock clip; + clip.set_length_and_media_out(rational(3)); + clip.set_media_in(rational(0)); + + Node::ConnectEdge(&footage, NodeInput(&clip, ClipBlock::kBufferIn)); + + WaveformSyncClip out; + EXPECT_TRUE(TimelineWaveformSync::GetWaveformSyncClip(&clip, &out)); + EXPECT_EQ(out.waveform, cache); + EXPECT_EQ(out.sample_rate, kSampleRate); + EXPECT_EQ(out.media_range, TimeRange(0, 3)); +} + +TEST(TimelineWaveformSync, EmptyCacheIsNotReady) +{ + Footage footage; + footage.SetValid(); + + AudioParams params = MakeMonoParams(48000); + footage.waveform_cache()->SetParameters(params); + + ClipBlock clip; + clip.set_length_and_media_out(rational(3)); + clip.set_media_in(rational(0)); + + Node::ConnectEdge( + &footage, NodeInput(&clip, ClipBlock::kBufferIn)); + + WaveformSyncClip out; + EXPECT_FALSE(TimelineWaveformSync::GetWaveformSyncClip(&clip, &out)); +}