From caafac42035859cdcc514c1a8887355157df5827 Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Thu, 16 Jul 2026 23:09:12 +0800 Subject: [PATCH] sync: masked waveform correlation, stretch sync, manual start time - Waveform sync no longer treats uncached waveform regions as silence: the envelope extraction now reports a per-window validity mask and the correlation skips invalid windows on either side, improving accuracy for partially cached clips - Add stretch/speed sync: AudioWaveformSync::EstimateStretchAndOffset searches a playback-rate range plus offset, and a new timeline context action 'Synchronize by Waveform (Adjust Speed)' applies the estimated rate as a clip speed change (with undo) when plain offset alignment is inconclusive - Footage properties dialog gains a Source Start Time field so the value used by source-time sync can be viewed and edited manually instead of relying solely on auto-detected metadata; applied via an undo command, with Footage::ClearSourceStartTime() for removal - Regression tests for masked correlation, stretch estimation, the envelope validity mask, and source-start-time set/clear --- app/audio/audiowaveformsync.cpp | 105 +++++++++++++++++- app/audio/audiowaveformsync.h | 45 ++++++++ .../footageproperties/footageproperties.cpp | 95 ++++++++++++++++ .../footageproperties/footageproperties.h | 34 ++++++ app/node/project/footage/footage.cpp | 7 ++ app/node/project/footage/footage.h | 5 + app/widget/timelinewidget/timelinewidget.cpp | 96 +++++++++++++--- app/widget/timelinewidget/timelinewidget.h | 4 + .../timelinewidgetwaveformsync.cpp | 19 +++- .../timelinewidgetwaveformsync.h | 8 +- tests/gtest/audio_waveform_sync_test.cpp | 93 ++++++++++++++++ tests/gtest/timecode_metadata_test.cpp | 29 +++++ tests/gtest/timeline_waveform_sync_test.cpp | 30 +++++ 13 files changed, 544 insertions(+), 26 deletions(-) diff --git a/app/audio/audiowaveformsync.cpp b/app/audio/audiowaveformsync.cpp index 48e30f90d..033b25767 100644 --- a/app/audio/audiowaveformsync.cpp +++ b/app/audio/audiowaveformsync.cpp @@ -86,12 +86,26 @@ AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateOffset( AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateEnvelopeOffset( const QVector &reference, const QVector &candidate, size_t window_samples, int64_t max_offset_windows) +{ + return EstimateEnvelopeOffset(reference, candidate, QVector(), + QVector(), window_samples, + max_offset_windows); +} + +AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateEnvelopeOffset( + const QVector &reference, const QVector &candidate, + const QVector &reference_valid, const QVector &candidate_valid, + size_t window_samples, int64_t max_offset_windows) { OffsetResult result; if (reference.isEmpty() || candidate.isEmpty() || !window_samples) { return result; } + const auto is_valid = [](const QVector &mask, int size, int index) { + return mask.size() != size || mask.at(index); + }; + double best_score = -2.0; int64_t best_lag = 0; @@ -106,23 +120,45 @@ AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateEnvelopeOffset( continue; } + // Only windows marked valid on both sides participate in the score double reference_mean = 0.0; double candidate_mean = 0.0; + int valid_count = 0; for (int i = 0; i < overlap; i++) { - reference_mean += reference.at(reference_start + i); - candidate_mean += candidate.at(candidate_start + i); + const int reference_index = reference_start + i; + const int candidate_index = candidate_start + i; + if (!is_valid(reference_valid, reference.size(), + reference_index) || + !is_valid(candidate_valid, candidate.size(), candidate_index)) { + continue; + } + reference_mean += reference.at(reference_index); + candidate_mean += candidate.at(candidate_index); + valid_count++; } - reference_mean /= static_cast(overlap); - candidate_mean /= static_cast(overlap); + + if (valid_count < 2) { + continue; + } + + reference_mean /= static_cast(valid_count); + candidate_mean /= static_cast(valid_count); double numerator = 0.0; double reference_energy = 0.0; double candidate_energy = 0.0; for (int i = 0; i < overlap; i++) { + const int reference_index = reference_start + i; + const int candidate_index = candidate_start + i; + if (!is_valid(reference_valid, reference.size(), + reference_index) || + !is_valid(candidate_valid, candidate.size(), candidate_index)) { + continue; + } const double reference_value = - reference.at(reference_start + i) - reference_mean; + reference.at(reference_index) - reference_mean; const double candidate_value = - candidate.at(candidate_start + i) - candidate_mean; + candidate.at(candidate_index) - candidate_mean; numerator += reference_value * candidate_value; reference_energy += reference_value * reference_value; candidate_energy += candidate_value * candidate_value; @@ -149,4 +185,61 @@ AudioWaveformSync::OffsetResult AudioWaveformSync::EstimateEnvelopeOffset( return result; } +AudioWaveformSync::StretchOffsetResult AudioWaveformSync::EstimateStretchAndOffset( + const QVector &reference, const QVector &candidate, + const QVector &reference_valid, const QVector &candidate_valid, + size_t window_samples, int64_t max_offset_windows, double min_rate, + double max_rate, double rate_step) +{ + StretchOffsetResult result; + if (reference.isEmpty() || candidate.isEmpty() || !window_samples || + min_rate <= 0.0 || max_rate < min_rate || rate_step <= 0.0) { + return result; + } + + double best_confidence = -2.0; + + for (double rate = min_rate; rate <= max_rate + rate_step * 0.5; + rate += rate_step) { + // Resample the candidate envelope so that window i of the resampled + // envelope corresponds to window i*rate of the original + const int resampled_size = + static_cast(candidate.size() / rate); + if (resampled_size < 2) { + continue; + } + + QVector resampled(resampled_size); + QVector resampled_valid(resampled_size); + for (int i = 0; i < resampled_size; i++) { + const double position = i * rate; + const int lower = static_cast(position); + const int upper = + std::min(lower + 1, static_cast(candidate.size()) - 1); + const double fraction = position - lower; + + resampled[i] = candidate.at(lower) * (1.0 - fraction) + + candidate.at(upper) * fraction; + + resampled_valid[i] = + (candidate_valid.size() != candidate.size() || + (candidate_valid.at(lower) && candidate_valid.at(upper))); + } + + const OffsetResult offset = EstimateEnvelopeOffset( + reference, resampled, reference_valid, resampled_valid, + window_samples, max_offset_windows); + + if (offset.valid && offset.confidence > best_confidence) { + best_confidence = offset.confidence; + result.valid = true; + result.rate = rate; + result.confidence = offset.confidence; + result.offset_samples = offset.offset_samples; + } + } + + return result; +} + } diff --git a/app/audio/audiowaveformsync.h b/app/audio/audiowaveformsync.h index 9aa9ee310..d8bb0035b 100644 --- a/app/audio/audiowaveformsync.h +++ b/app/audio/audiowaveformsync.h @@ -38,6 +38,16 @@ public: bool valid = false; }; + struct StretchOffsetResult { + // Playback rate the candidate must be played at to align with the + // reference (e.g. 2.0 = candidate runs at half speed and needs to be + // sped up 2x) + double rate = 1.0; + int64_t offset_samples = 0; + double confidence = 0.0; + bool valid = false; + }; + static QVector ExtractRmsEnvelope(const core::SampleBuffer &samples, size_t window_samples); @@ -50,6 +60,41 @@ public: const QVector &candidate, size_t window_samples, int64_t max_offset_windows); + + /** + * @brief Offset estimation that ignores windows flagged as invalid + * + * @p reference_valid and @p candidate_valid mark which envelope windows + * contain real data (e.g. actually cached waveform regions). Windows + * flagged false on either side are excluded from the correlation instead + * of being treated as silence, which improves accuracy when parts of the + * waveform cache have not been generated yet. Empty masks are treated as + * "all windows valid". + */ + static OffsetResult EstimateEnvelopeOffset(const QVector &reference, + const QVector &candidate, + const QVector &reference_valid, + const QVector &candidate_valid, + size_t window_samples, + int64_t max_offset_windows); + + /** + * @brief Estimates a playback-rate change plus offset aligning the + * candidate to the reference + * + * The candidate envelope is resampled at each candidate rate in + * [min_rate, max_rate] (step rate_step) and correlated against the + * reference. rate > 1 means the candidate runs slower than the reference + * and must be sped up. The search is O(rates * lags * overlap), so + * callers should bound max_offset_windows to a sensible range. + */ + static StretchOffsetResult + EstimateStretchAndOffset(const QVector &reference, + const QVector &candidate, + const QVector &reference_valid, + const QVector &candidate_valid, size_t window_samples, + int64_t max_offset_windows, double min_rate, + double max_rate, double rate_step); }; } diff --git a/app/dialog/footageproperties/footageproperties.cpp b/app/dialog/footageproperties/footageproperties.cpp index 113c64676..000272c0f 100644 --- a/app/dialog/footageproperties/footageproperties.cpp +++ b/app/dialog/footageproperties/footageproperties.cpp @@ -22,6 +22,7 @@ #include "footageproperties.h" #include +#include #include #include #include @@ -58,6 +59,48 @@ FootagePropertiesDialog::FootagePropertiesDialog(QWidget *parent, layout->addWidget(footage_name_field_, row, 1); row++; + // Manual source start time: audio/timecode sync relies on this value, + // which is otherwise only auto-detected from file metadata + layout->addWidget(new QLabel(tr("Source Start Time:")), row, 0); + + { + QHBoxLayout *start_time_layout = new QHBoxLayout(); + + source_start_time_enable_ = new QCheckBox(tr("Set")); + source_start_time_enable_->setChecked(footage_->HasSourceStartTime()); + start_time_layout->addWidget(source_start_time_enable_); + + source_start_time_spin_ = new QDoubleSpinBox(); + source_start_time_spin_->setRange(-86400.0, 86400.0); + source_start_time_spin_->setDecimals(3); + source_start_time_spin_->setSuffix(QStringLiteral(" s")); + source_start_time_spin_->setValue( + footage_->HasSourceStartTime() ? + footage_->source_start_time().toDouble() : + 0.0); + source_start_time_spin_->setEnabled( + source_start_time_enable_->isChecked()); + start_time_layout->addWidget(source_start_time_spin_, 1); + + QString detection_note; + if (footage_->HasSourceStartTime()) { + const QString &source = footage_->source_start_time_source(); + detection_note = + (source == QStringLiteral("manual")) ? + tr("(set manually)") : + tr("(auto-detected: %1)").arg(source); + } else { + detection_note = tr("(not detected)"); + } + start_time_layout->addWidget(new QLabel(detection_note)); + + connect(source_start_time_enable_, &QCheckBox::toggled, + source_start_time_spin_, &QDoubleSpinBox::setEnabled); + + layout->addLayout(start_time_layout, row, 1); + } + row++; + layout->addWidget(new QLabel(tr("Tracks:")), row, 0, 1, 2); row++; @@ -165,6 +208,18 @@ void FootagePropertiesDialog::accept() command->add_child(nrc); } + // Apply source start time changes + { + const bool new_enabled = source_start_time_enable_->isChecked(); + const rational new_time = + rational::fromDouble(source_start_time_spin_->value()); + if (new_enabled != footage_->HasSourceStartTime() || + (new_enabled && new_time != footage_->source_start_time())) { + command->add_child(new FootageSetSourceStartTimeCommand( + footage_, new_enabled, new_time, QStringLiteral("manual"))); + } + } + for (int i = 0; i < footage_->GetTotalStreamCount(); i++) { Track::Reference reference = footage_->GetReferenceFromRealIndex(i); bool new_stream_enabled = @@ -279,4 +334,44 @@ void FootagePropertiesDialog::StreamEnableChangeCommand::undo() } } +FootagePropertiesDialog::FootageSetSourceStartTimeCommand:: + FootageSetSourceStartTimeCommand(Footage *footage, bool enabled, + const rational &time, + const QString &source) + : footage_(footage) + , new_enabled_(enabled) + , new_time_(time) + , new_source_(source) +{ +} + +Project * +FootagePropertiesDialog::FootageSetSourceStartTimeCommand::GetRelevantProject() + const +{ + return footage_->project(); +} + +void FootagePropertiesDialog::FootageSetSourceStartTimeCommand::redo() +{ + old_enabled_ = footage_->HasSourceStartTime(); + old_time_ = footage_->source_start_time(); + old_source_ = footage_->source_start_time_source(); + + if (new_enabled_) { + footage_->SetSourceStartTime(new_time_, new_source_); + } else { + footage_->ClearSourceStartTime(); + } +} + +void FootagePropertiesDialog::FootageSetSourceStartTimeCommand::undo() +{ + if (old_enabled_) { + footage_->SetSourceStartTime(old_time_, old_source_); + } else { + footage_->ClearSourceStartTime(); + } +} + } diff --git a/app/dialog/footageproperties/footageproperties.h b/app/dialog/footageproperties/footageproperties.h index 37987063c..ea06e8615 100644 --- a/app/dialog/footageproperties/footageproperties.h +++ b/app/dialog/footageproperties/footageproperties.h @@ -79,6 +79,30 @@ private: bool new_enabled_; }; + class FootageSetSourceStartTimeCommand : public UndoCommand { + public: + FootageSetSourceStartTimeCommand(Footage *footage, bool enabled, + const rational &time, + const QString &source); + + virtual Project *GetRelevantProject() const override; + + protected: + virtual void redo() override; + virtual void undo() override; + + private: + Footage *footage_; + + bool new_enabled_; + rational new_time_; + QString new_source_; + + bool old_enabled_; + rational old_time_; + QString old_source_; + }; + /** * @brief Stack of widgets that changes based on whether the stream is a video or audio stream */ @@ -89,6 +113,16 @@ private: */ QLineEdit *footage_name_field_; + /** + * @brief Whether a manual source start time should be used + */ + QCheckBox *source_start_time_enable_; + + /** + * @brief Source start time in seconds + */ + QDoubleSpinBox *source_start_time_spin_; + /** * @brief Internal pointer to Media object (set in constructor) */ diff --git a/app/node/project/footage/footage.cpp b/app/node/project/footage/footage.cpp index ec5dc435c..fb2e65399 100644 --- a/app/node/project/footage/footage.cpp +++ b/app/node/project/footage/footage.cpp @@ -231,6 +231,13 @@ void Footage::SetSourceStartTime(const rational &time, const QString &source) has_source_start_time_ = true; } +void Footage::ClearSourceStartTime() +{ + source_start_time_ = rational(); + source_start_time_source_.clear(); + has_source_start_time_ = false; +} + void Footage::set_proxy_enabled(bool enabled) { if (proxy_enabled_ != enabled) { diff --git a/app/node/project/footage/footage.h b/app/node/project/footage/footage.h index effc6e50a..1557c78c6 100644 --- a/app/node/project/footage/footage.h +++ b/app/node/project/footage/footage.h @@ -173,6 +173,11 @@ public: void SetSourceStartTime(const rational &time, const QString &source); + /** + * @brief Removes any source start time (auto-detected or manual) + */ + void ClearSourceStartTime(); + bool proxy_enabled() const { return proxy_enabled_; diff --git a/app/widget/timelinewidget/timelinewidget.cpp b/app/widget/timelinewidget/timelinewidget.cpp index eacea6815..f91cfb072 100644 --- a/app/widget/timelinewidget/timelinewidget.cpp +++ b/app/widget/timelinewidget/timelinewidget.cpp @@ -1023,6 +1023,17 @@ void TimelineWidget::SynchronizeSelectedClipsBySourceTime() } void TimelineWidget::SynchronizeSelectedClipsByWaveform() +{ + SynchronizeSelectedClipsByWaveformInternal(false); +} + +void TimelineWidget::SynchronizeSelectedClipsByWaveformWithSpeed() +{ + SynchronizeSelectedClipsByWaveformInternal(true); +} + +void TimelineWidget::SynchronizeSelectedClipsByWaveformInternal( + bool allow_speed) { if (!GetConnectedNode()) { return; @@ -1052,8 +1063,10 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform() static_cast(sample_rate) * 10 * 60; const int64_t max_offset_windows = max_offset_samples / static_cast(window_samples); - const QVector reference_envelope = - ExtractWaveformCacheEnvelope(reference, sample_rate, window_samples); + + QVector reference_valid; + const QVector reference_envelope = ExtractWaveformCacheEnvelope( + reference, sample_rate, window_samples, &reference_valid); qDebug() << "TimelineWidget::SynchronizeSelectedClipsByWaveform: sample_rate=" << sample_rate << "window_samples=" << window_samples @@ -1063,27 +1076,62 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform() struct SyncPlacement { ClipBlock *clip = nullptr; rational timeline_in; + double speed = 1.0; }; QVector placements; - placements.append({ reference.clip, reference.clip->in() }); + placements.append({ reference.clip, reference.clip->in(), 1.0 }); for (const WaveformSyncClip &sync_clip : sync_clips) { if (sync_clip.clip == reference.clip) { continue; } + QVector candidate_valid; const QVector candidate_envelope = ExtractWaveformCacheEnvelope( - sync_clip, sample_rate, window_samples); - const AudioWaveformSync::OffsetResult offset = - AudioWaveformSync::EstimateEnvelopeOffset(reference_envelope, - candidate_envelope, - window_samples, - max_offset_windows); + sync_clip, sample_rate, window_samples, &candidate_valid); + + // Skip uncached (zero-filled) windows on both sides so partially + // cached waveforms don't drag the correlation down + AudioWaveformSync::OffsetResult offset = + AudioWaveformSync::EstimateEnvelopeOffset( + reference_envelope, candidate_envelope, reference_valid, + candidate_valid, window_samples, max_offset_windows); + + double speed = 1.0; + + if (allow_speed && (!offset.valid || offset.confidence < 0.6)) { + // Plain offset alignment is inconclusive; the clips may run at + // different speeds (e.g. 24fps vs 25fps pull-down). Search a + // rate range with a tighter offset radius to keep the search + // interactive. + const int64_t stretch_radius_windows = std::min( + max_offset_windows, + (static_cast(sample_rate) * 30) / + static_cast(window_samples)); + const AudioWaveformSync::StretchOffsetResult stretch = + AudioWaveformSync::EstimateStretchAndOffset( + reference_envelope, candidate_envelope, reference_valid, + candidate_valid, window_samples, stretch_radius_windows, + 0.75, 1.34, 0.005); + qDebug() << "TimelineWidget::SynchronizeSelectedClipsByWaveform: " + "stretch estimate valid=" + << stretch.valid << "rate=" << stretch.rate + << "confidence=" << stretch.confidence; + + if (stretch.valid && + stretch.confidence > (offset.valid ? offset.confidence : 0.0)) { + speed = stretch.rate; + offset.valid = true; + offset.confidence = stretch.confidence; + offset.offset_samples = stretch.offset_samples; + } + } + qDebug() << "TimelineWidget::SynchronizeSelectedClipsByWaveform: candidate" << sync_clip.clip << "envelope_size=" - << candidate_envelope.size() << "offset_valid=" - << offset.valid << "offset_samples=" << offset.offset_samples - << "confidence=" << offset.confidence; + << candidate_envelope.size() << "offset_valid=" << offset.valid + << "offset_samples=" << offset.offset_samples + << "confidence=" << offset.confidence << "speed=" << speed; if (!offset.valid) { continue; } @@ -1095,7 +1143,7 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform() << "valid=" << placement.valid << "timeline_in=" << placement.timeline_in.toDouble(); if (placement.valid && placement.timeline_in >= 0) { - placements.append({ sync_clip.clip, placement.timeline_in }); + placements.append({ sync_clip.clip, placement.timeline_in, speed }); } } @@ -1111,6 +1159,13 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform() for (const SyncPlacement &placement : placements) { command->add_child(new TrackReplaceBlockWithGapCommand( placement.clip->track(), placement.clip, false)); + + if (placement.speed != 1.0) { + command->add_child(new NodeParamSetStandardValueCommand( + NodeKeyframeTrackReference( + NodeInput(placement.clip, ClipBlock::kSpeedInput)), + placement.clip->speed() * placement.speed)); + } } TimelineWidgetSelections new_selections; @@ -1120,9 +1175,15 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform() placement.clip->track()->Index(), placement.clip, placement.timeline_in)); + // A speed change scales the clip's timeline length accordingly + const rational placed_length = + placement.speed == 1.0 ? + placement.clip->length() : + rational::fromDouble(placement.clip->length().toDouble() / + placement.speed); new_selections[placement.clip->track()->ToReference()].insert( TimeRange(placement.timeline_in, - placement.timeline_in + placement.clip->length())); + placement.timeline_in + placed_length)); } command->add_child( @@ -1719,6 +1780,13 @@ void TimelineWidget::ShowContextMenu() connect(sync_by_waveform, &QAction::triggered, this, &TimelineWidget::SynchronizeSelectedClipsByWaveform); + QAction *sync_by_waveform_speed = + menu.addAction(tr("Synchronize by Waveform (Adjust Speed)")); + sync_by_waveform_speed->setEnabled( + GetSelectedWaveformSyncClips(selected).size() >= 2); + connect(sync_by_waveform_speed, &QAction::triggered, this, + &TimelineWidget::SynchronizeSelectedClipsByWaveformWithSpeed); + menu.addSeparator(); if (ClipBlock *clip = dynamic_cast(selected.first())) { diff --git a/app/widget/timelinewidget/timelinewidget.h b/app/widget/timelinewidget/timelinewidget.h index dee2f56b9..268ff6d80 100644 --- a/app/widget/timelinewidget/timelinewidget.h +++ b/app/widget/timelinewidget/timelinewidget.h @@ -109,6 +109,8 @@ public: void SynchronizeSelectedClipsByWaveform(); + void SynchronizeSelectedClipsByWaveformWithSpeed(); + void GenerateProxiesForSelectedClips(); void SetSelectedClipsProxyEnabled(bool enabled); @@ -345,6 +347,8 @@ private: bool PasteInternal(bool insert); + void SynchronizeSelectedClipsByWaveformInternal(bool allow_speed); + TimelineAndTrackView *AddTimelineAndTrackView(Qt::Alignment alignment); QHash diff --git a/app/widget/timelinewidget/timelinewidgetwaveformsync.cpp b/app/widget/timelinewidget/timelinewidgetwaveformsync.cpp index f30faa085..07028e420 100644 --- a/app/widget/timelinewidget/timelinewidgetwaveformsync.cpp +++ b/app/widget/timelinewidget/timelinewidgetwaveformsync.cpp @@ -81,20 +81,26 @@ GetSelectedWaveformSyncClips(const QVector &blocks) QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, int sample_rate, - size_t window_samples) + size_t window_samples, + QVector *valid_mask) { QVector envelope; if (sample_rate <= 0 || !window_samples) { return envelope; } + if (valid_mask) { + valid_mask->clear(); + } + const rational window_time(static_cast(window_samples), sample_rate); // Only trust regions that have actually been validated. Unvalidated cache // returns zero samples, which both drags the correlation score down and // can produce false peaks if one clip happens to have more cached data - // than another. Using zero placeholders keeps every envelope aligned to - // the same absolute timeline. + // than another. Zero placeholders keep every envelope aligned to the same + // absolute timeline, while the validity mask lets the correlation skip + // those placeholders entirely. const TimeRangeList validated_ranges = clip.waveform->GetValidatedRanges().Intersects(clip.media_range); @@ -103,8 +109,10 @@ QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, const rational length = qMin(window_time, clip.media_range.out() - t); const TimeRange window(t, t + length); + const bool window_valid = validated_ranges.contains(window); + double peak = 0.0; - if (validated_ranges.contains(window)) { + if (window_valid) { const AudioVisualWaveform::Sample summary = clip.waveform->GetSummaryFromTime(t, length); @@ -118,6 +126,9 @@ QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, } envelope.append(peak); + if (valid_mask) { + valid_mask->append(window_valid); + } } return envelope; } diff --git a/app/widget/timelinewidget/timelinewidgetwaveformsync.h b/app/widget/timelinewidget/timelinewidgetwaveformsync.h index 4a77cee61..619db973a 100644 --- a/app/widget/timelinewidget/timelinewidgetwaveformsync.h +++ b/app/widget/timelinewidget/timelinewidgetwaveformsync.h @@ -71,11 +71,15 @@ GetSelectedWaveformSyncClips(const QVector &blocks); * @brief Extract a peak envelope from the validated regions of a waveform cache. * * Windows that have not been cached yet are filled with zero so that every - * envelope stays aligned to the same absolute timeline. + * envelope stays aligned to the same absolute timeline; when @p valid_mask is + * provided it receives one flag per window marking whether the window was + * actually cached, allowing the correlation to skip uncached regions instead + * of treating them as silence. */ QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, int sample_rate, - size_t window_samples); + size_t window_samples, + QVector *valid_mask = nullptr); } // namespace TimelineWaveformSync diff --git a/tests/gtest/audio_waveform_sync_test.cpp b/tests/gtest/audio_waveform_sync_test.cpp index 26acc60ec..e5b12d519 100644 --- a/tests/gtest/audio_waveform_sync_test.cpp +++ b/tests/gtest/audio_waveform_sync_test.cpp @@ -87,3 +87,96 @@ TEST(AudioWaveformSync, RejectsSilence) EXPECT_FALSE(result.valid); } + +TEST(AudioWaveformSync, MaskedEstimationIgnoresInvalidWindows) +{ + const QVector reference = { 0.5, 0.5, 0.5, 0.5, 0.9, 0.1, + 0.7, 0.2, 0.8, 0.3, 0.6, 0.4 }; + + // Candidate is the reference delayed by 3 windows, but windows 3..7 were + // never cached (zeroed out) and are flagged invalid + QVector candidate(12, 0.0); + QVector candidate_valid(12, true); + for (int i = 0; i + 3 < candidate.size(); i++) { + candidate[i + 3] = reference.at(i); + } + for (int i = 3; i <= 7; i++) { + candidate[i] = 0.0; + candidate_valid[i] = false; + } + + const olive::AudioWaveformSync::OffsetResult unmasked = + olive::AudioWaveformSync::EstimateEnvelopeOffset(reference, candidate, 1, + 8); + const olive::AudioWaveformSync::OffsetResult masked = + olive::AudioWaveformSync::EstimateEnvelopeOffset( + reference, candidate, QVector(), candidate_valid, 1, 8); + + ASSERT_TRUE(masked.valid); + EXPECT_EQ(masked.offset_samples, 3); + EXPECT_GT(masked.confidence, 0.99); + + // Ignoring the uncached placeholder windows must not make the estimate + // worse than treating them as silence + if (unmasked.valid) { + EXPECT_GE(masked.confidence, unmasked.confidence); + } +} + +TEST(AudioWaveformSync, EstimatesStretchAndOffset) +{ + // 20-window reference pattern + const QVector reference = { 0.1, 0.9, 0.2, 0.8, 0.3, + 0.7, 0.4, 0.6, 0.5, 1.0, + 0.15, 0.85, 0.25, 0.75, 0.35, + 0.65, 0.45, 0.55, 0.95, 0.05 }; + + // Candidate runs at half speed (each window duplicated) and is delayed by + // 6 candidate windows: candidate[j] = reference[(j-6)/2] + QVector candidate(6 + 2 * reference.size(), 0.0); + for (int i = 0; i < reference.size(); i++) { + candidate[6 + 2 * i] = reference.at(i); + candidate[6 + 2 * i + 1] = reference.at(i); + } + + const olive::AudioWaveformSync::StretchOffsetResult result = + olive::AudioWaveformSync::EstimateStretchAndOffset( + reference, candidate, QVector(), QVector(), 1, 12, + 0.8, 2.5, 0.005); + + ASSERT_TRUE(result.valid); + EXPECT_NEAR(result.rate, 2.0, 0.01); + // After resampling at 2x, the candidate lags the reference by 3 windows + EXPECT_EQ(result.offset_samples, 3); + EXPECT_GT(result.confidence, 0.95); +} + +TEST(AudioWaveformSync, StretchEstimationRejectsSilence) +{ + const QVector silence(16, 0.0); + + const olive::AudioWaveformSync::StretchOffsetResult result = + olive::AudioWaveformSync::EstimateStretchAndOffset( + silence, silence, QVector(), QVector(), 1, 8, 0.5, + 2.0, 0.1); + + EXPECT_FALSE(result.valid); +} + +TEST(AudioWaveformSync, StretchEstimationRejectsInvalidParameters) +{ + const QVector envelope = { 0.5, 0.6, 0.7, 0.8 }; + + EXPECT_FALSE(olive::AudioWaveformSync::EstimateStretchAndOffset( + envelope, envelope, QVector(), QVector(), 1, + 8, 0.0, 2.0, 0.1) + .valid); + EXPECT_FALSE(olive::AudioWaveformSync::EstimateStretchAndOffset( + envelope, envelope, QVector(), QVector(), 1, + 8, 2.0, 0.5, 0.1) + .valid); + EXPECT_FALSE(olive::AudioWaveformSync::EstimateStretchAndOffset( + envelope, envelope, QVector(), QVector(), 1, + 8, 0.5, 2.0, 0.0) + .valid); +} diff --git a/tests/gtest/timecode_metadata_test.cpp b/tests/gtest/timecode_metadata_test.cpp index cb0b58410..1090eda47 100644 --- a/tests/gtest/timecode_metadata_test.cpp +++ b/tests/gtest/timecode_metadata_test.cpp @@ -129,3 +129,32 @@ TEST(TimecodeMetadata, FootagePersistsSourceStartTime) EXPECT_EQ(footage.source_start_time(), olive::core::rational(3600)); EXPECT_EQ(footage.source_start_time_source(), QStringLiteral("timecode")); } + +TEST(TimecodeMetadata, FootageClearSourceStartTime) +{ + olive::Footage footage; + footage.SetSourceStartTime(olive::core::rational(3600), + QStringLiteral("manual")); + ASSERT_TRUE(footage.HasSourceStartTime()); + + footage.ClearSourceStartTime(); + + EXPECT_FALSE(footage.HasSourceStartTime()); + EXPECT_EQ(footage.source_start_time(), olive::core::rational()); + EXPECT_TRUE(footage.source_start_time_source().isEmpty()); +} + +TEST(TimecodeMetadata, FootageSetSourceStartTimeOverridesPreviousValue) +{ + olive::Footage footage; + footage.SetSourceStartTime(olive::core::rational(3600), + QStringLiteral("timecode")); + + // A manual edit replaces both the value and the recorded source + footage.SetSourceStartTime(olive::core::rational(1800), + QStringLiteral("manual")); + + EXPECT_TRUE(footage.HasSourceStartTime()); + EXPECT_EQ(footage.source_start_time(), olive::core::rational(1800)); + EXPECT_EQ(footage.source_start_time_source(), QStringLiteral("manual")); +} diff --git a/tests/gtest/timeline_waveform_sync_test.cpp b/tests/gtest/timeline_waveform_sync_test.cpp index 76e241992..e73a3f5e0 100644 --- a/tests/gtest/timeline_waveform_sync_test.cpp +++ b/tests/gtest/timeline_waveform_sync_test.cpp @@ -95,6 +95,36 @@ TEST(TimelineWaveformSync, ExtractEnvelopeUsesOnlyValidatedRanges) EXPECT_DOUBLE_EQ(envelope.at(59), 0.0); } +TEST(TimelineWaveformSync, ExtractEnvelopeReportsValidityMask) +{ + constexpr int kSampleRate = 48000; + constexpr size_t kWindowSamples = kSampleRate / 20; // 50 ms windows + + AudioWaveformCache cache; + WritePartialWaveform(&cache, kSampleRate); + + WaveformSyncClip clip; + clip.waveform = &cache; + clip.media_range = TimeRange(0, 3); + clip.sample_rate = kSampleRate; + + QVector valid_mask; + const QVector envelope = + TimelineWaveformSync::ExtractWaveformCacheEnvelope( + clip, kSampleRate, kWindowSamples, &valid_mask); + + // One flag per envelope window + ASSERT_EQ(valid_mask.size(), envelope.size()); + + // Windows outside the validated second are flagged invalid, windows + // inside it are flagged valid + EXPECT_FALSE(valid_mask.at(0)); + EXPECT_FALSE(valid_mask.at(59)); + for (int i = 20; i < 40; ++i) { + EXPECT_TRUE(valid_mask.at(i)); + } +} + TEST(TimelineWaveformSync, PartialCacheIsConsideredReady) { constexpr int kSampleRate = 48000;