oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sync: masked waveform correlation, stretch sync, manual start time

Browse Source 
- Waveform sync no longer treats uncached waveform regions as silence:
  the envelope extraction now reports a per-window validity mask and the
  correlation skips invalid windows on either side, improving accuracy
  for partially cached clips
- Add stretch/speed sync: AudioWaveformSync::EstimateStretchAndOffset
  searches a playback-rate range plus offset, and a new timeline
  context action 'Synchronize by Waveform (Adjust Speed)' applies the
  estimated rate as a clip speed change (with undo) when plain offset
  alignment is inconclusive
- Footage properties dialog gains a Source Start Time field so the
  value used by source-time sync can be viewed and edited manually
  instead of relying solely on auto-detected metadata; applied via an
  undo command, with Footage::ClearSourceStartTime() for removal
- Regression tests for masked correlation, stretch estimation, the
  envelope validity mask, and source-start-time set/clear
This commit is contained in:
Mike-Solar
2026-07-16 23:09:12 +08:00
parent 547c2480e0
commit caafac4203
13 changed files with 544 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/widget/timelinewidget/timelinewidget.cpp
+82 -14
View File
@@ -1023,6 +1023,17 @@ void TimelineWidget::SynchronizeSelectedClipsBySourceTime()
}
void TimelineWidget::SynchronizeSelectedClipsByWaveform()
{
SynchronizeSelectedClipsByWaveformInternal(false);
}
void TimelineWidget::SynchronizeSelectedClipsByWaveformWithSpeed()
{
SynchronizeSelectedClipsByWaveformInternal(true);
}
void TimelineWidget::SynchronizeSelectedClipsByWaveformInternal(
bool allow_speed)
{
if (!GetConnectedNode()) {
return;
@@ -1052,8 +1063,10 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform()
static_cast<int64_t>(sample_rate) * 10 * 60;
const int64_t max_offset_windows =
max_offset_samples / static_cast<int64_t>(window_samples);
const QVector<double> reference_envelope =
ExtractWaveformCacheEnvelope(reference, sample_rate, window_samples);
QVector<bool> reference_valid;
const QVector<double> reference_envelope = ExtractWaveformCacheEnvelope(
reference, sample_rate, window_samples, &reference_valid);
qDebug() << "TimelineWidget::SynchronizeSelectedClipsByWaveform: sample_rate="
<< sample_rate << "window_samples=" << window_samples
@@ -1063,27 +1076,62 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform()
struct SyncPlacement {
ClipBlock *clip = nullptr;
rational timeline_in;
double speed = 1.0;
};
QVector<SyncPlacement> placements;
placements.append({ reference.clip, reference.clip->in() });
placements.append({ reference.clip, reference.clip->in(), 1.0 });
for (const WaveformSyncClip &sync_clip : sync_clips) {
if (sync_clip.clip == reference.clip) {
continue;
}
QVector<bool> candidate_valid;
const QVector<double> candidate_envelope = ExtractWaveformCacheEnvelope(
sync_clip, sample_rate, window_samples);
const AudioWaveformSync::OffsetResult offset =
AudioWaveformSync::EstimateEnvelopeOffset(reference_envelope,
candidate_envelope,
window_samples,
max_offset_windows);
sync_clip, sample_rate, window_samples, &candidate_valid);
// Skip uncached (zero-filled) windows on both sides so partially
// cached waveforms don't drag the correlation down
AudioWaveformSync::OffsetResult offset =
AudioWaveformSync::EstimateEnvelopeOffset(
reference_envelope, candidate_envelope, reference_valid,
candidate_valid, window_samples, max_offset_windows);
double speed = 1.0;
if (allow_speed && (!offset.valid || offset.confidence < 0.6)) {
// Plain offset alignment is inconclusive; the clips may run at
// different speeds (e.g. 24fps vs 25fps pull-down). Search a
// rate range with a tighter offset radius to keep the search
// interactive.
const int64_t stretch_radius_windows = std::min<int64_t>(
max_offset_windows,
(static_cast<int64_t>(sample_rate) * 30) /
static_cast<int64_t>(window_samples));
const AudioWaveformSync::StretchOffsetResult stretch =
AudioWaveformSync::EstimateStretchAndOffset(
reference_envelope, candidate_envelope, reference_valid,
candidate_valid, window_samples, stretch_radius_windows,
0.75, 1.34, 0.005);
qDebug() << "TimelineWidget::SynchronizeSelectedClipsByWaveform: "
"stretch estimate valid="
<< stretch.valid << "rate=" << stretch.rate
<< "confidence=" << stretch.confidence;
if (stretch.valid &&
stretch.confidence > (offset.valid ? offset.confidence : 0.0)) {
speed = stretch.rate;
offset.valid = true;
offset.confidence = stretch.confidence;
offset.offset_samples = stretch.offset_samples;
}
}
qDebug() << "TimelineWidget::SynchronizeSelectedClipsByWaveform: candidate"
<< sync_clip.clip << "envelope_size="
<< candidate_envelope.size() << "offset_valid="
<< offset.valid << "offset_samples=" << offset.offset_samples
<< "confidence=" << offset.confidence;
<< candidate_envelope.size() << "offset_valid=" << offset.valid
<< "offset_samples=" << offset.offset_samples
<< "confidence=" << offset.confidence << "speed=" << speed;
if (!offset.valid) {
continue;
}
@@ -1095,7 +1143,7 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform()
<< "valid=" << placement.valid << "timeline_in="
<< placement.timeline_in.toDouble();
if (placement.valid && placement.timeline_in >= 0) {
placements.append({ sync_clip.clip, placement.timeline_in });
placements.append({ sync_clip.clip, placement.timeline_in, speed });
}
}
@@ -1111,6 +1159,13 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform()
for (const SyncPlacement &placement : placements) {
command->add_child(new TrackReplaceBlockWithGapCommand(
placement.clip->track(), placement.clip, false));
if (placement.speed != 1.0) {
command->add_child(new NodeParamSetStandardValueCommand(
NodeKeyframeTrackReference(
NodeInput(placement.clip, ClipBlock::kSpeedInput)),
placement.clip->speed() * placement.speed));
}
}
TimelineWidgetSelections new_selections;
@@ -1120,9 +1175,15 @@ void TimelineWidget::SynchronizeSelectedClipsByWaveform()
placement.clip->track()->Index(), placement.clip,
placement.timeline_in));
// A speed change scales the clip's timeline length accordingly
const rational placed_length =
placement.speed == 1.0 ?
placement.clip->length() :
rational::fromDouble(placement.clip->length().toDouble() /
placement.speed);
new_selections[placement.clip->track()->ToReference()].insert(
TimeRange(placement.timeline_in,
placement.timeline_in + placement.clip->length()));
placement.timeline_in + placed_length));
}
command->add_child(
@@ -1719,6 +1780,13 @@ void TimelineWidget::ShowContextMenu()
connect(sync_by_waveform, &QAction::triggered, this,
&TimelineWidget::SynchronizeSelectedClipsByWaveform);
QAction *sync_by_waveform_speed =
menu.addAction(tr("Synchronize by Waveform (Adjust Speed)"));
sync_by_waveform_speed->setEnabled(
GetSelectedWaveformSyncClips(selected).size() >= 2);
connect(sync_by_waveform_speed, &QAction::triggered, this,
&TimelineWidget::SynchronizeSelectedClipsByWaveformWithSpeed);
menu.addSeparator();
if (ClipBlock *clip = dynamic_cast<ClipBlock *>(selected.first())) {
app/widget/timelinewidget/timelinewidget.h
+4
View File
@@ -109,6 +109,8 @@ public:
void SynchronizeSelectedClipsByWaveform();
void SynchronizeSelectedClipsByWaveformWithSpeed();
void GenerateProxiesForSelectedClips();
void SetSelectedClipsProxyEnabled(bool enabled);
@@ -345,6 +347,8 @@ private:
bool PasteInternal(bool insert);
void SynchronizeSelectedClipsByWaveformInternal(bool allow_speed);
TimelineAndTrackView *AddTimelineAndTrackView(Qt::Alignment alignment);
QHash<Node *, Node *>
app/widget/timelinewidget/timelinewidgetwaveformsync.cpp
+15 -4
View File
@@ -81,20 +81,26 @@ GetSelectedWaveformSyncClips(const QVector<Block *> &blocks)
QVector<double> ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip,
int sample_rate,
size_t window_samples)
size_t window_samples,
QVector<bool> *valid_mask)
{
QVector<double> envelope;
if (sample_rate <= 0 || !window_samples) {
return envelope;
}
if (valid_mask) {
valid_mask->clear();
}
const rational window_time(static_cast<int>(window_samples), sample_rate);
// Only trust regions that have actually been validated. Unvalidated cache
// returns zero samples, which both drags the correlation score down and
// can produce false peaks if one clip happens to have more cached data
// than another. Using zero placeholders keeps every envelope aligned to
// the same absolute timeline.
// than another. Zero placeholders keep every envelope aligned to the same
// absolute timeline, while the validity mask lets the correlation skip
// those placeholders entirely.
const TimeRangeList validated_ranges =
clip.waveform->GetValidatedRanges().Intersects(clip.media_range);
@@ -103,8 +109,10 @@ QVector<double> ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip,
const rational length = qMin(window_time, clip.media_range.out() - t);
const TimeRange window(t, t + length);
const bool window_valid = validated_ranges.contains(window);
double peak = 0.0;
if (validated_ranges.contains(window)) {
if (window_valid) {
const AudioVisualWaveform::Sample summary =
clip.waveform->GetSummaryFromTime(t, length);
@@ -118,6 +126,9 @@ QVector<double> ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip,
}
envelope.append(peak);
if (valid_mask) {
valid_mask->append(window_valid);
}
}
return envelope;
}
app/widget/timelinewidget/timelinewidgetwaveformsync.h
+6 -2
View File
@@ -71,11 +71,15 @@ GetSelectedWaveformSyncClips(const QVector<Block *> &blocks);
* @brief Extract a peak envelope from the validated regions of a waveform cache.
*
* Windows that have not been cached yet are filled with zero so that every
* envelope stays aligned to the same absolute timeline.
* envelope stays aligned to the same absolute timeline; when @p valid_mask is
* provided it receives one flag per window marking whether the window was
* actually cached, allowing the correlation to skip uncached regions instead
* of treating them as silence.
*/
QVector<double> ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip,
int sample_rate,
size_t window_samples);
size_t window_samples,
QVector<bool> *valid_mask = nullptr);
} // namespace TimelineWaveformSync