diff --git a/app/audio/audiowaveformsync.h b/app/audio/audiowaveformsync.h index 35f3c1ecb..539b17d1c 100644 --- a/app/audio/audiowaveformsync.h +++ b/app/audio/audiowaveformsync.h @@ -46,7 +46,6 @@ public: size_t window_samples, int64_t max_offset_samples); -private: static OffsetResult EstimateEnvelopeOffset(const QVector &reference, const QVector &candidate, size_t window_samples, diff --git a/app/node/project/footage/footage.cpp b/app/node/project/footage/footage.cpp index 186e82014..4c508369c 100644 --- a/app/node/project/footage/footage.cpp +++ b/app/node/project/footage/footage.cpp @@ -44,6 +44,7 @@ const QString Footage::kFilenameInput = QStringLiteral("file_in"); Footage::Footage(const QString &filename) : ViewerOutput(false, false) , timestamp_(0) + , has_source_start_time_(false) , valid_(false) , cancelled_(nullptr) , total_stream_count_(0) @@ -134,6 +135,10 @@ void Footage::Clear() // Clear decoder link decoder_.clear(); + has_source_start_time_ = false; + source_start_time_ = rational(); + source_start_time_source_.clear(); + // Clear total stream count total_stream_count_ = 0; @@ -212,6 +217,13 @@ const QString &Footage::decoder() const return decoder_; } +void Footage::SetSourceStartTime(const rational &time, const QString &source) +{ + source_start_time_ = time; + source_start_time_source_ = source; + has_source_start_time_ = true; +} + QString Footage::DescribeVideoStream(const VideoParams ¶ms) { if (params.video_type() == VideoParams::kVideoTypeStill) { @@ -461,6 +473,28 @@ bool Footage::LoadCustom(QXmlStreamReader *reader, SerializedData *data) while (XMLReadNextStartElement(reader)) { if (reader->name() == QStringLiteral("timestamp")) { this->set_timestamp(reader->readElementText().toLongLong()); + } else if (reader->name() == QStringLiteral("sourcestarttime")) { + QString source; + { + XMLAttributeLoop(reader, attr) + { + if (attr.name() == QStringLiteral("source")) { + source = attr.value().toString(); + } + } + } + + const QStringList split = reader->readElementText().split('/'); + if (split.size() == 2) { + bool numerator_ok = false; + bool denominator_ok = false; + const int numerator = split.at(0).toInt(&numerator_ok); + const int denominator = split.at(1).toInt(&denominator_ok); + if (numerator_ok && denominator_ok && denominator) { + SetSourceStartTime(rational(numerator, denominator), + source); + } + } } else if (reader->name() == QStringLiteral("viewer")) { if (!ViewerOutput::LoadCustom(reader, data)) { return false; @@ -478,6 +512,16 @@ void Footage::SaveCustom(QXmlStreamWriter *writer) const writer->writeTextElement(QStringLiteral("timestamp"), QString::number(this->timestamp())); + if (has_source_start_time_) { + writer->writeStartElement(QStringLiteral("sourcestarttime")); + writer->writeAttribute(QStringLiteral("source"), + source_start_time_source_); + writer->writeCharacters(QStringLiteral("%1/%2").arg( + QString::number(source_start_time_.numerator()), + QString::number(source_start_time_.denominator()))); + writer->writeEndElement(); + } + writer->writeStartElement(QStringLiteral("viewer")); ViewerOutput::SaveCustom(writer); @@ -587,6 +631,10 @@ void Footage::Reprobe() } total_stream_count_ = footage_info.GetStreamCount(); + if (footage_info.HasSourceStartTime()) { + SetSourceStartTime(footage_info.source_start_time(), + footage_info.source_start_time_source()); + } SetValid(); } diff --git a/app/node/project/footage/footage.h b/app/node/project/footage/footage.h index c3332e22c..13b97b308 100644 --- a/app/node/project/footage/footage.h +++ b/app/node/project/footage/footage.h @@ -155,6 +155,23 @@ public: */ const QString &decoder() const; + bool HasSourceStartTime() const + { + return has_source_start_time_; + } + + const rational &source_start_time() const + { + return source_start_time_; + } + + const QString &source_start_time_source() const + { + return source_start_time_source_; + } + + void SetSourceStartTime(const rational &time, const QString &source); + static QString DescribeVideoStream(const VideoParams ¶ms); static QString DescribeAudioStream(const AudioParams ¶ms); static QString DescribeSubtitleStream(const SubtitleParams ¶ms); @@ -213,6 +230,12 @@ private: */ QString decoder_; + rational source_start_time_; + + QString source_start_time_source_; + + bool has_source_start_time_; + bool valid_; CancelAtom *cancelled_; diff --git a/app/widget/timelinewidget/timelinewidget.cpp b/app/widget/timelinewidget/timelinewidget.cpp index d71e13c6c..f6ed3be99 100644 --- a/app/widget/timelinewidget/timelinewidget.cpp +++ b/app/widget/timelinewidget/timelinewidget.cpp @@ -21,18 +21,24 @@ #include "timelinewidget.h" +#include #include +#include #include #include #include +#include "audio/audiosynchronizer.h" +#include "audio/audiowaveformsync.h" #include "core.h" #include "common/range.h" #include "dialog/sequence/sequence.h" #include "dialog/speedduration/speeddurationdialog.h" #include "node/block/transition/transition.h" #include "node/nodeundo.h" +#include "node/project/footage/footage.h" #include "node/project/serializer/serializer.h" +#include "render/audiowaveformcache.h" #include "task/project/import/import.h" #include "timeline/timelineundogeneral.h" #include "timeline/timelineundopointer.h" @@ -63,6 +69,122 @@ namespace olive #define super TimeBasedWidget +namespace { + +struct SourceSyncClip { + ClipBlock *clip = nullptr; + AudioSynchronizer::SourceClip source; + rational source_head; +}; + +struct WaveformSyncClip { + ClipBlock *clip = nullptr; + const AudioWaveformCache *waveform = nullptr; + TimeRange media_range; + int sample_rate = 0; +}; + +bool GetSourceSyncClip(Block *block, SourceSyncClip *out) +{ + ClipBlock *clip = dynamic_cast(block); + if (!clip) { + return false; + } + + Footage *footage = dynamic_cast(clip->connected_viewer()); + if (!footage || !footage->HasSourceStartTime()) { + return false; + } + + out->clip = clip; + out->source.source_start_time = footage->source_start_time(); + out->source.media_in = clip->media_in(); + out->source.has_source_start_time = true; + out->source_head = out->source.source_start_time + out->source.media_in; + return true; +} + +QVector GetSelectedSourceSyncClips( + const QVector &blocks) +{ + QVector clips; + for (Block *block : blocks) { + SourceSyncClip sync_clip; + if (GetSourceSyncClip(block, &sync_clip)) { + clips.append(sync_clip); + } + } + return clips; +} + +bool GetWaveformSyncClip(Block *block, WaveformSyncClip *out) +{ + ClipBlock *clip = dynamic_cast(block); + if (!clip || !clip->waveform()) { + return false; + } + + const TimeRange media_range = clip->media_range(); + if (media_range.length().isNull()) { + return false; + } + + const AudioWaveformCache *waveform = clip->waveform(); + if (waveform->GetParameters().sample_rate() <= 0 || + !waveform->GetInvalidatedRanges(media_range).isEmpty()) { + return false; + } + + out->clip = clip; + out->waveform = waveform; + out->media_range = media_range; + out->sample_rate = waveform->GetParameters().sample_rate(); + return true; +} + +QVector GetSelectedWaveformSyncClips( + const QVector &blocks) +{ + QVector clips; + for (Block *block : blocks) { + WaveformSyncClip sync_clip; + if (GetWaveformSyncClip(block, &sync_clip)) { + clips.append(sync_clip); + } + } + return clips; +} + +QVector ExtractWaveformCacheEnvelope(const WaveformSyncClip &clip, + int sample_rate, + size_t window_samples) +{ + QVector envelope; + if (sample_rate <= 0 || !window_samples) { + return envelope; + } + + const rational window_time(static_cast(window_samples), sample_rate); + for (rational t = clip.media_range.in(); t < clip.media_range.out(); + t += window_time) { + const rational length = qMin(window_time, clip.media_range.out() - t); + const AudioVisualWaveform::Sample summary = + clip.waveform->GetSummaryFromTime(t, length); + + double peak = 0.0; + for (const AudioVisualWaveform::SamplePerChannel &channel : summary) { + const double channel_peak = + std::max(std::abs(static_cast(channel.min)), + std::abs(static_cast(channel.max))); + peak = std::max(peak, channel_peak); + } + envelope.append(peak); + } + return envelope; +} + +} // namespace + TimelineWidget::TimelineWidget(QWidget *parent) : super(true, true, parent) , rubberband_(QRubberBand::Rectangle, this) @@ -874,6 +996,162 @@ void TimelineWidget::ShowSpeedDurationDialogForSelectedClips() } } +void TimelineWidget::SynchronizeSelectedClipsBySourceTime() +{ + if (!GetConnectedNode()) { + return; + } + + const QVector sync_clips = + GetSelectedSourceSyncClips(selected_blocks_); + if (sync_clips.size() < 2) { + return; + } + + SourceSyncClip reference = sync_clips.first(); + rational anchor_timeline_in = sync_clips.first().clip->in(); + for (const SourceSyncClip &sync_clip : sync_clips) { + if (sync_clip.source_head < reference.source_head) { + reference = sync_clip; + } + if (sync_clip.clip->in() < anchor_timeline_in) { + anchor_timeline_in = sync_clip.clip->in(); + } + } + + struct SyncPlacement { + ClipBlock *clip = nullptr; + rational timeline_in; + }; + + QVector placements; + for (const SourceSyncClip &sync_clip : sync_clips) { + const AudioSynchronizer::Placement placement = + AudioSynchronizer::PlaceBySourceTime( + reference.source, sync_clip.source, anchor_timeline_in); + if (placement.valid) { + placements.append({ sync_clip.clip, placement.timeline_in }); + } + } + + if (placements.size() < 2) { + return; + } + + MultiUndoCommand *command = new MultiUndoCommand(); + for (const SyncPlacement &placement : placements) { + command->add_child(new TrackReplaceBlockWithGapCommand( + placement.clip->track(), placement.clip, false)); + } + + TimelineWidgetSelections new_selections; + for (const SyncPlacement &placement : placements) { + command->add_child(new TrackPlaceBlockCommand( + sequence()->track_list(placement.clip->track()->type()), + placement.clip->track()->Index(), placement.clip, + placement.timeline_in)); + + new_selections[placement.clip->track()->ToReference()].insert( + TimeRange(placement.timeline_in, + placement.timeline_in + placement.clip->length())); + } + + command->add_child( + new SetSelectionsCommand(this, new_selections, GetSelections())); + + Core::instance()->undo_stack()->push( + command, tr("Synchronize Clips by Source Time")); +} + +void TimelineWidget::SynchronizeSelectedClipsByWaveform() +{ + if (!GetConnectedNode()) { + return; + } + + const QVector sync_clips = + GetSelectedWaveformSyncClips(selected_blocks_); + if (sync_clips.size() < 2) { + return; + } + + WaveformSyncClip reference = sync_clips.first(); + for (const WaveformSyncClip &sync_clip : sync_clips) { + if (sync_clip.clip->in() < reference.clip->in()) { + reference = sync_clip; + } + } + + const int sample_rate = reference.sample_rate; + const size_t window_samples = + static_cast(std::max(1, sample_rate / 20)); + const int64_t max_offset_samples = + static_cast(sample_rate) * 10 * 60; + const int64_t max_offset_windows = + max_offset_samples / static_cast(window_samples); + const QVector reference_envelope = + ExtractWaveformCacheEnvelope(reference, sample_rate, window_samples); + + struct SyncPlacement { + ClipBlock *clip = nullptr; + rational timeline_in; + }; + + QVector placements; + placements.append({ reference.clip, reference.clip->in() }); + for (const WaveformSyncClip &sync_clip : sync_clips) { + if (sync_clip.clip == reference.clip) { + continue; + } + + const QVector candidate_envelope = + ExtractWaveformCacheEnvelope(sync_clip, sample_rate, + window_samples); + const AudioWaveformSync::OffsetResult offset = + AudioWaveformSync::EstimateEnvelopeOffset( + reference_envelope, candidate_envelope, window_samples, + max_offset_windows); + if (!offset.valid) { + continue; + } + + const AudioSynchronizer::Placement placement = + AudioSynchronizer::PlaceByWaveformOffset( + reference.clip->in(), offset.offset_samples, sample_rate); + if (placement.valid && placement.timeline_in >= 0) { + placements.append({ sync_clip.clip, placement.timeline_in }); + } + } + + if (placements.size() < 2) { + return; + } + + MultiUndoCommand *command = new MultiUndoCommand(); + for (const SyncPlacement &placement : placements) { + command->add_child(new TrackReplaceBlockWithGapCommand( + placement.clip->track(), placement.clip, false)); + } + + TimelineWidgetSelections new_selections; + for (const SyncPlacement &placement : placements) { + command->add_child(new TrackPlaceBlockCommand( + sequence()->track_list(placement.clip->track()->type()), + placement.clip->track()->Index(), placement.clip, + placement.timeline_in)); + + new_selections[placement.clip->track()->ToReference()].insert( + TimeRange(placement.timeline_in, + placement.timeline_in + placement.clip->length())); + } + + command->add_child( + new SetSelectionsCommand(this, new_selections, GetSelections())); + + Core::instance()->undo_stack()->push( + command, tr("Synchronize Clips by Waveform")); +} + void TimelineWidget::RecordingCallback(const QString &filename, const TimeRange &time, const Track::Reference &track) @@ -1332,6 +1610,22 @@ void TimelineWidget::ShowContextMenu() menu.addSeparator(); + QAction *sync_by_source_time = + menu.addAction(tr("Synchronize by Source Time")); + sync_by_source_time->setEnabled( + GetSelectedSourceSyncClips(selected).size() >= 2); + connect(sync_by_source_time, &QAction::triggered, this, + &TimelineWidget::SynchronizeSelectedClipsBySourceTime); + + QAction *sync_by_waveform = + menu.addAction(tr("Synchronize by Waveform")); + sync_by_waveform->setEnabled( + GetSelectedWaveformSyncClips(selected).size() >= 2); + connect(sync_by_waveform, &QAction::triggered, this, + &TimelineWidget::SynchronizeSelectedClipsByWaveform); + + menu.addSeparator(); + if (ClipBlock *clip = dynamic_cast(selected.first())) { { Menu *cache_menu = new Menu(tr("Cache"), &menu); diff --git a/app/widget/timelinewidget/timelinewidget.h b/app/widget/timelinewidget/timelinewidget.h index 2f3f27df6..366834834 100644 --- a/app/widget/timelinewidget/timelinewidget.h +++ b/app/widget/timelinewidget/timelinewidget.h @@ -105,6 +105,10 @@ public: void ShowSpeedDurationDialogForSelectedClips(); + void SynchronizeSelectedClipsBySourceTime(); + + void SynchronizeSelectedClipsByWaveform(); + void RecordingCallback(const QString &filename, const TimeRange &time, const Track::Reference &track); diff --git a/tests/gtest/timecode_metadata_test.cpp b/tests/gtest/timecode_metadata_test.cpp index d646d39f2..c08e07d08 100644 --- a/tests/gtest/timecode_metadata_test.cpp +++ b/tests/gtest/timecode_metadata_test.cpp @@ -2,8 +2,11 @@ #include #include +#include +#include #include "codec/timecodemetadata.h" +#include "node/project/footage/footage.h" #include "node/project/footage/footagedescription.h" TEST(TimecodeMetadata, ParsesNonDropFrameTimecode) @@ -84,3 +87,29 @@ TEST(TimecodeMetadata, FootageDescriptionCachesSourceStartTime) EXPECT_EQ(loaded.source_start_time_source(), QStringLiteral("bwf_time_reference")); } + +TEST(TimecodeMetadata, FootagePersistsSourceStartTime) +{ + QString xml; + QXmlStreamWriter writer(&xml); + writer.writeStartDocument(); + writer.writeStartElement(QStringLiteral("custom")); + writer.writeTextElement(QStringLiteral("timestamp"), QStringLiteral("0")); + writer.writeStartElement(QStringLiteral("sourcestarttime")); + writer.writeAttribute(QStringLiteral("source"), + QStringLiteral("timecode")); + writer.writeCharacters(QStringLiteral("3600/1")); + writer.writeEndElement(); + writer.writeEndElement(); + writer.writeEndDocument(); + + QXmlStreamReader reader(xml); + ASSERT_TRUE(reader.readNextStartElement()); + ASSERT_EQ(reader.name(), QStringLiteral("custom")); + + olive::Footage footage; + ASSERT_TRUE(footage.LoadCustom(&reader, nullptr)); + ASSERT_TRUE(footage.HasSourceStartTime()); + EXPECT_EQ(footage.source_start_time(), olive::core::rational(3600)); + EXPECT_EQ(footage.source_start_time_source(), QStringLiteral("timecode")); +}