From f6760386ced7f7d2bd916120de06d8c2a42c3afa Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Fri, 21 Aug 2020 14:43:13 +1000 Subject: [PATCH] renderer: improved handling of pixel aspect ratio and footage cache --- app/node/node.cpp | 3 ++ app/render/backend/opengl/openglproxy.cpp | 42 ++++++++----------- app/render/backend/opengl/opengltexture.cpp | 30 ++++--------- app/render/backend/opengl/opengltexture.h | 39 ++++++++++++++--- .../backend/opengl/opengltexturecache.cpp | 2 + app/render/backend/renderbackend.cpp | 20 ++++----- app/render/backend/renderbackend.h | 11 ++++- app/render/backend/renderworker.cpp | 24 +++++++---- app/render/backend/renderworker.h | 6 +++ app/render/framehashcache.cpp | 13 +++++- app/render/framehashcache.h | 2 + app/task/export/export.cpp | 3 ++ 12 files changed, 122 insertions(+), 73 deletions(-) diff --git a/app/node/node.cpp b/app/node/node.cpp index f54822bfb..15e28512a 100644 --- a/app/node/node.cpp +++ b/app/node/node.cpp @@ -375,6 +375,9 @@ void Node::Hash(QCryptographicHash &hash, const rational& time) const // Alpha associated setting hash.addData(QString::number(image_stream->premultiplied_alpha()).toUtf8()); + + // Pixel aspect ratio + hash.addData(reinterpret_cast(&image_stream->pixel_aspect_ratio()), sizeof(rational)); } // Footage timestamp diff --git a/app/render/backend/opengl/openglproxy.cpp b/app/render/backend/opengl/openglproxy.cpp index 224f52c41..7cc61df47 100644 --- a/app/render/backend/opengl/openglproxy.cpp +++ b/app/render/backend/opengl/openglproxy.cpp @@ -148,30 +148,6 @@ QVariant OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream, const Video VideoParams frame_params = frame->video_params(); - // Check frame aspect ratio - rational true_pixel_aspect_ratio = frame_params.pixel_aspect_ratio() / params.pixel_aspect_ratio(); - - if (true_pixel_aspect_ratio != 1) { - int new_width = frame_params.width(); - int new_height = frame_params.height(); - - // Scale the frame in a way that does not reduce the resolution - if (frame_params.pixel_aspect_ratio() > 1) { - // Make wider - new_width = qRound(static_cast(new_width) * frame_params.pixel_aspect_ratio().toDouble()); - } else { - // Make taller - new_height = qRound(static_cast(new_height) / frame_params.pixel_aspect_ratio().toDouble()); - } - - frame_params = VideoParams(new_width, - new_height, - frame_params.format(), - frame_params.pixel_aspect_ratio(), - frame_params.interlacing(), - frame_params.divider()); - } - PixelFormat::Format texture_fmt; if (PixelFormat::FormatHasAlphaChannel(frame_params.format())) { texture_fmt = PixelFormat::GetFormatWithAlphaChannel(params.format()); @@ -385,8 +361,18 @@ QVariant OpenGLProxy::RunNodeAccelerated(const Node *node, // Set texture resolution if shader wants it int res_param_location = shader->uniformLocation(QStringLiteral("%1_resolution").arg(it.key())); if (res_param_location > -1) { + int adjusted_width = texture->texture()->width() * texture->texture()->divider(); + + // Adjust virtual width by pixel aspect if necessary + if (texture->texture()->params().pixel_aspect_ratio() != 1 + || params.pixel_aspect_ratio() != 1) { + double relative_pixel_aspect = texture->texture()->params().pixel_aspect_ratio().toDouble() / params.pixel_aspect_ratio().toDouble(); + + adjusted_width = qRound(static_cast(adjusted_width) * relative_pixel_aspect); + } + shader->setUniformValue(res_param_location, - static_cast(texture->texture()->width() * texture->texture()->divider()), + adjusted_width, static_cast(texture->texture()->height() * texture->texture()->divider())); } } @@ -507,6 +493,12 @@ void OpenGLProxy::TextureToBuffer(const QVariant& tex_in, OpenGLTextureCache::ReferencePtr download_tex; + if (!frame->is_allocated()) { + // If the frame isn't allocated, we'll assume that we're allocating it to the texture dimensions + frame->set_video_params(texture->texture()->params()); + frame->allocate(); + } + functions_->glViewport(0, 0, frame->width(), frame->height()); if (frame->width() != texture->texture()->width() diff --git a/app/render/backend/opengl/opengltexture.cpp b/app/render/backend/opengl/opengltexture.cpp index 2f54beec8..9954e863c 100644 --- a/app/render/backend/opengl/opengltexture.cpp +++ b/app/render/backend/opengl/opengltexture.cpp @@ -100,29 +100,15 @@ void OpenGLTexture::Release() created_ctx_->functions()->glBindTexture(GL_TEXTURE_2D, 0); } -const int &OpenGLTexture::width() const +void OpenGLTexture::SetPixelAspectRatio(const rational &r) { - return params_.effective_width(); -} - -const int &OpenGLTexture::height() const -{ - return params_.effective_height(); -} - -const PixelFormat::Format &OpenGLTexture::format() const -{ - return params_.format(); -} - -const GLuint &OpenGLTexture::texture() const -{ - return texture_; -} - -const int &OpenGLTexture::divider() const -{ - return params_.divider(); + params_ = VideoParams(params_.width(), + params_.height(), + params_.time_base(), + params_.format(), + r, + params_.interlacing(), + params_.divider()); } void OpenGLTexture::Upload(FramePtr frame) diff --git a/app/render/backend/opengl/opengltexture.h b/app/render/backend/opengl/opengltexture.h index e71fa2a51..301383da9 100644 --- a/app/render/backend/opengl/opengltexture.h +++ b/app/render/backend/opengl/opengltexture.h @@ -52,15 +52,44 @@ public: void Release(); - const int& width() const; + const VideoParams& params() const + { + return params_; + } - const int& height() const; + const int& width() const + { + return params_.effective_width(); + } - const PixelFormat::Format &format() const; + const int& height() const + { + return params_.effective_height(); + } - const GLuint& texture() const; + const PixelFormat::Format &format() const + { + return params_.format(); + } - const int& divider() const; + const GLuint& texture() const + { + return texture_; + } + + const int& divider() const + { + return params_.divider(); + } + + /** + * @brief Changes the pixel aspect ratio metadata of this textuer + * + * This metadata is important for our render pipeline, but we don't need to do any re-allocation + * to set it like we do with other VideoParam changes, so we provide a function to change only + * the PAR here. + */ + void SetPixelAspectRatio(const rational& r); void Upload(FramePtr frame); void Upload(Frame* frame); diff --git a/app/render/backend/opengl/opengltexturecache.cpp b/app/render/backend/opengl/opengltexturecache.cpp index f892a7e7f..e00fc8f42 100644 --- a/app/render/backend/opengl/opengltexturecache.cpp +++ b/app/render/backend/opengl/opengltexturecache.cpp @@ -64,6 +64,8 @@ OpenGLTextureCache::ReferencePtr OpenGLTextureCache::Get(QOpenGLContext* ctx, co texture->Create(ctx, params); } + texture->SetPixelAspectRatio(params.pixel_aspect_ratio()); + ReferencePtr ref = std::make_shared(this, texture); existing_references_.append(ref.get()); diff --git a/app/render/backend/renderbackend.cpp b/app/render/backend/renderbackend.cpp index 01d97451d..b7660f93f 100644 --- a/app/render/backend/renderbackend.cpp +++ b/app/render/backend/renderbackend.cpp @@ -20,6 +20,7 @@ #include "renderbackend.h" +#include #include #include @@ -39,6 +40,7 @@ QThreadPool RenderBackend::thread_pool_; RenderBackend::RenderBackend(QObject *parent) : QObject(parent), viewer_node_(nullptr), + video_force_download_resolution_(false), autocache_enabled_(false), autocache_paused_(false), generate_audio_previews_(false), @@ -252,11 +254,6 @@ void RenderBackend::SetAudioParams(const AudioParams ¶ms) audio_params_ = params; } -void RenderBackend::SetVideoDownloadMatrix(const QMatrix4x4 &mat) -{ - video_download_matrix_ = mat; -} - std::list RenderBackend::SplitRangeIntoChunks(const TimeRange &r) { // FIXME: Magic number @@ -424,6 +421,7 @@ void RenderBackend::RunNextJob() worker->SetVideoParams(video_params_); worker->SetAudioParams(audio_params_); + worker->SetForceDownloadResolution(video_force_download_resolution_); worker->SetVideoDownloadMatrix(video_download_matrix_); worker->SetRenderMode(render_mode_); worker->SetPreviewGenerationEnabled(generate_audio_previews_); @@ -520,11 +518,13 @@ void RenderBackend::AutoCacheVideoInvalidated(const TimeRange &range) ClearVideoQueue(); // Hash these frames since that should be relatively quick. - RenderTicketWatcher* watcher = new RenderTicketWatcher(); - QVector frames = viewer_node_->video_frame_cache()->GetFrameListFromTimeRange({range}); - autocache_hash_tasks_.insert(watcher, frames); - connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheHashesGenerated); - watcher->SetTicket(Hash(frames)); + if (!(qApp->mouseButtons() & Qt::LeftButton)) { + RenderTicketWatcher* watcher = new RenderTicketWatcher(); + QVector frames = viewer_node_->video_frame_cache()->GetFrameListFromTimeRange({range}); + autocache_hash_tasks_.insert(watcher, frames); + connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheHashesGenerated); + watcher->SetTicket(Hash(frames)); + } } void RenderBackend::AutoCacheAudioInvalidated(const TimeRange &range) diff --git a/app/render/backend/renderbackend.h b/app/render/backend/renderbackend.h index 3ec853bbc..6925a60ea 100644 --- a/app/render/backend/renderbackend.h +++ b/app/render/backend/renderbackend.h @@ -131,7 +131,15 @@ public: void SetAudioParams(const AudioParams& params); - void SetVideoDownloadMatrix(const QMatrix4x4& mat); + void SetForceDownloadResolution(bool e) + { + video_force_download_resolution_ = e; + } + + void SetVideoDownloadMatrix(const QMatrix4x4& mat) + { + video_download_matrix_ = mat; + } static std::list SplitRangeIntoChunks(const TimeRange& r); @@ -162,6 +170,7 @@ private: // VIDEO MEMBERS VideoParams video_params_; + bool video_force_download_resolution_; QMatrix4x4 video_download_matrix_; // AUDIO MEMBERS diff --git a/app/render/backend/renderworker.cpp b/app/render/backend/renderworker.cpp index 14cb5304b..d1af07535 100644 --- a/app/render/backend/renderworker.cpp +++ b/app/render/backend/renderworker.cpp @@ -38,6 +38,7 @@ const int RenderWorker::kMaxDecoderLife = 6000; RenderWorker::RenderWorker(RenderBackend* parent) : parent_(parent), + video_force_download_resolution_(false), available_(true), generate_audio_previews_(false), render_mode_(RenderMode::kOnline) @@ -123,15 +124,19 @@ void RenderWorker::RenderFrame(RenderTicketPtr ticket, ViewerOutput* viewer, con } FramePtr frame = Frame::Create(); - frame->set_video_params(VideoParams(video_params_.width(), - video_params_.height(), - video_params_.time_base(), - output_format, - video_params_.pixel_aspect_ratio(), - video_params_.interlacing(), - video_params_.divider())); frame->set_timestamp(time); - frame->allocate(); + + if (video_force_download_resolution_ || texture.isNull()) { + // If we're setting the resolution ourselves or we're zeroing it out, allocate the frame now + frame->set_video_params(VideoParams(video_params_.width(), + video_params_.height(), + video_params_.time_base(), + output_format, + video_params_.pixel_aspect_ratio(), + video_params_.interlacing(), + video_params_.divider())); + frame->allocate(); + } if (texture.isNull()) { // Blank frame out @@ -312,7 +317,8 @@ QVariant RenderWorker::ProcessFrameGeneration(const Node* node, const GenerateJo QVariant RenderWorker::GetCachedFrame(const Node* node, const rational& time) { - if (node->id() == QStringLiteral("org.olivevideoeditor.Olive.videoinput")) { + if (render_mode_ == RenderMode::kOffline + && node->id() == QStringLiteral("org.olivevideoeditor.Olive.videoinput")) { QByteArray hash = HashNode(node, video_params(), time); FramePtr f = viewer_->video_frame_cache()->LoadCacheFrame(hash); diff --git a/app/render/backend/renderworker.h b/app/render/backend/renderworker.h index 6d4729cb9..4cfb4d729 100644 --- a/app/render/backend/renderworker.h +++ b/app/render/backend/renderworker.h @@ -65,6 +65,11 @@ public: audio_params_ = params; } + void SetForceDownloadResolution(bool e) + { + video_force_download_resolution_ = e; + } + void SetVideoDownloadMatrix(const QMatrix4x4& mat) { video_download_matrix_ = mat; @@ -169,6 +174,7 @@ private: QHash still_image_cache_; + bool video_force_download_resolution_; QMatrix4x4 video_download_matrix_; QMutex decoder_lock_; diff --git a/app/render/framehashcache.cpp b/app/render/framehashcache.cpp index 7ed84d274..bedfe7adb 100644 --- a/app/render/framehashcache.cpp +++ b/app/render/framehashcache.cpp @@ -253,7 +253,8 @@ FramePtr FrameHashCache::LoadCacheFrame(const QString &fn) const frame = Frame::Create(); frame->set_video_params(VideoParams(width, height, - image_format)); + image_format, + rational::fromDouble(file.header().pixelAspectRatio()))); frame->allocate(); @@ -333,6 +334,15 @@ void FrameHashCache::ShiftEvent(const rational &from, const rational &to) } } +void FrameHashCache::InvalidateEvent(const TimeRange &range) +{ + QVector invalid_frames = GetFrameListFromTimeRange({range}); + + foreach (const rational& r, invalid_frames) { + time_hash_map_.remove(r); + } +} + void FrameHashCache::HashDeleted(const QString& s, const QByteArray &hash) { QString cache_dir = GetCacheDirectory(); @@ -401,6 +411,7 @@ bool FrameHashCache::SaveCacheFrame(const QString &filename, char *data, const V header.compression() = Imf::DWAA_COMPRESSION; header.insert("dwaCompressionLevel", Imf::FloatAttribute(200.0f)); + header.pixelAspectRatio() = vparam.pixel_aspect_ratio().toDouble(); Imf::OutputFile out(filename.toUtf8(), header, 0); diff --git a/app/render/framehashcache.h b/app/render/framehashcache.h index d3314e587..bdf5d81bb 100644 --- a/app/render/framehashcache.h +++ b/app/render/framehashcache.h @@ -81,6 +81,8 @@ protected: virtual void ShiftEvent(const rational& from, const rational& to) override; + virtual void InvalidateEvent(const TimeRange& range) override; + private: QMap time_hash_map_; diff --git a/app/task/export/export.cpp b/app/task/export/export.cpp index 69cdfe502..6b1739622 100644 --- a/app/task/export/export.cpp +++ b/app/task/export/export.cpp @@ -66,6 +66,9 @@ bool ExportTask::Run() if (params_.video_enabled()) { + // Ensure renderer always provides the same resolution + backend()->SetForceDownloadResolution(true); + // If a transformation matrix is applied to this video, create it here if (params_.video_scaling_method() != ExportParams::kStretch) { QMatrix4x4 mat = ExportParams::GenerateMatrix(params_.video_scaling_method(),