renderer: improved handling of pixel aspect ratio and footage cache

app/node/node.cpp

@@ -375,6 +375,9 @@ void Node::Hash(QCryptographicHash &hash, const rational& time) const
 
           // Alpha associated setting
           hash.addData(QString::number(image_stream->premultiplied_alpha()).toUtf8());
+
+          // Pixel aspect ratio
+          hash.addData(reinterpret_cast<const char*>(&image_stream->pixel_aspect_ratio()), sizeof(rational));
         }
 
         // Footage timestamp

app/render/backend/opengl/openglproxy.cpp

@@ -148,30 +148,6 @@ QVariant OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream, const Video
 
     VideoParams frame_params = frame->video_params();
 
-    // Check frame aspect ratio
-    rational true_pixel_aspect_ratio = frame_params.pixel_aspect_ratio() / params.pixel_aspect_ratio();
-
-    if (true_pixel_aspect_ratio != 1) {
-      int new_width = frame_params.width();
-      int new_height = frame_params.height();
-
-      // Scale the frame in a way that does not reduce the resolution
-      if (frame_params.pixel_aspect_ratio() > 1) {
-        // Make wider
-        new_width = qRound(static_cast<double>(new_width) * frame_params.pixel_aspect_ratio().toDouble());
-      } else {
-        // Make taller
-        new_height = qRound(static_cast<double>(new_height) / frame_params.pixel_aspect_ratio().toDouble());
-      }
-
-      frame_params = VideoParams(new_width,
-                                 new_height,
-                                 frame_params.format(),
-                                 frame_params.pixel_aspect_ratio(),
-                                 frame_params.interlacing(),
-                                 frame_params.divider());
-    }
-
     PixelFormat::Format texture_fmt;
     if (PixelFormat::FormatHasAlphaChannel(frame_params.format())) {
       texture_fmt = PixelFormat::GetFormatWithAlphaChannel(params.format());
@@ -385,8 +361,18 @@ QVariant OpenGLProxy::RunNodeAccelerated(const Node *node,
         // Set texture resolution if shader wants it
         int res_param_location = shader->uniformLocation(QStringLiteral("%1_resolution").arg(it.key()));
         if (res_param_location > -1) {
+          int adjusted_width = texture->texture()->width() * texture->texture()->divider();
+
+          // Adjust virtual width by pixel aspect if necessary
+          if (texture->texture()->params().pixel_aspect_ratio() != 1
+              || params.pixel_aspect_ratio() != 1) {
+            double relative_pixel_aspect = texture->texture()->params().pixel_aspect_ratio().toDouble() / params.pixel_aspect_ratio().toDouble();
+
+            adjusted_width = qRound(static_cast<double>(adjusted_width) * relative_pixel_aspect);
+          }
+
           shader->setUniformValue(res_param_location,
-                                  static_cast<GLfloat>(texture->texture()->width() * texture->texture()->divider()),
+                                  adjusted_width,
                                   static_cast<GLfloat>(texture->texture()->height() * texture->texture()->divider()));
         }
       }
@@ -507,6 +493,12 @@ void OpenGLProxy::TextureToBuffer(const QVariant& tex_in,
 
   OpenGLTextureCache::ReferencePtr download_tex;
 
+  if (!frame->is_allocated()) {
+    // If the frame isn't allocated, we'll assume that we're allocating it to the texture dimensions
+    frame->set_video_params(texture->texture()->params());
+    frame->allocate();
+  }
+
   functions_->glViewport(0, 0, frame->width(), frame->height());
 
   if (frame->width() != texture->texture()->width()

app/render/backend/opengl/opengltexture.cpp

@@ -100,29 +100,15 @@ void OpenGLTexture::Release()
   created_ctx_->functions()->glBindTexture(GL_TEXTURE_2D, 0);
 }
 
-const int &OpenGLTexture::width() const
+void OpenGLTexture::SetPixelAspectRatio(const rational &r)
 {
-  return params_.effective_width();
-}
-
-const int &OpenGLTexture::height() const
-{
-  return params_.effective_height();
-}
-
-const PixelFormat::Format &OpenGLTexture::format() const
-{
-  return params_.format();
-}
-
-const GLuint &OpenGLTexture::texture() const
-{
-  return texture_;
-}
-
-const int &OpenGLTexture::divider() const
-{
-  return params_.divider();
+  params_ = VideoParams(params_.width(),
+                        params_.height(),
+                        params_.time_base(),
+                        params_.format(),
+                        r,
+                        params_.interlacing(),
+                        params_.divider());
 }
 
 void OpenGLTexture::Upload(FramePtr frame)

app/render/backend/opengl/opengltexture.h

@@ -52,15 +52,44 @@ public:
 
   void Release();
 
-  const int& width() const;
+  const VideoParams& params() const
+  {
+    return params_;
+  }
 
-  const int& height() const;
+  const int& width() const
+  {
+    return params_.effective_width();
+  }
 
-  const PixelFormat::Format &format() const;
+  const int& height() const
+  {
+    return params_.effective_height();
+  }
 
-  const GLuint& texture() const;
+  const PixelFormat::Format &format() const
+  {
+    return params_.format();
+  }
 
-  const int& divider() const;
+  const GLuint& texture() const
+  {
+    return texture_;
+  }
+
+  const int& divider() const
+  {
+    return params_.divider();
+  }
+
+  /**
+   * @brief Changes the pixel aspect ratio metadata of this textuer
+   *
+   * This metadata is important for our render pipeline, but we don't need to do any re-allocation
+   * to set it like we do with other VideoParam changes, so we provide a function to change only
+   * the PAR here.
+   */
+  void SetPixelAspectRatio(const rational& r);
 
   void Upload(FramePtr frame);
   void Upload(Frame* frame);

app/render/backend/opengl/opengltexturecache.cpp

@@ -64,6 +64,8 @@ OpenGLTextureCache::ReferencePtr OpenGLTextureCache::Get(QOpenGLContext* ctx, co
     texture->Create(ctx, params);
   }
 
+  texture->SetPixelAspectRatio(params.pixel_aspect_ratio());
+
   ReferencePtr ref = std::make_shared<Reference>(this, texture);
   existing_references_.append(ref.get());
 

app/render/backend/renderbackend.cpp

@@ -20,6 +20,7 @@
 
 #include "renderbackend.h"
 
+#include <QApplication>
 #include <QDateTime>
 #include <QThread>
 
@@ -39,6 +40,7 @@ QThreadPool RenderBackend::thread_pool_;
 RenderBackend::RenderBackend(QObject *parent) :
   QObject(parent),
   viewer_node_(nullptr),
+  video_force_download_resolution_(false),
   autocache_enabled_(false),
   autocache_paused_(false),
   generate_audio_previews_(false),
@@ -252,11 +254,6 @@ void RenderBackend::SetAudioParams(const AudioParams &params)
   audio_params_ = params;
 }
 
-void RenderBackend::SetVideoDownloadMatrix(const QMatrix4x4 &mat)
-{
-  video_download_matrix_ = mat;
-}
-
 std::list<TimeRange> RenderBackend::SplitRangeIntoChunks(const TimeRange &r)
 {
   // FIXME: Magic number
@@ -424,6 +421,7 @@ void RenderBackend::RunNextJob()
 
       worker->SetVideoParams(video_params_);
       worker->SetAudioParams(audio_params_);
+      worker->SetForceDownloadResolution(video_force_download_resolution_);
       worker->SetVideoDownloadMatrix(video_download_matrix_);
       worker->SetRenderMode(render_mode_);
       worker->SetPreviewGenerationEnabled(generate_audio_previews_);
@@ -520,11 +518,13 @@ void RenderBackend::AutoCacheVideoInvalidated(const TimeRange &range)
   ClearVideoQueue();
 
   // Hash these frames since that should be relatively quick.
-  RenderTicketWatcher* watcher = new RenderTicketWatcher();
-  QVector<rational> frames = viewer_node_->video_frame_cache()->GetFrameListFromTimeRange({range});
-  autocache_hash_tasks_.insert(watcher, frames);
-  connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheHashesGenerated);
-  watcher->SetTicket(Hash(frames));
+  if (!(qApp->mouseButtons() & Qt::LeftButton)) {
+    RenderTicketWatcher* watcher = new RenderTicketWatcher();
+    QVector<rational> frames = viewer_node_->video_frame_cache()->GetFrameListFromTimeRange({range});
+    autocache_hash_tasks_.insert(watcher, frames);
+    connect(watcher, &RenderTicketWatcher::Finished, this, &RenderBackend::AutoCacheHashesGenerated);
+    watcher->SetTicket(Hash(frames));
+  }
 }
 
 void RenderBackend::AutoCacheAudioInvalidated(const TimeRange &range)

app/render/backend/renderbackend.h

@@ -131,7 +131,15 @@ public:
 
   void SetAudioParams(const AudioParams& params);
 
-  void SetVideoDownloadMatrix(const QMatrix4x4& mat);
+  void SetForceDownloadResolution(bool e)
+  {
+    video_force_download_resolution_ = e;
+  }
+
+  void SetVideoDownloadMatrix(const QMatrix4x4& mat)
+  {
+    video_download_matrix_ = mat;
+  }
 
   static std::list<TimeRange> SplitRangeIntoChunks(const TimeRange& r);
 
@@ -162,6 +170,7 @@ private:
 
   // VIDEO MEMBERS
   VideoParams video_params_;
+  bool video_force_download_resolution_;
   QMatrix4x4 video_download_matrix_;
 
   // AUDIO MEMBERS

app/render/backend/renderworker.cpp

@@ -38,6 +38,7 @@ const int RenderWorker::kMaxDecoderLife = 6000;
 
 RenderWorker::RenderWorker(RenderBackend* parent) :
   parent_(parent),
+  video_force_download_resolution_(false),
   available_(true),
   generate_audio_previews_(false),
   render_mode_(RenderMode::kOnline)
@@ -123,15 +124,19 @@ void RenderWorker::RenderFrame(RenderTicketPtr ticket, ViewerOutput* viewer, con
   }
 
   FramePtr frame = Frame::Create();
-  frame->set_video_params(VideoParams(video_params_.width(),
-                                      video_params_.height(),
-                                      video_params_.time_base(),
-                                      output_format,
-                                      video_params_.pixel_aspect_ratio(),
-                                      video_params_.interlacing(),
-                                      video_params_.divider()));
   frame->set_timestamp(time);
-  frame->allocate();
+
+  if (video_force_download_resolution_ || texture.isNull()) {
+    // If we're setting the resolution ourselves or we're zeroing it out, allocate the frame now
+    frame->set_video_params(VideoParams(video_params_.width(),
+                                        video_params_.height(),
+                                        video_params_.time_base(),
+                                        output_format,
+                                        video_params_.pixel_aspect_ratio(),
+                                        video_params_.interlacing(),
+                                        video_params_.divider()));
+    frame->allocate();
+  }
 
   if (texture.isNull()) {
     // Blank frame out
@@ -312,7 +317,8 @@ QVariant RenderWorker::ProcessFrameGeneration(const Node* node, const GenerateJo
 
 QVariant RenderWorker::GetCachedFrame(const Node* node, const rational& time)
 {
-  if (node->id() == QStringLiteral("org.olivevideoeditor.Olive.videoinput")) {
+  if (render_mode_ == RenderMode::kOffline
+      && node->id() == QStringLiteral("org.olivevideoeditor.Olive.videoinput")) {
     QByteArray hash = HashNode(node, video_params(), time);
 
     FramePtr f = viewer_->video_frame_cache()->LoadCacheFrame(hash);

app/render/backend/renderworker.h

@@ -65,6 +65,11 @@ public:
     audio_params_ = params;
   }
 
+  void SetForceDownloadResolution(bool e)
+  {
+    video_force_download_resolution_ = e;
+  }
+
   void SetVideoDownloadMatrix(const QMatrix4x4& mat)
   {
     video_download_matrix_ = mat;
@@ -169,6 +174,7 @@ private:
 
   QHash<Stream*, CachedStill> still_image_cache_;
 
+  bool video_force_download_resolution_;
   QMatrix4x4 video_download_matrix_;
 
   QMutex decoder_lock_;

app/render/framehashcache.cpp

@@ -253,7 +253,8 @@ FramePtr FrameHashCache::LoadCacheFrame(const QString &fn) const
     frame = Frame::Create();
     frame->set_video_params(VideoParams(width,
                                         height,
-                                        image_format));
+                                        image_format,
+                                        rational::fromDouble(file.header().pixelAspectRatio())));
 
     frame->allocate();
 
@@ -333,6 +334,15 @@ void FrameHashCache::ShiftEvent(const rational &from, const rational &to)
   }
 }
 
+void FrameHashCache::InvalidateEvent(const TimeRange &range)
+{
+  QVector<rational> invalid_frames = GetFrameListFromTimeRange({range});
+
+  foreach (const rational& r, invalid_frames) {
+    time_hash_map_.remove(r);
+  }
+}
+
 void FrameHashCache::HashDeleted(const QString& s, const QByteArray &hash)
 {
   QString cache_dir = GetCacheDirectory();
@@ -401,6 +411,7 @@ bool FrameHashCache::SaveCacheFrame(const QString &filename, char *data, const V
 
   header.compression() = Imf::DWAA_COMPRESSION;
   header.insert("dwaCompressionLevel", Imf::FloatAttribute(200.0f));
+  header.pixelAspectRatio() = vparam.pixel_aspect_ratio().toDouble();
 
   Imf::OutputFile out(filename.toUtf8(), header, 0);
 

app/render/framehashcache.h

@@ -81,6 +81,8 @@ protected:
 
   virtual void ShiftEvent(const rational& from, const rational& to) override;
 
+  virtual void InvalidateEvent(const TimeRange& range) override;
+
 private:
   QMap<rational, QByteArray> time_hash_map_;
 

app/task/export/export.cpp

@@ -66,6 +66,9 @@ bool ExportTask::Run()
 
   if (params_.video_enabled()) {
 
+    // Ensure renderer always provides the same resolution
+    backend()->SetForceDownloadResolution(true);
+
     // If a transformation matrix is applied to this video, create it here
     if (params_.video_scaling_method() != ExportParams::kStretch) {
       QMatrix4x4 mat = ExportParams::GenerateMatrix(params_.video_scaling_method(),