Merge branch 'olive-editor:master' into av1

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -52,12 +52,15 @@ extern "C" {
 
 namespace olive {
 
+QVariant Yuv2RgbShader;
+
 FFmpegDecoder::FFmpegDecoder() :
   filter_graph_(nullptr),
   buffersrc_ctx_(nullptr),
   buffersink_ctx_(nullptr),
   input_fmt_(AV_PIX_FMT_NONE),
-  native_pix_fmt_(VideoParams::kFormatInvalid),
+  native_internal_pix_fmt_(VideoParams::kFormatInvalid),
+  native_output_pix_fmt_(VideoParams::kFormatInvalid),
   working_frame_(nullptr),
   working_packet_(nullptr),
   is_working_(false),
@@ -142,28 +145,116 @@ bool FFmpegDecoder::OpenInternal()
 TexturePtr FFmpegDecoder::RetrieveVideoInternal(Renderer *renderer, const rational &timecode, const RetrieveVideoParams &params, const QAtomicInt *cancelled)
 {
   if (AVFramePtr f = RetrieveFrame(timecode, cancelled)) {
-    if (InitScaler(f.get(), params)) {
-      int r;
-      r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF);
-      if (r < 0) {
-        return nullptr;
-      }
-      r = av_buffersink_get_frame(buffersink_ctx_, working_frame_);
-      if (r < 0) {
-        return nullptr;
-      }
+    if (cancelled && *cancelled) {
+      return nullptr;
+    }
 
+    if (InitScaler(f.get(), params)) {
       VideoParams vp(instance_.avstream()->codecpar->width,
                      instance_.avstream()->codecpar->height,
-                     native_pix_fmt_,
+                     native_output_pix_fmt_,
                      native_channel_count_,
                      av_guess_sample_aspect_ratio(instance_.fmt_ctx(), instance_.avstream(), nullptr),
                      VideoParams::kInterlaceNone,
                      params.divider);
 
-      TexturePtr tex = renderer->CreateTexture(vp, working_frame_->data[0], working_frame_->linesize[0] / vp.GetBytesPerPixel());
+      TexturePtr tex = nullptr;
+      const bool hwscale = true;
 
-      av_frame_unref(working_frame_);
+      // Attempt to use GLSL shader for faster YUV to RGB conversion
+      if (hwscale) {
+        AVPixelFormat src_fmt = AVPixelFormat(f.get()->format);
+        if (src_fmt == AV_PIX_FMT_YUV420P
+            || src_fmt == AV_PIX_FMT_YUV422P
+            || src_fmt == AV_PIX_FMT_YUV444P
+            || src_fmt == AV_PIX_FMT_YUV420P10LE
+            || src_fmt == AV_PIX_FMT_YUV422P10LE
+            || src_fmt == AV_PIX_FMT_YUV444P10LE
+            || src_fmt == AV_PIX_FMT_YUV420P12LE
+            || src_fmt == AV_PIX_FMT_YUV422P12LE
+            || src_fmt == AV_PIX_FMT_YUV444P12LE) {
+          if (Yuv2RgbShader.isNull()) {
+            // Compile shader
+            Yuv2RgbShader = renderer->CreateNativeShader(ShaderCode(FileFunctions::ReadFileAsString(QStringLiteral(":/shaders/yuv2rgb.frag"))));
+          }
+
+          if (!Yuv2RgbShader.isNull()) {
+            int px_size;
+            int bits_per_pixel;
+            switch (src_fmt) {
+            case AV_PIX_FMT_YUV420P:
+            case AV_PIX_FMT_YUV422P:
+            case AV_PIX_FMT_YUV444P:
+            default:
+              px_size = 1;
+              bits_per_pixel = 8;
+              break;
+            case AV_PIX_FMT_YUV420P10LE:
+            case AV_PIX_FMT_YUV422P10LE:
+            case AV_PIX_FMT_YUV444P10LE:
+              px_size = 2;
+              bits_per_pixel = 10;
+              break;
+            case AV_PIX_FMT_YUV420P12LE:
+            case AV_PIX_FMT_YUV422P12LE:
+            case AV_PIX_FMT_YUV444P12LE:
+              px_size = 2;
+              bits_per_pixel = 12;
+              break;
+            }
+
+            VideoParams plane_params = vp;
+            plane_params.set_channel_count(1);
+            plane_params.set_divider(1);
+            plane_params.set_format(native_internal_pix_fmt_);
+            TexturePtr y_plane = renderer->CreateTexture(plane_params, f->data[0], f->linesize[0] / px_size);
+
+            if (src_fmt == AV_PIX_FMT_YUV420P
+                || src_fmt == AV_PIX_FMT_YUV422P
+                || src_fmt == AV_PIX_FMT_YUV420P10LE
+                || src_fmt == AV_PIX_FMT_YUV422P10LE
+                || src_fmt == AV_PIX_FMT_YUV420P12LE
+                || src_fmt == AV_PIX_FMT_YUV422P12LE) {
+              plane_params.set_width(plane_params.width()/2);
+            }
+
+            if (src_fmt == AV_PIX_FMT_YUV420P
+                || src_fmt == AV_PIX_FMT_YUV420P10LE
+                || src_fmt == AV_PIX_FMT_YUV420P12LE) {
+              plane_params.set_height(plane_params.height()/2);
+            }
+
+            TexturePtr u_plane = renderer->CreateTexture(plane_params, f->data[1], f->linesize[1] / px_size);
+            TexturePtr v_plane = renderer->CreateTexture(plane_params, f->data[2], f->linesize[2] / px_size);
+
+            ShaderJob job;
+            job.Insert(QStringLiteral("y_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(y_plane)));
+            job.Insert(QStringLiteral("u_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(u_plane)));
+            job.Insert(QStringLiteral("v_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(v_plane)));
+            job.Insert(QStringLiteral("bits_per_pixel"), NodeValue(NodeValue::kInt, bits_per_pixel));
+
+            tex = renderer->CreateTexture(vp);
+            renderer->BlitToTexture(Yuv2RgbShader, job, tex.get(), false);
+          }
+        }
+      }
+
+      if (!tex) {
+        // Fallback to software pixel format conversion
+        int r;
+        r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF);
+        if (r < 0) {
+          return nullptr;
+        }
+        r = av_buffersink_get_frame(buffersink_ctx_, working_frame_);
+        if (r < 0) {
+          return nullptr;
+        }
+
+        tex = renderer->CreateTexture(vp, working_frame_->data[0], working_frame_->linesize[0] / vp.GetBytesPerPixel());
+
+        av_frame_unref(working_frame_);
+      }
 
       return tex;
     }
@@ -190,7 +281,8 @@ void FFmpegDecoder::CloseInternal()
   instance_.Close();
 
   input_fmt_ = AV_PIX_FMT_NONE;
-  native_pix_fmt_ = VideoParams::kFormatInvalid;
+  native_internal_pix_fmt_ = VideoParams::kFormatInvalid;
+  native_output_pix_fmt_ = VideoParams::kFormatInvalid;
 }
 
 QString FFmpegDecoder::id() const
@@ -702,6 +794,10 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational& time, const QAtomicInt *
     // Pull from the decoder
     ret = instance_.GetFrame(working_packet_, filtered.get());
 
+    if (cancelled && *cancelled) {
+      break;
+    }
+
     // Handle any errors that aren't EOF (EOF is handled later on)
     if (ret < 0 && ret != AVERROR_EOF) {
       qCritical() << "Failed to retrieve frame:" << ret;
@@ -804,10 +900,14 @@ bool FFmpegDecoder::InitScaler(AVFrame *input, const RetrieveVideoParams& params
 
   // Determine which Olive native pixel format we retrieved
   // Note that FFmpeg doesn't support float formats
-  native_pix_fmt_ = GetNativePixelFormat(ideal_pix_fmt);
+  native_output_pix_fmt_ = GetNativePixelFormat(ideal_pix_fmt);
   native_channel_count_ = GetNativeChannelCount(ideal_pix_fmt);
 
-  if (native_pix_fmt_ == VideoParams::kFormatInvalid
+  AVPixelFormat ideal_internal_pix_fmt = FFmpegUtils::GetCompatiblePixelFormat(static_cast<AVPixelFormat>(input_fmt_));
+  native_internal_pix_fmt_ = GetNativePixelFormat(ideal_internal_pix_fmt);
+
+  if (native_output_pix_fmt_ == VideoParams::kFormatInvalid
+      || native_internal_pix_fmt_ == VideoParams::kFormatInvalid
       || native_channel_count_ == 0) {
     qCritical() << "Failed to find valid native pixel format for" << ideal_pix_fmt;
     return false;

app/codec/ffmpeg/ffmpegdecoder.h

@@ -160,7 +160,8 @@ private:
   AVFilterContext* buffersrc_ctx_;
   AVFilterContext* buffersink_ctx_;
   AVPixelFormat input_fmt_;
-  VideoParams::Format native_pix_fmt_;
+  VideoParams::Format native_internal_pix_fmt_;
+  VideoParams::Format native_output_pix_fmt_;
   int native_channel_count_;
 
   AVFrame *working_frame_;

app/shaders/yuv2rgb.frag

@@ -0,0 +1,35 @@
+uniform sampler2D y_channel;
+uniform sampler2D u_channel;
+uniform sampler2D v_channel;
+
+uniform int bits_per_pixel;
+
+in vec2 ove_texcoord;
+out vec4 frag_color;
+
+void main() {
+  vec4 rgba;
+
+  vec3 yuv;
+
+  yuv.r = texture(y_channel, ove_texcoord).r;
+  yuv.g = texture(u_channel, ove_texcoord).r;
+  yuv.b = texture(v_channel, ove_texcoord).r;
+
+  if (bits_per_pixel == 10) {
+    yuv *= 64.0;
+  } else if (bits_per_pixel == 12) {
+    yuv *= 16.0;
+  }
+
+  yuv.r = 1.1643 * (yuv.r - 0.0625);
+  yuv.g = yuv.g - 0.5;
+  yuv.b = yuv.b - 0.5;
+
+  rgba.r = yuv.r + 1.5958 * yuv.b;
+  rgba.g = yuv.r - 0.39173 * yuv.g - 0.81290 * yuv.b;
+  rgba.b = yuv.r + 2.017 * yuv.g;
+  rgba.a = 1.0;
+
+  frag_color = rgba;
+}

app/threading/threadpool.cpp

@@ -29,52 +29,67 @@ ThreadPool::ThreadPool(unsigned threads, QObject *parent) :
     threads = std::thread::hardware_concurrency();
   }
 
+  available_count_ = threads;
   for (unsigned i = 0; i < threads; i += 1) {
-    worker_threads_.emplace_back(std::bind(&ThreadPool::thread_exec, this));
+    worker_threads_.emplace_back(std::bind(&ThreadPool::thread_exec, this, &tasks_, &task_mutex_, &cond_));
   }
+
+  // Make single reserved thread for high priority tasks (usually audio) so they don't get stuck
+  // behind a lot of slow tasks
+  high_thread_ = std::thread(std::thread(std::bind(&ThreadPool::thread_exec, this, &high_tasks_, &high_mutex_, &high_cond_)));
 }
 
 void ThreadPool::AddTicket(RenderTicketPtr ticket, RenderTicketPriority priority)
 {
-  std::lock_guard<std::mutex> lock(task_mutex_);
-
   if (priority == RenderTicketPriority::kHigh) {
-    tasks_.emplace_front(std::move(ticket));
+    std::lock_guard<std::mutex> lock(high_mutex_);
+    high_tasks_.emplace_back(std::move(ticket));
+    high_cond_.notify_one();
   } else {
+    std::lock_guard<std::mutex> lock(task_mutex_);
     tasks_.emplace_back(std::move(ticket));
+    cond_.notify_one();
   }
-
-  cond_.notify_one();
 }
 
 bool ThreadPool::RemoveTicket(RenderTicketPtr ticket)
 {
-  std::lock_guard<std::mutex> lock(task_mutex_);
-
-  const auto it = std::find(tasks_.begin(), tasks_.end(), ticket);
-  if (it == tasks_.end()) {
-    return false;
+  {
+    std::lock_guard<std::mutex> lock(task_mutex_);
+    const auto it = std::find(tasks_.begin(), tasks_.end(), ticket);
+    if (it != tasks_.end()) {
+      tasks_.erase(it);
+      return true;
+    }
   }
 
-  tasks_.erase(it);
-  return true;
+  {
+    std::lock_guard<std::mutex> lock(high_mutex_);
+    const auto it = std::find(high_tasks_.begin(), high_tasks_.end(), ticket);
+    if (it != high_tasks_.end()) {
+      high_tasks_.erase(it);
+      return true;
+    }
+  }
+
+  return false;
 }
 
-void ThreadPool::thread_exec()
+void ThreadPool::thread_exec(std::deque<TaskType> *queue, std::mutex *mutex, std::condition_variable *cond)
 {
   while (true) {
     TaskType task;
 
     {
-      std::unique_lock<std::mutex> lock(task_mutex_);
-      cond_.wait(lock, [this]{ return this->end_threadp_ || !this->tasks_.empty(); });
+      std::unique_lock<std::mutex> lock(*mutex);
+      cond->wait(lock, [this, queue]{ return this->end_threadp_ || !queue->empty(); });
 
-      if (this->end_threadp_ && this->tasks_.empty()) {
+      if (this->end_threadp_ && queue->empty()) {
         break;
       }
 
-      task = std::move(tasks_.front());
-      tasks_.pop_front();
+      task = std::move(queue->front());
+      queue->pop_front();
     }
 
     RunTicket(task);
@@ -85,10 +100,12 @@ ThreadPool::~ThreadPool()
 {
   end_threadp_ = true;
   cond_.notify_all();
+  high_cond_.notify_all();
 
   for (auto &e : worker_threads_) {
     e.join();
   }
+  high_thread_.join();
 }
 
 }

app/threading/threadpool.h

@@ -49,13 +49,20 @@ public:
   virtual ~ThreadPool() override;
 
 private:
-  void thread_exec();
+  void thread_exec(std::deque<TaskType> *queue, std::mutex *mutex, std::condition_variable *cond);
 
   std::vector<std::thread> worker_threads_;
   std::deque<TaskType> tasks_;
   std::mutex task_mutex_;
   std::condition_variable cond_;
+
+  std::thread high_thread_;
+  std::deque<TaskType> high_tasks_;
+  std::mutex high_mutex_;
+  std::condition_variable high_cond_;
+
   std::atomic_bool end_threadp_{false};
+  std::atomic_int available_count_;
 
 };
 

app/widget/viewer/viewer.cpp

@@ -60,6 +60,8 @@ QVector<ViewerWidget*> ViewerWidget::instances_;
 //       changing values. 1/4 second seems to be a good middleground.
 const rational ViewerWidget::kAudioPlaybackInterval = rational(1, 4);
 
+const rational kVideoPlaybackInterval = rational(1, 2);
+
 ViewerWidget::ViewerWidget(QWidget *parent) :
   super(false, true, parent),
   playback_speed_(0),
@@ -69,6 +71,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) :
   prequeuing_audio_(0),
   record_armed_(false),
   recording_(false),
+  first_requeue_watcher_(nullptr),
   enable_audio_scrubbing_(true)
 {
   // Set up main layout
@@ -635,12 +638,20 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing()
 
 void ViewerWidget::QueueStarved()
 {
-  static const int kMaximumWaitTime = 250;
+  static const int kMaximumWaitTimeMs = 250;
+  static const rational kMaximumWaitTime(kMaximumWaitTimeMs, 1000);
   qint64 now = QDateTime::currentMSecsSinceEpoch();
 
   if (!queue_starved_start_) {
     queue_starved_start_ = now;
-  } else if (now > queue_starved_start_ + kMaximumWaitTime) {
+  } else if (now > queue_starved_start_ + kMaximumWaitTimeMs) {
+    if (first_requeue_watcher_) {
+      if (GetTime() + kMaximumWaitTime < first_requeue_watcher_->property("time").value<rational>()) {
+        // We still have time
+        return;
+      }
+    }
+
     ForceRequeueFromCurrentTime();
     queue_starved_start_ = 0;
   }
@@ -653,11 +664,19 @@ void ViewerWidget::QueueNoLongerStarved()
 
 void ViewerWidget::ForceRequeueFromCurrentTime()
 {
+  // Allow half a second for requeue to complete
+  static const rational kRequeueWaitTime(1);
+
   ClearVideoAutoCacherQueue();
+  queue_watchers_.clear();
   int queue = DeterminePlaybackQueueSize();
-  playback_queue_next_frame_ = GetTimestamp() + playback_speed_;
-  for (int i=queue_watchers_.size(); i<queue; i++) {
-    RequestNextFrameForQueue();
+  playback_queue_next_frame_ = GetTimestamp() + playback_speed_ * Timecode::time_to_timestamp(kRequeueWaitTime, timebase(), Timecode::kFloor);;
+  first_requeue_watcher_ = nullptr;
+  for (int i=0; i<queue; i++) {
+    RenderTicketWatcher *watcher = RequestNextFrameForQueue();
+    if (!first_requeue_watcher_) {
+      first_requeue_watcher_ = watcher;
+    }
   }
 }
 
@@ -899,8 +918,10 @@ void ViewerWidget::SetDisplayImage(QVariant frame)
   }
 }
 
-void ViewerWidget::RequestNextFrameForQueue(RenderTicketPriority priority, bool increment)
+RenderTicketWatcher *ViewerWidget::RequestNextFrameForQueue(RenderTicketPriority priority, bool increment)
 {
+  RenderTicketWatcher *watcher = nullptr;
+
   rational next_time = Timecode::timestamp_to_time(playback_queue_next_frame_,
                                                    timebase());
 
@@ -909,12 +930,14 @@ void ViewerWidget::RequestNextFrameForQueue(RenderTicketPriority priority, bool
       playback_queue_next_frame_ += playback_speed_;
     }
 
-    RenderTicketWatcher* watcher = new RenderTicketWatcher();
+    watcher = new RenderTicketWatcher();
     watcher->setProperty("time", QVariant::fromValue(next_time));
     connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::RendererGeneratedFrameForQueue);
     queue_watchers_.append(watcher);
     watcher->SetTicket(GetFrame(next_time, priority));
   }
+
+  return watcher;
 }
 
 RenderTicketPtr ViewerWidget::GetFrame(const rational &t, RenderTicketPriority priority)
@@ -986,7 +1009,7 @@ int ViewerWidget::DeterminePlaybackQueueSize()
   int remaining_frames = (end_ts - GetTimestamp()) / playback_speed_;
 
   // Generate maximum queue
-  int max_frames = qCeil(kAudioPlaybackInterval.toDouble() / timebase().toDouble());
+  int max_frames = qCeil(kVideoPlaybackInterval.toDouble() / timebase().toDouble());
 
   return qMin(max_frames, remaining_frames);
 }
@@ -1095,6 +1118,10 @@ void ViewerWidget::RendererGeneratedFrameForQueue()
     }
   }
 
+  if (first_requeue_watcher_ == watcher) {
+    first_requeue_watcher_ = nullptr;
+  }
+
   delete watcher;
 }
 
@@ -1465,10 +1492,11 @@ void ViewerWidget::PlaybackTimerUpdate()
   }
 
   if (IsPlaying()) {
-    int count = 0;
-    for (int i=display_widget_->queue()->size(); i<DeterminePlaybackQueueSize(); i++) {
-      RequestNextFrameForQueue();
-      count++;
+    while (queue_watchers_.size() < DeterminePlaybackQueueSize()) {
+      if (!RequestNextFrameForQueue()) {
+        // Prevent infinite loop
+        break;
+      }
     }
   }
 

app/widget/viewer/viewer.h

@@ -195,7 +195,7 @@ private:
 
   void SetDisplayImage(QVariant frame);
 
-  void RequestNextFrameForQueue(RenderTicketPriority priority = RenderTicketPriority::kNormal, bool increment = true);
+  RenderTicketWatcher *RequestNextFrameForQueue(RenderTicketPriority priority = RenderTicketPriority::kNormal, bool increment = true);
 
   RenderTicketPtr GetFrame(const rational& t, RenderTicketPriority priority);
 
@@ -280,6 +280,7 @@ private:
   QString recording_filename_;
 
   qint64 queue_starved_start_;
+  RenderTicketWatcher *first_requeue_watcher_;
 
   bool enable_audio_scrubbing_;