diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 81ef1a306..977f1c251 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -52,12 +52,15 @@ extern "C" { namespace olive { +QVariant Yuv2RgbShader; + FFmpegDecoder::FFmpegDecoder() : filter_graph_(nullptr), buffersrc_ctx_(nullptr), buffersink_ctx_(nullptr), input_fmt_(AV_PIX_FMT_NONE), - native_pix_fmt_(VideoParams::kFormatInvalid), + native_internal_pix_fmt_(VideoParams::kFormatInvalid), + native_output_pix_fmt_(VideoParams::kFormatInvalid), working_frame_(nullptr), working_packet_(nullptr), is_working_(false), @@ -142,28 +145,116 @@ bool FFmpegDecoder::OpenInternal() TexturePtr FFmpegDecoder::RetrieveVideoInternal(Renderer *renderer, const rational &timecode, const RetrieveVideoParams ¶ms, const QAtomicInt *cancelled) { if (AVFramePtr f = RetrieveFrame(timecode, cancelled)) { - if (InitScaler(f.get(), params)) { - int r; - r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF); - if (r < 0) { - return nullptr; - } - r = av_buffersink_get_frame(buffersink_ctx_, working_frame_); - if (r < 0) { - return nullptr; - } + if (cancelled && *cancelled) { + return nullptr; + } + if (InitScaler(f.get(), params)) { VideoParams vp(instance_.avstream()->codecpar->width, instance_.avstream()->codecpar->height, - native_pix_fmt_, + native_output_pix_fmt_, native_channel_count_, av_guess_sample_aspect_ratio(instance_.fmt_ctx(), instance_.avstream(), nullptr), VideoParams::kInterlaceNone, params.divider); - TexturePtr tex = renderer->CreateTexture(vp, working_frame_->data[0], working_frame_->linesize[0] / vp.GetBytesPerPixel()); + TexturePtr tex = nullptr; + const bool hwscale = true; - av_frame_unref(working_frame_); + // Attempt to use GLSL shader for faster YUV to RGB conversion + if (hwscale) { + AVPixelFormat src_fmt = AVPixelFormat(f.get()->format); + if (src_fmt == AV_PIX_FMT_YUV420P + || src_fmt == AV_PIX_FMT_YUV422P + || src_fmt == AV_PIX_FMT_YUV444P + || src_fmt == AV_PIX_FMT_YUV420P10LE + || src_fmt == AV_PIX_FMT_YUV422P10LE + || src_fmt == AV_PIX_FMT_YUV444P10LE + || src_fmt == AV_PIX_FMT_YUV420P12LE + || src_fmt == AV_PIX_FMT_YUV422P12LE + || src_fmt == AV_PIX_FMT_YUV444P12LE) { + if (Yuv2RgbShader.isNull()) { + // Compile shader + Yuv2RgbShader = renderer->CreateNativeShader(ShaderCode(FileFunctions::ReadFileAsString(QStringLiteral(":/shaders/yuv2rgb.frag")))); + } + + if (!Yuv2RgbShader.isNull()) { + int px_size; + int bits_per_pixel; + switch (src_fmt) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUV444P: + default: + px_size = 1; + bits_per_pixel = 8; + break; + case AV_PIX_FMT_YUV420P10LE: + case AV_PIX_FMT_YUV422P10LE: + case AV_PIX_FMT_YUV444P10LE: + px_size = 2; + bits_per_pixel = 10; + break; + case AV_PIX_FMT_YUV420P12LE: + case AV_PIX_FMT_YUV422P12LE: + case AV_PIX_FMT_YUV444P12LE: + px_size = 2; + bits_per_pixel = 12; + break; + } + + VideoParams plane_params = vp; + plane_params.set_channel_count(1); + plane_params.set_divider(1); + plane_params.set_format(native_internal_pix_fmt_); + TexturePtr y_plane = renderer->CreateTexture(plane_params, f->data[0], f->linesize[0] / px_size); + + if (src_fmt == AV_PIX_FMT_YUV420P + || src_fmt == AV_PIX_FMT_YUV422P + || src_fmt == AV_PIX_FMT_YUV420P10LE + || src_fmt == AV_PIX_FMT_YUV422P10LE + || src_fmt == AV_PIX_FMT_YUV420P12LE + || src_fmt == AV_PIX_FMT_YUV422P12LE) { + plane_params.set_width(plane_params.width()/2); + } + + if (src_fmt == AV_PIX_FMT_YUV420P + || src_fmt == AV_PIX_FMT_YUV420P10LE + || src_fmt == AV_PIX_FMT_YUV420P12LE) { + plane_params.set_height(plane_params.height()/2); + } + + TexturePtr u_plane = renderer->CreateTexture(plane_params, f->data[1], f->linesize[1] / px_size); + TexturePtr v_plane = renderer->CreateTexture(plane_params, f->data[2], f->linesize[2] / px_size); + + ShaderJob job; + job.Insert(QStringLiteral("y_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(y_plane))); + job.Insert(QStringLiteral("u_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(u_plane))); + job.Insert(QStringLiteral("v_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(v_plane))); + job.Insert(QStringLiteral("bits_per_pixel"), NodeValue(NodeValue::kInt, bits_per_pixel)); + + tex = renderer->CreateTexture(vp); + renderer->BlitToTexture(Yuv2RgbShader, job, tex.get(), false); + } + } + } + + if (!tex) { + // Fallback to software pixel format conversion + int r; + r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF); + if (r < 0) { + return nullptr; + } + r = av_buffersink_get_frame(buffersink_ctx_, working_frame_); + if (r < 0) { + return nullptr; + } + + tex = renderer->CreateTexture(vp, working_frame_->data[0], working_frame_->linesize[0] / vp.GetBytesPerPixel()); + + av_frame_unref(working_frame_); + } return tex; } @@ -190,7 +281,8 @@ void FFmpegDecoder::CloseInternal() instance_.Close(); input_fmt_ = AV_PIX_FMT_NONE; - native_pix_fmt_ = VideoParams::kFormatInvalid; + native_internal_pix_fmt_ = VideoParams::kFormatInvalid; + native_output_pix_fmt_ = VideoParams::kFormatInvalid; } QString FFmpegDecoder::id() const @@ -702,6 +794,10 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational& time, const QAtomicInt * // Pull from the decoder ret = instance_.GetFrame(working_packet_, filtered.get()); + if (cancelled && *cancelled) { + break; + } + // Handle any errors that aren't EOF (EOF is handled later on) if (ret < 0 && ret != AVERROR_EOF) { qCritical() << "Failed to retrieve frame:" << ret; @@ -804,10 +900,14 @@ bool FFmpegDecoder::InitScaler(AVFrame *input, const RetrieveVideoParams& params // Determine which Olive native pixel format we retrieved // Note that FFmpeg doesn't support float formats - native_pix_fmt_ = GetNativePixelFormat(ideal_pix_fmt); + native_output_pix_fmt_ = GetNativePixelFormat(ideal_pix_fmt); native_channel_count_ = GetNativeChannelCount(ideal_pix_fmt); - if (native_pix_fmt_ == VideoParams::kFormatInvalid + AVPixelFormat ideal_internal_pix_fmt = FFmpegUtils::GetCompatiblePixelFormat(static_cast(input_fmt_)); + native_internal_pix_fmt_ = GetNativePixelFormat(ideal_internal_pix_fmt); + + if (native_output_pix_fmt_ == VideoParams::kFormatInvalid + || native_internal_pix_fmt_ == VideoParams::kFormatInvalid || native_channel_count_ == 0) { qCritical() << "Failed to find valid native pixel format for" << ideal_pix_fmt; return false; diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index c8d92a9b6..80c46fb1f 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -160,7 +160,8 @@ private: AVFilterContext* buffersrc_ctx_; AVFilterContext* buffersink_ctx_; AVPixelFormat input_fmt_; - VideoParams::Format native_pix_fmt_; + VideoParams::Format native_internal_pix_fmt_; + VideoParams::Format native_output_pix_fmt_; int native_channel_count_; AVFrame *working_frame_; diff --git a/app/shaders/yuv2rgb.frag b/app/shaders/yuv2rgb.frag new file mode 100644 index 000000000..62c73e149 --- /dev/null +++ b/app/shaders/yuv2rgb.frag @@ -0,0 +1,35 @@ +uniform sampler2D y_channel; +uniform sampler2D u_channel; +uniform sampler2D v_channel; + +uniform int bits_per_pixel; + +in vec2 ove_texcoord; +out vec4 frag_color; + +void main() { + vec4 rgba; + + vec3 yuv; + + yuv.r = texture(y_channel, ove_texcoord).r; + yuv.g = texture(u_channel, ove_texcoord).r; + yuv.b = texture(v_channel, ove_texcoord).r; + + if (bits_per_pixel == 10) { + yuv *= 64.0; + } else if (bits_per_pixel == 12) { + yuv *= 16.0; + } + + yuv.r = 1.1643 * (yuv.r - 0.0625); + yuv.g = yuv.g - 0.5; + yuv.b = yuv.b - 0.5; + + rgba.r = yuv.r + 1.5958 * yuv.b; + rgba.g = yuv.r - 0.39173 * yuv.g - 0.81290 * yuv.b; + rgba.b = yuv.r + 2.017 * yuv.g; + rgba.a = 1.0; + + frag_color = rgba; +} diff --git a/app/threading/threadpool.cpp b/app/threading/threadpool.cpp index b048f3519..8b33d72db 100644 --- a/app/threading/threadpool.cpp +++ b/app/threading/threadpool.cpp @@ -29,52 +29,67 @@ ThreadPool::ThreadPool(unsigned threads, QObject *parent) : threads = std::thread::hardware_concurrency(); } + available_count_ = threads; for (unsigned i = 0; i < threads; i += 1) { - worker_threads_.emplace_back(std::bind(&ThreadPool::thread_exec, this)); + worker_threads_.emplace_back(std::bind(&ThreadPool::thread_exec, this, &tasks_, &task_mutex_, &cond_)); } + + // Make single reserved thread for high priority tasks (usually audio) so they don't get stuck + // behind a lot of slow tasks + high_thread_ = std::thread(std::thread(std::bind(&ThreadPool::thread_exec, this, &high_tasks_, &high_mutex_, &high_cond_))); } void ThreadPool::AddTicket(RenderTicketPtr ticket, RenderTicketPriority priority) { - std::lock_guard lock(task_mutex_); - if (priority == RenderTicketPriority::kHigh) { - tasks_.emplace_front(std::move(ticket)); + std::lock_guard lock(high_mutex_); + high_tasks_.emplace_back(std::move(ticket)); + high_cond_.notify_one(); } else { + std::lock_guard lock(task_mutex_); tasks_.emplace_back(std::move(ticket)); + cond_.notify_one(); } - - cond_.notify_one(); } bool ThreadPool::RemoveTicket(RenderTicketPtr ticket) { - std::lock_guard lock(task_mutex_); - - const auto it = std::find(tasks_.begin(), tasks_.end(), ticket); - if (it == tasks_.end()) { - return false; + { + std::lock_guard lock(task_mutex_); + const auto it = std::find(tasks_.begin(), tasks_.end(), ticket); + if (it != tasks_.end()) { + tasks_.erase(it); + return true; + } } - tasks_.erase(it); - return true; + { + std::lock_guard lock(high_mutex_); + const auto it = std::find(high_tasks_.begin(), high_tasks_.end(), ticket); + if (it != high_tasks_.end()) { + high_tasks_.erase(it); + return true; + } + } + + return false; } -void ThreadPool::thread_exec() +void ThreadPool::thread_exec(std::deque *queue, std::mutex *mutex, std::condition_variable *cond) { while (true) { TaskType task; { - std::unique_lock lock(task_mutex_); - cond_.wait(lock, [this]{ return this->end_threadp_ || !this->tasks_.empty(); }); + std::unique_lock lock(*mutex); + cond->wait(lock, [this, queue]{ return this->end_threadp_ || !queue->empty(); }); - if (this->end_threadp_ && this->tasks_.empty()) { + if (this->end_threadp_ && queue->empty()) { break; } - task = std::move(tasks_.front()); - tasks_.pop_front(); + task = std::move(queue->front()); + queue->pop_front(); } RunTicket(task); @@ -85,10 +100,12 @@ ThreadPool::~ThreadPool() { end_threadp_ = true; cond_.notify_all(); + high_cond_.notify_all(); for (auto &e : worker_threads_) { e.join(); } + high_thread_.join(); } } diff --git a/app/threading/threadpool.h b/app/threading/threadpool.h index 3c3204bb1..3c1143f55 100644 --- a/app/threading/threadpool.h +++ b/app/threading/threadpool.h @@ -49,13 +49,20 @@ public: virtual ~ThreadPool() override; private: - void thread_exec(); + void thread_exec(std::deque *queue, std::mutex *mutex, std::condition_variable *cond); std::vector worker_threads_; std::deque tasks_; std::mutex task_mutex_; std::condition_variable cond_; + + std::thread high_thread_; + std::deque high_tasks_; + std::mutex high_mutex_; + std::condition_variable high_cond_; + std::atomic_bool end_threadp_{false}; + std::atomic_int available_count_; }; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 41681b009..35eb765d0 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -60,6 +60,8 @@ QVector ViewerWidget::instances_; // changing values. 1/4 second seems to be a good middleground. const rational ViewerWidget::kAudioPlaybackInterval = rational(1, 4); +const rational kVideoPlaybackInterval = rational(1, 2); + ViewerWidget::ViewerWidget(QWidget *parent) : super(false, true, parent), playback_speed_(0), @@ -69,6 +71,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) : prequeuing_audio_(0), record_armed_(false), recording_(false), + first_requeue_watcher_(nullptr), enable_audio_scrubbing_(true) { // Set up main layout @@ -635,12 +638,20 @@ void ViewerWidget::ReceivedAudioBufferForScrubbing() void ViewerWidget::QueueStarved() { - static const int kMaximumWaitTime = 250; + static const int kMaximumWaitTimeMs = 250; + static const rational kMaximumWaitTime(kMaximumWaitTimeMs, 1000); qint64 now = QDateTime::currentMSecsSinceEpoch(); if (!queue_starved_start_) { queue_starved_start_ = now; - } else if (now > queue_starved_start_ + kMaximumWaitTime) { + } else if (now > queue_starved_start_ + kMaximumWaitTimeMs) { + if (first_requeue_watcher_) { + if (GetTime() + kMaximumWaitTime < first_requeue_watcher_->property("time").value()) { + // We still have time + return; + } + } + ForceRequeueFromCurrentTime(); queue_starved_start_ = 0; } @@ -653,11 +664,19 @@ void ViewerWidget::QueueNoLongerStarved() void ViewerWidget::ForceRequeueFromCurrentTime() { + // Allow half a second for requeue to complete + static const rational kRequeueWaitTime(1); + ClearVideoAutoCacherQueue(); + queue_watchers_.clear(); int queue = DeterminePlaybackQueueSize(); - playback_queue_next_frame_ = GetTimestamp() + playback_speed_; - for (int i=queue_watchers_.size(); isetProperty("time", QVariant::fromValue(next_time)); connect(watcher, &RenderTicketWatcher::Finished, this, &ViewerWidget::RendererGeneratedFrameForQueue); queue_watchers_.append(watcher); watcher->SetTicket(GetFrame(next_time, priority)); } + + return watcher; } RenderTicketPtr ViewerWidget::GetFrame(const rational &t, RenderTicketPriority priority) @@ -986,7 +1009,7 @@ int ViewerWidget::DeterminePlaybackQueueSize() int remaining_frames = (end_ts - GetTimestamp()) / playback_speed_; // Generate maximum queue - int max_frames = qCeil(kAudioPlaybackInterval.toDouble() / timebase().toDouble()); + int max_frames = qCeil(kVideoPlaybackInterval.toDouble() / timebase().toDouble()); return qMin(max_frames, remaining_frames); } @@ -1095,6 +1118,10 @@ void ViewerWidget::RendererGeneratedFrameForQueue() } } + if (first_requeue_watcher_ == watcher) { + first_requeue_watcher_ = nullptr; + } + delete watcher; } @@ -1465,10 +1492,11 @@ void ViewerWidget::PlaybackTimerUpdate() } if (IsPlaying()) { - int count = 0; - for (int i=display_widget_->queue()->size(); i