diff --git a/app/codec/ffmpeg/ffmpegdecoder.cpp b/app/codec/ffmpeg/ffmpegdecoder.cpp index 5a486a180..1d211e355 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.cpp +++ b/app/codec/ffmpeg/ffmpegdecoder.cpp @@ -1235,6 +1235,9 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational &time, previous = cached_frames_.back(); } + // Transfer hardware decoded frames to system memory before caching. + filtered = TransferHardwareFrame(filtered); + // Append this frame and signal to other threads that a new frame has arrived cached_frames_.push_back(filtered); @@ -1261,6 +1264,36 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational &time, return return_frame; } +AVFramePtr FFmpegDecoder::TransferHardwareFrame(AVFramePtr f) +{ + if (!instance_.hwaccel_enabled() || + f->format != instance_.hw_pix_fmt()) { + return f; + } + + AVFrame *sw_frame = av_frame_alloc(); + if (!sw_frame) { + qCritical() << "Failed to allocate software frame for hardware transfer"; + return nullptr; + } + + int ret = av_hwframe_transfer_data(sw_frame, f.get(), 0); + if (ret < 0) { + qWarning() << "Failed to transfer hardware frame to system memory:" + << FFmpegError(ret); + av_frame_free(&sw_frame); + return nullptr; + } + + ret = av_frame_copy_props(sw_frame, f.get()); + if (ret < 0) { + qWarning() << "Failed to copy frame properties during hardware transfer:" + << FFmpegError(ret); + } + + return CreateAVFramePtr(sw_frame); +} + void FFmpegDecoder::FreeScaler() { if (sws_ctx_) { @@ -1323,6 +1356,10 @@ FFmpegDecoder::Instance::Instance() , codec_ctx_(nullptr) , avstream_(nullptr) , opts_(nullptr) + , hw_device_ctx_(nullptr) + , hw_device_type_(AV_HWDEVICE_TYPE_NONE) + , hw_pix_fmt_(AV_PIX_FMT_NONE) + , hwaccel_enabled_(false) { } @@ -1379,7 +1416,7 @@ bool FFmpegDecoder::Instance::Open(const char *filename, int stream_index) // Handle failure to copy parameters if (error_code < 0) { qCritical() - << "Failed to copy parameters from AVStream to AVCodecContext"; + << "Failed to copy parameters from AVStream to AVCodecContext"; return false; } @@ -1391,7 +1428,41 @@ bool FFmpegDecoder::Instance::Open(const char *filename, int stream_index) qCritical() << "Failed to set codec options, performance may suffer"; } - // Open codec + // Attempt hardware accelerated decoding first, then fall back to software. + if (InitHardwareAcceleration(codec)) { + error_code = avcodec_open2(codec_ctx_, codec, &opts_); + if (error_code == 0) { + hwaccel_enabled_ = true; + qDebug() << "Hardware decoding enabled for" << filename + << "using" << av_hwdevice_get_type_name(hw_device_type_) + << "pixel format" << av_get_pix_fmt_name(hw_pix_fmt_); + return true; + } + + qWarning() << "Failed to open hardware codec, falling back to software decoding:"; + char buf[512]; + av_strerror(error_code, buf, 512); + qWarning() << FFmpegError(error_code) << buf; + + // Free the failed context and recreate it for software decoding. + avcodec_free_context(&codec_ctx_); + CleanupHardwareAcceleration(); + + codec_ctx_ = avcodec_alloc_context3(codec); + if (codec_ctx_ == nullptr) { + qCritical() << "Failed to allocate codec context for software fallback"; + return false; + } + + error_code = avcodec_parameters_to_context(codec_ctx_, avstream_->codecpar); + if (error_code < 0) { + qCritical() + << "Failed to copy parameters from AVStream to AVCodecContext"; + return false; + } + } + + // Open codec (software path, or if hardware was not available) error_code = avcodec_open2(codec_ctx_, codec, &opts_); if (error_code < 0) { char buf[512]; @@ -1403,6 +1474,110 @@ bool FFmpegDecoder::Instance::Open(const char *filename, int stream_index) return true; } +AVHWDeviceType FFmpegDecoder::Instance::ChooseHardwareDevice() +{ +#ifdef Q_OS_LINUX + // Prefer NVIDIA's NVDEC where available, then VAAPI/VDPAU. + for (AVHWDeviceType type : { AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_VAAPI, + AV_HWDEVICE_TYPE_VDPAU }) { + if (av_hwdevice_find_type_by_name(av_hwdevice_get_type_name(type)) != + AV_HWDEVICE_TYPE_NONE) { + return type; + } + } +#elif defined(Q_OS_WIN) + for (AVHWDeviceType type : { AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_DXVA2, + AV_HWDEVICE_TYPE_CUDA }) { + if (av_hwdevice_find_type_by_name(av_hwdevice_get_type_name(type)) != + AV_HWDEVICE_TYPE_NONE) { + return type; + } + } +#elif defined(Q_OS_MACOS) + if (av_hwdevice_find_type_by_name( + av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_VIDEOTOOLBOX)) != + AV_HWDEVICE_TYPE_NONE) { + return AV_HWDEVICE_TYPE_VIDEOTOOLBOX; + } +#endif + return AV_HWDEVICE_TYPE_NONE; +} + +AVPixelFormat FFmpegDecoder::Instance::GetHardwareFormat( + AVCodecContext *ctx, const AVPixelFormat *pix_fmts) +{ + const Instance *inst = static_cast(ctx->opaque); + for (const AVPixelFormat *p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { + if (*p == inst->hw_pix_fmt_) { + return *p; + } + } + + qWarning() << "Hardware pixel format not supported by decoder, using first software format"; + return pix_fmts[0]; +} + +bool FFmpegDecoder::Instance::InitHardwareAcceleration(const AVCodec *codec) +{ + const AVHWDeviceType device_type = ChooseHardwareDevice(); + if (device_type == AV_HWDEVICE_TYPE_NONE) { + return false; + } + + // Find the pixel format associated with this device type for this codec. + hw_pix_fmt_ = AV_PIX_FMT_NONE; + for (int i = 0;; i++) { + const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i); + if (!config) { + break; + } + if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) && + config->device_type == device_type) { + hw_pix_fmt_ = config->pix_fmt; + break; + } + } + + if (hw_pix_fmt_ == AV_PIX_FMT_NONE) { + qDebug() << "Codec" << codec->id + << "does not support hardware device type" + << av_hwdevice_get_type_name(device_type); + return false; + } + + hw_device_type_ = device_type; + + int ret = av_hwdevice_ctx_create(&hw_device_ctx_, device_type, nullptr, + nullptr, 0); + if (ret < 0) { + qWarning() << "Failed to create hardware device context for" + << av_hwdevice_get_type_name(device_type) << ":" + << FFmpegError(ret); + CleanupHardwareAcceleration(); + return false; + } + + codec_ctx_->hw_device_ctx = av_buffer_ref(hw_device_ctx_); + codec_ctx_->opaque = this; + codec_ctx_->get_format = GetHardwareFormat; + // Most hardware decoders do not support frame threading. + av_dict_set(&opts_, "threads", "1", 0); + + return true; +} + +void FFmpegDecoder::Instance::CleanupHardwareAcceleration() +{ + hwaccel_enabled_ = false; + hw_device_type_ = AV_HWDEVICE_TYPE_NONE; + hw_pix_fmt_ = AV_PIX_FMT_NONE; + + if (hw_device_ctx_) { + av_buffer_unref(&hw_device_ctx_); + hw_device_ctx_ = nullptr; + } +} + void FFmpegDecoder::Instance::Close() { if (opts_) { @@ -1415,6 +1590,8 @@ void FFmpegDecoder::Instance::Close() codec_ctx_ = nullptr; } + CleanupHardwareAcceleration(); + if (fmt_ctx_) { avformat_close_input(&fmt_ctx_); fmt_ctx_ = nullptr; diff --git a/app/codec/ffmpeg/ffmpegdecoder.h b/app/codec/ffmpeg/ffmpegdecoder.h index f187fd2d4..42a44ad6f 100644 --- a/app/codec/ffmpeg/ffmpegdecoder.h +++ b/app/codec/ffmpeg/ffmpegdecoder.h @@ -29,6 +29,7 @@ extern "C" { #include #include #include +#include #include #include } @@ -131,11 +132,33 @@ private: return codec_ctx_; } + bool hwaccel_enabled() const + { + return hwaccel_enabled_; + } + + AVPixelFormat hw_pix_fmt() const + { + return hw_pix_fmt_; + } + private: + static AVHWDeviceType ChooseHardwareDevice(); + static AVPixelFormat GetHardwareFormat(AVCodecContext *ctx, + const AVPixelFormat *pix_fmts); + + bool InitHardwareAcceleration(const AVCodec *codec); + void CleanupHardwareAcceleration(); + AVFormatContext *fmt_ctx_; AVCodecContext *codec_ctx_; AVStream *avstream_; AVDictionary *opts_; + + AVBufferRef *hw_device_ctx_; + AVHWDeviceType hw_device_type_; + AVPixelFormat hw_pix_fmt_; + bool hwaccel_enabled_; }; /** @@ -150,6 +173,8 @@ private: void FreeScaler(); + AVFramePtr TransferHardwareFrame(AVFramePtr f); + static PixelFormat GetNativePixelFormat(AVPixelFormat pix_fmt); static int GetNativeChannelCount(AVPixelFormat pix_fmt); diff --git a/app/node/color/ociolut/ociolut.cpp b/app/node/color/ociolut/ociolut.cpp index 3845567a6..ee6c27c08 100644 --- a/app/node/color/ociolut/ociolut.cpp +++ b/app/node/color/ociolut/ociolut.cpp @@ -21,6 +21,7 @@ #include "ociolut.h" #include +#include #include "node/color/colormanager/colormanager.h" @@ -54,6 +55,8 @@ OCIOLutNode::OCIOLutNode() AddInput(kDirectionInput, NodeValue::kCombo, 0, InputFlags(kInputFlagNotKeyframable | kInputFlagNotConnectable)); + + qRegisterMetaType(); } QString OCIOLutNode::Name() const @@ -102,14 +105,29 @@ void OCIOLutNode::ConfigChanged() void OCIOLutNode::GenerateProcessor() { + QMutexLocker locker(&gen_mutex_); + if (!manager()) { set_processor(nullptr); + last_processor_.reset(); + last_path_.clear(); + last_direction_ = -1; return; } const QString path = GetStandardValue(kFileInput).toString(); + const int direction = GetStandardValue(kDirectionInput).toInt(); + if (path.isEmpty()) { set_processor(nullptr); + last_processor_.reset(); + last_path_.clear(); + last_direction_ = -1; + return; + } + + // Re-use the existing processor if the file and direction haven't changed. + if (path == last_path_ && direction == last_direction_ && last_processor_) { return; } @@ -117,6 +135,9 @@ void OCIOLutNode::GenerateProcessor() if (!info.exists() || !info.isFile()) { qWarning() << "OCIO LUT file does not exist:" << path; set_processor(nullptr); + last_processor_.reset(); + last_path_.clear(); + last_direction_ = -1; return; } @@ -124,24 +145,87 @@ void OCIOLutNode::GenerateProcessor() if (!IsSupportedLutExtension(suffix)) { qWarning() << "Unsupported OCIO LUT file extension:" << path; set_processor(nullptr); + last_processor_.reset(); + last_path_.clear(); + last_direction_ = -1; return; } + pending_path_ = path; + pending_direction_ = direction; + pending_generation_++; + const int generation = pending_generation_; + ColorManager *manager = this->manager(); + + locker.unlock(); + + QThreadPool::globalInstance()->start( + new GenerateProcessorTask(this, path, direction, generation, manager)); +} + +void OCIOLutNode::SetProcessorResult(ColorProcessorPtr processor, + const QString &path, int direction, + int generation) +{ + QMutexLocker locker(&gen_mutex_); + + if (generation != pending_generation_) { + // A newer request was issued while this one was in flight; ignore. + return; + } + + if (path != pending_path_ || direction != pending_direction_) { + return; + } + + last_path_ = path; + last_direction_ = direction; + last_processor_ = processor; + set_processor(processor); +} + +OCIOLutNode::GenerateProcessorTask::GenerateProcessorTask( + OCIOLutNode *node, const QString &path, int direction, int generation, + ColorManager *manager) + : node_(node), path_(path), direction_(direction), generation_(generation), + manager_(manager) +{ + setAutoDelete(true); +} + +void OCIOLutNode::GenerateProcessorTask::run() +{ + ColorProcessorPtr processor = CreateProcessor(path_, direction_, manager_); + + if (node_) { + QMetaObject::invokeMethod( + node_, "SetProcessorResult", Qt::QueuedConnection, + Q_ARG(olive::ColorProcessorPtr, processor), Q_ARG(QString, path_), + Q_ARG(int, direction_), Q_ARG(int, generation_)); + } +} + +ColorProcessorPtr OCIOLutNode::GenerateProcessorTask::CreateProcessor( + const QString &path, int direction, ColorManager *manager) +{ + if (!manager) { + return nullptr; + } + try { OCIO::FileTransformRcPtr transform = OCIO::FileTransform::Create(); transform->setSrc(path.toUtf8().constData()); transform->setInterpolation(OCIO::INTERP_LINEAR); transform->setDirection( - static_cast( - GetStandardValue(kDirectionInput).toInt()) == ColorProcessor::kNormal + static_cast(direction) == + ColorProcessor::kNormal ? OCIO::TRANSFORM_DIR_FORWARD : OCIO::TRANSFORM_DIR_INVERSE); - set_processor(ColorProcessor::Create( - manager()->GetConfig()->getProcessor(transform))); - } catch (const OCIO::Exception &e) { + return ColorProcessor::Create(manager->GetConfig()->getProcessor(transform)); + } catch (const std::exception &e) { qWarning() << "OCIO LUT processor error:" << e.what(); - set_processor(nullptr); + return nullptr; } } diff --git a/app/node/color/ociolut/ociolut.h b/app/node/color/ociolut/ociolut.h index 98daa8177..3380f059b 100644 --- a/app/node/color/ociolut/ociolut.h +++ b/app/node/color/ociolut/ociolut.h @@ -21,6 +21,11 @@ #ifndef OCIOLUTNODE_H #define OCIOLUTNODE_H +#include +#include +#include +#include + #include "node/color/ociobase/ociobase.h" #include "render/colorprocessor.h" @@ -41,7 +46,7 @@ public: virtual void Retranslate() override; virtual void InputValueChangedEvent(const QString &input, - int element) override; + int element) override; static const QString kFileInput; static const QString kDirectionInput; @@ -49,8 +54,41 @@ public: protected slots: virtual void ConfigChanged() override; +private slots: + void SetProcessorResult(olive::ColorProcessorPtr processor, + const QString &path, int direction, int generation); + private: + class GenerateProcessorTask : public QRunnable { + public: + GenerateProcessorTask(OCIOLutNode *node, const QString &path, + int direction, int generation, + ColorManager *manager); + + void run() override; + + private: + static ColorProcessorPtr CreateProcessor(const QString &path, + int direction, + ColorManager *manager); + + QPointer node_; + QString path_; + int direction_; + int generation_; + ColorManager *manager_; + }; + void GenerateProcessor(); + + QMutex gen_mutex_; + QString last_path_; + int last_direction_ = -1; + ColorProcessorPtr last_processor_; + + QString pending_path_; + int pending_direction_ = -1; + int pending_generation_ = 0; }; } // namespace olive diff --git a/app/render/renderworkerpool.cpp b/app/render/renderworkerpool.cpp index fdd66987f..7b102df51 100644 --- a/app/render/renderworkerpool.cpp +++ b/app/render/renderworkerpool.cpp @@ -229,12 +229,29 @@ bool DecodeInputFrames(DecoderCache *decoder_cache, QString WorkerProgramPath() { - const QString dir = QCoreApplication::applicationDirPath(); #if defined(Q_OS_WIN) - return QDir(dir).filePath(QStringLiteral("olive-render-worker.exe")); + const QString file = QStringLiteral("olive-render-worker.exe"); #else - return QDir(dir).filePath(QStringLiteral("olive-render-worker")); + const QString file = QStringLiteral("olive-render-worker"); #endif + + const QString app_dir = QCoreApplication::applicationDirPath(); + const QStringList candidates = { + QDir(app_dir).filePath(file), + QDir(app_dir).filePath(QStringLiteral("../app/") + file), + }; + + for (const QString &path : candidates) { + if (QFileInfo::exists(path)) { + return path; + } + } + + if (qEnvironmentVariableIsSet("OAK_RENDER_WORKER")) { + return QString::fromUtf8(qgetenv("OAK_RENDER_WORKER")); + } + + return candidates.first(); } bool WriteControlMessage(QProcess *process, const QJsonObject &obj) diff --git a/tests/gtest/render_worker_footage_test.cpp b/tests/gtest/render_worker_footage_test.cpp index ba7eb1f87..4a6edb203 100644 --- a/tests/gtest/render_worker_footage_test.cpp +++ b/tests/gtest/render_worker_footage_test.cpp @@ -260,30 +260,6 @@ protected: return false; } - // ---- publish the decoded frame to the input pool ---- - uint32_t input_slot = 0; - if (!input_pool_->Acquire(&input_slot)) { - return false; - } - EXPECT_EQ(input_slot, 0u); - std::memcpy(input_pool_->SlotData(input_slot), frame->const_data(), - input_data_bytes_); - ipc::FrameSlotMeta *meta = input_pool_->Meta(input_slot); - meta->id = 0; - meta->time_num = 0; - meta->time_den = 1; - meta->width = input_width_; - meta->height = input_height_; - meta->format = int32_t(frame->format()); - meta->channel_count = frame->channel_count(); - meta->linesize = input_stride_; - meta->data_size = int32_t(input_data_bytes_); - std::strncpy(meta->colorspace, - frame->video_params().colorspace().toUtf8().constData(), - sizeof(meta->colorspace) - 1); - meta->colorspace[sizeof(meta->colorspace) - 1] = '\0'; - input_pool_->Publish(input_slot); - // ---- load graph ---- ipc::LoadGraphMsg load; load.path = project_file_; @@ -306,6 +282,30 @@ protected: bool RenderFrameAndWait(int *output_slot) { + // Publish the decoded frame to the input pool. The worker consumes it and + // releases it back, so we re-publish before every render. + uint32_t input_slot = 0; + if (!input_pool_->Acquire(&input_slot)) { + return false; + } + std::memcpy(input_pool_->SlotData(input_slot), decoded_frame_->const_data(), + input_data_bytes_); + ipc::FrameSlotMeta *meta = input_pool_->Meta(input_slot); + meta->id = 0; + meta->time_num = 0; + meta->time_den = 1; + meta->width = input_width_; + meta->height = input_height_; + meta->format = int32_t(decoded_frame_->format()); + meta->channel_count = decoded_frame_->channel_count(); + meta->linesize = input_stride_; + meta->data_size = int32_t(input_data_bytes_); + std::strncpy(meta->colorspace, + decoded_frame_->video_params().colorspace().toUtf8().constData(), + sizeof(meta->colorspace) - 1); + meta->colorspace[sizeof(meta->colorspace) - 1] = '\0'; + input_pool_->Publish(input_slot); + ipc::RenderFrameMsg req; req.ticket_id = 1; req.node_uuid = footage_id_; @@ -318,15 +318,24 @@ protected: req.mode = int(RenderMode::kOnline); req.input_slot = 0; if (!ipc::WriteMessage(&worker_, req.ToJson())) { + std::cerr << "RenderFrameAndWait: failed to write request" << std::endl; return false; } QJsonObject ready; if (!WaitForMessage(&ready)) { + std::cerr << "RenderFrameAndWait: failed to receive ready message" + << std::endl; return false; } if (ready[QStringLiteral("type")].toString() != QLatin1String(ipc::msgtype::kFrameReady)) { + std::cerr << "RenderFrameAndWait: unexpected message type " + << ready[QStringLiteral("type")].toString().toStdString() + << " body=" + << QJsonDocument(ready).toJson(QJsonDocument::Compact) + .toStdString() + << std::endl; return false; } *output_slot = ready[QStringLiteral("slot")].toInt(); @@ -346,12 +355,18 @@ protected: return true; } if (!ok) { + std::cerr << "WaitForMessage: parse error, buffer=" + << read_buffer_.toStdString() << std::endl; return false; } if (worker_.state() == QProcess::NotRunning) { + std::cerr << "WaitForMessage: worker exited with code " + << worker_.exitCode() << std::endl; return false; } } + std::cerr << "WaitForMessage: timeout, buffer=" + << read_buffer_.toStdString() << std::endl; return false; } @@ -397,7 +412,11 @@ TEST_F(RenderWorkerFootageTest, VulkanFootageIsNotBlack) ASSERT_GE(output_slot, 0); ASSERT_LT(output_slot, kOutputSlots); - const void *output_data = output_pool_->SlotData(uint32_t(output_slot)); + uint32_t consumed_slot = 0; + ASSERT_TRUE(output_pool_->Consume(&consumed_slot)); + ASSERT_EQ(int(consumed_slot), output_slot); + + const void *output_data = output_pool_->SlotData(consumed_slot); const double brightness = SampleBrightnessF32( output_data, output_width_, output_height_, output_width_ * 4 * int(sizeof(float))); @@ -411,6 +430,7 @@ TEST_F(RenderWorkerFootageTest, VulkanFootageIsNotBlack) QFile::copy(temp_dir_.filePath(QStringLiteral("worker_output_vulkan.png")), QStringLiteral("/tmp/worker_output_vulkan.png")); std::cerr << "Vulkan output copied to /tmp/worker_output_vulkan.png" << std::endl; + output_pool_->Release(consumed_slot); } TEST_F(RenderWorkerFootageTest, OpenGLFootageIsNotBlack) @@ -422,7 +442,11 @@ TEST_F(RenderWorkerFootageTest, OpenGLFootageIsNotBlack) ASSERT_GE(output_slot, 0); ASSERT_LT(output_slot, kOutputSlots); - const void *output_data = output_pool_->SlotData(uint32_t(output_slot)); + uint32_t consumed_slot = 0; + ASSERT_TRUE(output_pool_->Consume(&consumed_slot)); + ASSERT_EQ(int(consumed_slot), output_slot); + + const void *output_data = output_pool_->SlotData(consumed_slot); const double brightness = SampleBrightnessF32( output_data, output_width_, output_height_, output_width_ * 4 * int(sizeof(float))); @@ -436,5 +460,6 @@ TEST_F(RenderWorkerFootageTest, OpenGLFootageIsNotBlack) QFile::copy(temp_dir_.filePath(QStringLiteral("worker_output_opengl.png")), QStringLiteral("/tmp/worker_output_opengl.png")); std::cerr << "OpenGL output copied to /tmp/worker_output_opengl.png" << std::endl; + output_pool_->Release(consumed_slot); }