Add hardware decoding support

This commit is contained in:
2026-07-13 10:19:30 +08:00
parent f6211f97a5
commit 68c0477715
6 changed files with 404 additions and 38 deletions
+179 -2
View File
@@ -1235,6 +1235,9 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational &time,
previous = cached_frames_.back();
}
// Transfer hardware decoded frames to system memory before caching.
filtered = TransferHardwareFrame(filtered);
// Append this frame and signal to other threads that a new frame has arrived
cached_frames_.push_back(filtered);
@@ -1261,6 +1264,36 @@ AVFramePtr FFmpegDecoder::RetrieveFrame(const rational &time,
return return_frame;
}
AVFramePtr FFmpegDecoder::TransferHardwareFrame(AVFramePtr f)
{
if (!instance_.hwaccel_enabled() ||
f->format != instance_.hw_pix_fmt()) {
return f;
}
AVFrame *sw_frame = av_frame_alloc();
if (!sw_frame) {
qCritical() << "Failed to allocate software frame for hardware transfer";
return nullptr;
}
int ret = av_hwframe_transfer_data(sw_frame, f.get(), 0);
if (ret < 0) {
qWarning() << "Failed to transfer hardware frame to system memory:"
<< FFmpegError(ret);
av_frame_free(&sw_frame);
return nullptr;
}
ret = av_frame_copy_props(sw_frame, f.get());
if (ret < 0) {
qWarning() << "Failed to copy frame properties during hardware transfer:"
<< FFmpegError(ret);
}
return CreateAVFramePtr(sw_frame);
}
void FFmpegDecoder::FreeScaler()
{
if (sws_ctx_) {
@@ -1323,6 +1356,10 @@ FFmpegDecoder::Instance::Instance()
, codec_ctx_(nullptr)
, avstream_(nullptr)
, opts_(nullptr)
, hw_device_ctx_(nullptr)
, hw_device_type_(AV_HWDEVICE_TYPE_NONE)
, hw_pix_fmt_(AV_PIX_FMT_NONE)
, hwaccel_enabled_(false)
{
}
@@ -1379,7 +1416,7 @@ bool FFmpegDecoder::Instance::Open(const char *filename, int stream_index)
// Handle failure to copy parameters
if (error_code < 0) {
qCritical()
<< "Failed to copy parameters from AVStream to AVCodecContext";
<< "Failed to copy parameters from AVStream to AVCodecContext";
return false;
}
@@ -1391,7 +1428,41 @@ bool FFmpegDecoder::Instance::Open(const char *filename, int stream_index)
qCritical() << "Failed to set codec options, performance may suffer";
}
// Open codec
// Attempt hardware accelerated decoding first, then fall back to software.
if (InitHardwareAcceleration(codec)) {
error_code = avcodec_open2(codec_ctx_, codec, &opts_);
if (error_code == 0) {
hwaccel_enabled_ = true;
qDebug() << "Hardware decoding enabled for" << filename
<< "using" << av_hwdevice_get_type_name(hw_device_type_)
<< "pixel format" << av_get_pix_fmt_name(hw_pix_fmt_);
return true;
}
qWarning() << "Failed to open hardware codec, falling back to software decoding:";
char buf[512];
av_strerror(error_code, buf, 512);
qWarning() << FFmpegError(error_code) << buf;
// Free the failed context and recreate it for software decoding.
avcodec_free_context(&codec_ctx_);
CleanupHardwareAcceleration();
codec_ctx_ = avcodec_alloc_context3(codec);
if (codec_ctx_ == nullptr) {
qCritical() << "Failed to allocate codec context for software fallback";
return false;
}
error_code = avcodec_parameters_to_context(codec_ctx_, avstream_->codecpar);
if (error_code < 0) {
qCritical()
<< "Failed to copy parameters from AVStream to AVCodecContext";
return false;
}
}
// Open codec (software path, or if hardware was not available)
error_code = avcodec_open2(codec_ctx_, codec, &opts_);
if (error_code < 0) {
char buf[512];
@@ -1403,6 +1474,110 @@ bool FFmpegDecoder::Instance::Open(const char *filename, int stream_index)
return true;
}
AVHWDeviceType FFmpegDecoder::Instance::ChooseHardwareDevice()
{
#ifdef Q_OS_LINUX
// Prefer NVIDIA's NVDEC where available, then VAAPI/VDPAU.
for (AVHWDeviceType type : { AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_VAAPI,
AV_HWDEVICE_TYPE_VDPAU }) {
if (av_hwdevice_find_type_by_name(av_hwdevice_get_type_name(type)) !=
AV_HWDEVICE_TYPE_NONE) {
return type;
}
}
#elif defined(Q_OS_WIN)
for (AVHWDeviceType type : { AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_DXVA2,
AV_HWDEVICE_TYPE_CUDA }) {
if (av_hwdevice_find_type_by_name(av_hwdevice_get_type_name(type)) !=
AV_HWDEVICE_TYPE_NONE) {
return type;
}
}
#elif defined(Q_OS_MACOS)
if (av_hwdevice_find_type_by_name(
av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_VIDEOTOOLBOX)) !=
AV_HWDEVICE_TYPE_NONE) {
return AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
}
#endif
return AV_HWDEVICE_TYPE_NONE;
}
AVPixelFormat FFmpegDecoder::Instance::GetHardwareFormat(
AVCodecContext *ctx, const AVPixelFormat *pix_fmts)
{
const Instance *inst = static_cast<const Instance *>(ctx->opaque);
for (const AVPixelFormat *p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
if (*p == inst->hw_pix_fmt_) {
return *p;
}
}
qWarning() << "Hardware pixel format not supported by decoder, using first software format";
return pix_fmts[0];
}
bool FFmpegDecoder::Instance::InitHardwareAcceleration(const AVCodec *codec)
{
const AVHWDeviceType device_type = ChooseHardwareDevice();
if (device_type == AV_HWDEVICE_TYPE_NONE) {
return false;
}
// Find the pixel format associated with this device type for this codec.
hw_pix_fmt_ = AV_PIX_FMT_NONE;
for (int i = 0;; i++) {
const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i);
if (!config) {
break;
}
if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) &&
config->device_type == device_type) {
hw_pix_fmt_ = config->pix_fmt;
break;
}
}
if (hw_pix_fmt_ == AV_PIX_FMT_NONE) {
qDebug() << "Codec" << codec->id
<< "does not support hardware device type"
<< av_hwdevice_get_type_name(device_type);
return false;
}
hw_device_type_ = device_type;
int ret = av_hwdevice_ctx_create(&hw_device_ctx_, device_type, nullptr,
nullptr, 0);
if (ret < 0) {
qWarning() << "Failed to create hardware device context for"
<< av_hwdevice_get_type_name(device_type) << ":"
<< FFmpegError(ret);
CleanupHardwareAcceleration();
return false;
}
codec_ctx_->hw_device_ctx = av_buffer_ref(hw_device_ctx_);
codec_ctx_->opaque = this;
codec_ctx_->get_format = GetHardwareFormat;
// Most hardware decoders do not support frame threading.
av_dict_set(&opts_, "threads", "1", 0);
return true;
}
void FFmpegDecoder::Instance::CleanupHardwareAcceleration()
{
hwaccel_enabled_ = false;
hw_device_type_ = AV_HWDEVICE_TYPE_NONE;
hw_pix_fmt_ = AV_PIX_FMT_NONE;
if (hw_device_ctx_) {
av_buffer_unref(&hw_device_ctx_);
hw_device_ctx_ = nullptr;
}
}
void FFmpegDecoder::Instance::Close()
{
if (opts_) {
@@ -1415,6 +1590,8 @@ void FFmpegDecoder::Instance::Close()
codec_ctx_ = nullptr;
}
CleanupHardwareAcceleration();
if (fmt_ctx_) {
avformat_close_input(&fmt_ctx_);
fmt_ctx_ = nullptr;
+25
View File
@@ -29,6 +29,7 @@ extern "C" {
#include <libavcodec/avcodec.h>
#include <libavfilter/avfilter.h>
#include <libavformat/avformat.h>
#include <libavutil/hwcontext.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
}
@@ -131,11 +132,33 @@ private:
return codec_ctx_;
}
bool hwaccel_enabled() const
{
return hwaccel_enabled_;
}
AVPixelFormat hw_pix_fmt() const
{
return hw_pix_fmt_;
}
private:
static AVHWDeviceType ChooseHardwareDevice();
static AVPixelFormat GetHardwareFormat(AVCodecContext *ctx,
const AVPixelFormat *pix_fmts);
bool InitHardwareAcceleration(const AVCodec *codec);
void CleanupHardwareAcceleration();
AVFormatContext *fmt_ctx_;
AVCodecContext *codec_ctx_;
AVStream *avstream_;
AVDictionary *opts_;
AVBufferRef *hw_device_ctx_;
AVHWDeviceType hw_device_type_;
AVPixelFormat hw_pix_fmt_;
bool hwaccel_enabled_;
};
/**
@@ -150,6 +173,8 @@ private:
void FreeScaler();
AVFramePtr TransferHardwareFrame(AVFramePtr f);
static PixelFormat GetNativePixelFormat(AVPixelFormat pix_fmt);
static int GetNativeChannelCount(AVPixelFormat pix_fmt);
+90 -6
View File
@@ -21,6 +21,7 @@
#include "ociolut.h"
#include <QFileInfo>
#include <QMetaObject>
#include "node/color/colormanager/colormanager.h"
@@ -54,6 +55,8 @@ OCIOLutNode::OCIOLutNode()
AddInput(kDirectionInput, NodeValue::kCombo, 0,
InputFlags(kInputFlagNotKeyframable | kInputFlagNotConnectable));
qRegisterMetaType<olive::ColorProcessorPtr>();
}
QString OCIOLutNode::Name() const
@@ -102,14 +105,29 @@ void OCIOLutNode::ConfigChanged()
void OCIOLutNode::GenerateProcessor()
{
QMutexLocker locker(&gen_mutex_);
if (!manager()) {
set_processor(nullptr);
last_processor_.reset();
last_path_.clear();
last_direction_ = -1;
return;
}
const QString path = GetStandardValue(kFileInput).toString();
const int direction = GetStandardValue(kDirectionInput).toInt();
if (path.isEmpty()) {
set_processor(nullptr);
last_processor_.reset();
last_path_.clear();
last_direction_ = -1;
return;
}
// Re-use the existing processor if the file and direction haven't changed.
if (path == last_path_ && direction == last_direction_ && last_processor_) {
return;
}
@@ -117,6 +135,9 @@ void OCIOLutNode::GenerateProcessor()
if (!info.exists() || !info.isFile()) {
qWarning() << "OCIO LUT file does not exist:" << path;
set_processor(nullptr);
last_processor_.reset();
last_path_.clear();
last_direction_ = -1;
return;
}
@@ -124,24 +145,87 @@ void OCIOLutNode::GenerateProcessor()
if (!IsSupportedLutExtension(suffix)) {
qWarning() << "Unsupported OCIO LUT file extension:" << path;
set_processor(nullptr);
last_processor_.reset();
last_path_.clear();
last_direction_ = -1;
return;
}
pending_path_ = path;
pending_direction_ = direction;
pending_generation_++;
const int generation = pending_generation_;
ColorManager *manager = this->manager();
locker.unlock();
QThreadPool::globalInstance()->start(
new GenerateProcessorTask(this, path, direction, generation, manager));
}
void OCIOLutNode::SetProcessorResult(ColorProcessorPtr processor,
const QString &path, int direction,
int generation)
{
QMutexLocker locker(&gen_mutex_);
if (generation != pending_generation_) {
// A newer request was issued while this one was in flight; ignore.
return;
}
if (path != pending_path_ || direction != pending_direction_) {
return;
}
last_path_ = path;
last_direction_ = direction;
last_processor_ = processor;
set_processor(processor);
}
OCIOLutNode::GenerateProcessorTask::GenerateProcessorTask(
OCIOLutNode *node, const QString &path, int direction, int generation,
ColorManager *manager)
: node_(node), path_(path), direction_(direction), generation_(generation),
manager_(manager)
{
setAutoDelete(true);
}
void OCIOLutNode::GenerateProcessorTask::run()
{
ColorProcessorPtr processor = CreateProcessor(path_, direction_, manager_);
if (node_) {
QMetaObject::invokeMethod(
node_, "SetProcessorResult", Qt::QueuedConnection,
Q_ARG(olive::ColorProcessorPtr, processor), Q_ARG(QString, path_),
Q_ARG(int, direction_), Q_ARG(int, generation_));
}
}
ColorProcessorPtr OCIOLutNode::GenerateProcessorTask::CreateProcessor(
const QString &path, int direction, ColorManager *manager)
{
if (!manager) {
return nullptr;
}
try {
OCIO::FileTransformRcPtr transform = OCIO::FileTransform::Create();
transform->setSrc(path.toUtf8().constData());
transform->setInterpolation(OCIO::INTERP_LINEAR);
transform->setDirection(
static_cast<ColorProcessor::Direction>(
GetStandardValue(kDirectionInput).toInt()) == ColorProcessor::kNormal
static_cast<ColorProcessor::Direction>(direction) ==
ColorProcessor::kNormal
? OCIO::TRANSFORM_DIR_FORWARD
: OCIO::TRANSFORM_DIR_INVERSE);
set_processor(ColorProcessor::Create(
manager()->GetConfig()->getProcessor(transform)));
} catch (const OCIO::Exception &e) {
return ColorProcessor::Create(manager->GetConfig()->getProcessor(transform));
} catch (const std::exception &e) {
qWarning() << "OCIO LUT processor error:" << e.what();
set_processor(nullptr);
return nullptr;
}
}
+39 -1
View File
@@ -21,6 +21,11 @@
#ifndef OCIOLUTNODE_H
#define OCIOLUTNODE_H
#include <QMutex>
#include <QPointer>
#include <QRunnable>
#include <QThreadPool>
#include "node/color/ociobase/ociobase.h"
#include "render/colorprocessor.h"
@@ -41,7 +46,7 @@ public:
virtual void Retranslate() override;
virtual void InputValueChangedEvent(const QString &input,
int element) override;
int element) override;
static const QString kFileInput;
static const QString kDirectionInput;
@@ -49,8 +54,41 @@ public:
protected slots:
virtual void ConfigChanged() override;
private slots:
void SetProcessorResult(olive::ColorProcessorPtr processor,
const QString &path, int direction, int generation);
private:
class GenerateProcessorTask : public QRunnable {
public:
GenerateProcessorTask(OCIOLutNode *node, const QString &path,
int direction, int generation,
ColorManager *manager);
void run() override;
private:
static ColorProcessorPtr CreateProcessor(const QString &path,
int direction,
ColorManager *manager);
QPointer<OCIOLutNode> node_;
QString path_;
int direction_;
int generation_;
ColorManager *manager_;
};
void GenerateProcessor();
QMutex gen_mutex_;
QString last_path_;
int last_direction_ = -1;
ColorProcessorPtr last_processor_;
QString pending_path_;
int pending_direction_ = -1;
int pending_generation_ = 0;
};
} // namespace olive
+20 -3
View File
@@ -229,12 +229,29 @@ bool DecodeInputFrames(DecoderCache *decoder_cache,
QString WorkerProgramPath()
{
const QString dir = QCoreApplication::applicationDirPath();
#if defined(Q_OS_WIN)
return QDir(dir).filePath(QStringLiteral("olive-render-worker.exe"));
const QString file = QStringLiteral("olive-render-worker.exe");
#else
return QDir(dir).filePath(QStringLiteral("olive-render-worker"));
const QString file = QStringLiteral("olive-render-worker");
#endif
const QString app_dir = QCoreApplication::applicationDirPath();
const QStringList candidates = {
QDir(app_dir).filePath(file),
QDir(app_dir).filePath(QStringLiteral("../app/") + file),
};
for (const QString &path : candidates) {
if (QFileInfo::exists(path)) {
return path;
}
}
if (qEnvironmentVariableIsSet("OAK_RENDER_WORKER")) {
return QString::fromUtf8(qgetenv("OAK_RENDER_WORKER"));
}
return candidates.first();
}
bool WriteControlMessage(QProcess *process, const QJsonObject &obj)
+51 -26
View File
@@ -260,30 +260,6 @@ protected:
return false;
}
// ---- publish the decoded frame to the input pool ----
uint32_t input_slot = 0;
if (!input_pool_->Acquire(&input_slot)) {
return false;
}
EXPECT_EQ(input_slot, 0u);
std::memcpy(input_pool_->SlotData(input_slot), frame->const_data(),
input_data_bytes_);
ipc::FrameSlotMeta *meta = input_pool_->Meta(input_slot);
meta->id = 0;
meta->time_num = 0;
meta->time_den = 1;
meta->width = input_width_;
meta->height = input_height_;
meta->format = int32_t(frame->format());
meta->channel_count = frame->channel_count();
meta->linesize = input_stride_;
meta->data_size = int32_t(input_data_bytes_);
std::strncpy(meta->colorspace,
frame->video_params().colorspace().toUtf8().constData(),
sizeof(meta->colorspace) - 1);
meta->colorspace[sizeof(meta->colorspace) - 1] = '\0';
input_pool_->Publish(input_slot);
// ---- load graph ----
ipc::LoadGraphMsg load;
load.path = project_file_;
@@ -306,6 +282,30 @@ protected:
bool RenderFrameAndWait(int *output_slot)
{
// Publish the decoded frame to the input pool. The worker consumes it and
// releases it back, so we re-publish before every render.
uint32_t input_slot = 0;
if (!input_pool_->Acquire(&input_slot)) {
return false;
}
std::memcpy(input_pool_->SlotData(input_slot), decoded_frame_->const_data(),
input_data_bytes_);
ipc::FrameSlotMeta *meta = input_pool_->Meta(input_slot);
meta->id = 0;
meta->time_num = 0;
meta->time_den = 1;
meta->width = input_width_;
meta->height = input_height_;
meta->format = int32_t(decoded_frame_->format());
meta->channel_count = decoded_frame_->channel_count();
meta->linesize = input_stride_;
meta->data_size = int32_t(input_data_bytes_);
std::strncpy(meta->colorspace,
decoded_frame_->video_params().colorspace().toUtf8().constData(),
sizeof(meta->colorspace) - 1);
meta->colorspace[sizeof(meta->colorspace) - 1] = '\0';
input_pool_->Publish(input_slot);
ipc::RenderFrameMsg req;
req.ticket_id = 1;
req.node_uuid = footage_id_;
@@ -318,15 +318,24 @@ protected:
req.mode = int(RenderMode::kOnline);
req.input_slot = 0;
if (!ipc::WriteMessage(&worker_, req.ToJson())) {
std::cerr << "RenderFrameAndWait: failed to write request" << std::endl;
return false;
}
QJsonObject ready;
if (!WaitForMessage(&ready)) {
std::cerr << "RenderFrameAndWait: failed to receive ready message"
<< std::endl;
return false;
}
if (ready[QStringLiteral("type")].toString() !=
QLatin1String(ipc::msgtype::kFrameReady)) {
std::cerr << "RenderFrameAndWait: unexpected message type "
<< ready[QStringLiteral("type")].toString().toStdString()
<< " body="
<< QJsonDocument(ready).toJson(QJsonDocument::Compact)
.toStdString()
<< std::endl;
return false;
}
*output_slot = ready[QStringLiteral("slot")].toInt();
@@ -346,12 +355,18 @@ protected:
return true;
}
if (!ok) {
std::cerr << "WaitForMessage: parse error, buffer="
<< read_buffer_.toStdString() << std::endl;
return false;
}
if (worker_.state() == QProcess::NotRunning) {
std::cerr << "WaitForMessage: worker exited with code "
<< worker_.exitCode() << std::endl;
return false;
}
}
std::cerr << "WaitForMessage: timeout, buffer="
<< read_buffer_.toStdString() << std::endl;
return false;
}
@@ -397,7 +412,11 @@ TEST_F(RenderWorkerFootageTest, VulkanFootageIsNotBlack)
ASSERT_GE(output_slot, 0);
ASSERT_LT(output_slot, kOutputSlots);
const void *output_data = output_pool_->SlotData(uint32_t(output_slot));
uint32_t consumed_slot = 0;
ASSERT_TRUE(output_pool_->Consume(&consumed_slot));
ASSERT_EQ(int(consumed_slot), output_slot);
const void *output_data = output_pool_->SlotData(consumed_slot);
const double brightness = SampleBrightnessF32(
output_data, output_width_, output_height_,
output_width_ * 4 * int(sizeof(float)));
@@ -411,6 +430,7 @@ TEST_F(RenderWorkerFootageTest, VulkanFootageIsNotBlack)
QFile::copy(temp_dir_.filePath(QStringLiteral("worker_output_vulkan.png")),
QStringLiteral("/tmp/worker_output_vulkan.png"));
std::cerr << "Vulkan output copied to /tmp/worker_output_vulkan.png" << std::endl;
output_pool_->Release(consumed_slot);
}
TEST_F(RenderWorkerFootageTest, OpenGLFootageIsNotBlack)
@@ -422,7 +442,11 @@ TEST_F(RenderWorkerFootageTest, OpenGLFootageIsNotBlack)
ASSERT_GE(output_slot, 0);
ASSERT_LT(output_slot, kOutputSlots);
const void *output_data = output_pool_->SlotData(uint32_t(output_slot));
uint32_t consumed_slot = 0;
ASSERT_TRUE(output_pool_->Consume(&consumed_slot));
ASSERT_EQ(int(consumed_slot), output_slot);
const void *output_data = output_pool_->SlotData(consumed_slot);
const double brightness = SampleBrightnessF32(
output_data, output_width_, output_height_,
output_width_ * 4 * int(sizeof(float)));
@@ -436,5 +460,6 @@ TEST_F(RenderWorkerFootageTest, OpenGLFootageIsNotBlack)
QFile::copy(temp_dir_.filePath(QStringLiteral("worker_output_opengl.png")),
QStringLiteral("/tmp/worker_output_opengl.png"));
std::cerr << "OpenGL output copied to /tmp/worker_output_opengl.png" << std::endl;
output_pool_->Release(consumed_slot);
}