renderer: keep frames in vram while paused

Saves some GPU cycles when adjusting parameters.
This commit is contained in:
itsmattkc
2020-05-08 00:32:02 +10:00
parent 5b956729b3
commit 65705d580d
17 changed files with 212 additions and 150 deletions
+2
View File
@@ -47,7 +47,9 @@ OLIVE_NAMESPACE_EXIT
#define MACRO_NAME_AS_STR(s) #s
#define MACRO_VAL_AS_STR(s) MACRO_NAME_AS_STR(s)
#define OLIVE_NS_CONST_ARG(x, y) QArgument<const OLIVE_NAMESPACE::x>("const " MACRO_VAL_AS_STR(OLIVE_NAMESPACE) "::" #x, y)
#define OLIVE_NS_ARG(x, y) QArgument<OLIVE_NAMESPACE::x>(MACRO_VAL_AS_STR(OLIVE_NAMESPACE) "::" #x, y)
#define OLIVE_NS_RETURN_ARG(x, y) QReturnArgument<OLIVE_NAMESPACE::x>(MACRO_VAL_AS_STR(OLIVE_NAMESPACE) "::" #x, y)
/**
* Copy/move deleters. Similar to Q_DISABLE_COPY_MOVE, et al. but those functions are not present in Qt < 5.13 so we
+6
View File
@@ -120,6 +120,12 @@ void ImageStream::set_colorspace(const QString &color)
emit ParametersChanged();
}
QString ImageStream::get_colorspace_match_string() const
{
return QStringLiteral("%1:%2").arg(footage()->project()->color_manager()->GetConfigFilename(),
colorspace());
}
void ImageStream::ColorConfigChanged()
{
ColorManager* color_manager = footage()->project()->color_manager();
+2
View File
@@ -48,6 +48,8 @@ public:
const QString& colorspace(bool default_if_empty = true) const;
void set_colorspace(const QString& color);
QString get_colorspace_match_string() const;
protected:
virtual void FootageSetEvent(Footage*) override;
+4 -6
View File
@@ -27,21 +27,19 @@ AudioWorker::AudioWorker(QHash<Node *, Node *> *copy_map, QObject *parent) :
{
}
void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table)
NodeValue AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range)
{
if (stream->type() != Stream::kAudio) {
return;
}
if (decoder->HasConformedVersion(audio_params())) {
SampleBufferPtr frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params());
if (frame) {
table->Push(NodeParam::kSamples, QVariant::fromValue(frame));
return NodeValue(NodeParam::kSamples, QVariant::fromValue(frame));
}
} else {
emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params());
}
return NodeValue();
}
void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, NodeValueDatabase &input_params_in, NodeValueTable &output_params)
+1 -1
View File
@@ -31,7 +31,7 @@ public:
AudioWorker(QHash<Node*, Node*>* copy_map, QObject* parent = nullptr);
protected:
virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override;
virtual NodeValue FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range) override;
virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, NodeValueDatabase& input_params, NodeValueTable& output_params) override;
+11
View File
@@ -154,6 +154,17 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
return merged_table;
}
void AudioRenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable *table)
{
if (stream->type() != Stream::kAudio) {
return;
}
NodeValue value = GetDataFromStream(stream, input_time);
table->Push(value);
}
const AudioRenderingParams &AudioRenderWorker::audio_params() const
{
return audio_params_;
+2
View File
@@ -43,6 +43,8 @@ protected:
virtual NodeValueTable RenderBlock(const TrackOutput *track, const TimeRange& range) override;
virtual void FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable* table) override;
const AudioRenderingParams& audio_params() const;
private:
+1 -5
View File
@@ -63,13 +63,9 @@ bool OpenGLBackend::InitInternal()
// Initiate one thread per CPU core
for (int i=0;i<threads().size();i++) {
// Create one processor object for each thread
OpenGLWorker* processor = new OpenGLWorker(frame_cache());
OpenGLWorker* processor = new OpenGLWorker(frame_cache(), proxy_);
processor->SetParameters(params());
processors_.append(processor);
connect(processor, &OpenGLWorker::RequestFrameToValue, proxy_, &OpenGLProxy::FrameToValue, Qt::BlockingQueuedConnection);
connect(processor, &OpenGLWorker::RequestTextureToBuffer, proxy_, &OpenGLProxy::TextureToBuffer, Qt::BlockingQueuedConnection);
connect(processor, &OpenGLWorker::RequestRunNodeAccelerated, proxy_, &OpenGLProxy::RunNodeAccelerated, Qt::BlockingQueuedConnection);
}
return true;
+74 -100
View File
@@ -67,133 +67,107 @@ bool OpenGLProxy::Init()
return true;
}
void OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream, NodeValueTable* table)
NodeValue OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream)
{
// Ensure stream is video or image type
if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) {
return;
}
ImageStreamPtr video_stream = std::static_pointer_cast<ImageStream>(stream);
// Set up OCIO context
QString colorspace_match = QStringLiteral("%1:%2").arg(video_stream->footage()->project()->color_manager()->GetConfigFilename(), video_stream->colorspace());
QString colorspace_match = video_stream->get_colorspace_match_string();
OpenGLTextureCache::ReferencePtr footage_tex_ref = nullptr;
OpenGLColorProcessorPtr color_processor = std::static_pointer_cast<OpenGLColorProcessor>(color_cache_.Get(colorspace_match));
if (stream->type() == Stream::kImage && still_image_cache_.Has(stream.get())) {
CachedStill cs = still_image_cache_.Get(stream.get());
if (!color_processor) {
color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager(),
video_stream->colorspace(),
video_stream->footage()->project()->color_manager()->GetReferenceColorSpace());
color_cache_.Add(colorspace_match, color_processor);
}
if (cs.colorspace == colorspace_match
&& cs.alpha_is_associated == video_stream->premultiplied_alpha()
&& cs.divider == video_params_.divider()) {
footage_tex_ref = cs.texture;
} else {
still_image_cache_.Remove(stream.get());
ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode());
// OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU
if (ocio_method == ColorManager::kOCIOAccurate) {
bool has_alpha = PixelFormat::FormatHasAlphaChannel(frame->format());
// Convert frame to float for OCIO
frame = PixelFormat::ConvertPixelFormat(frame,
has_alpha
? PixelFormat::PIX_FMT_RGBA32F
: PixelFormat::PIX_FMT_RGB32F);
// If alpha is associated, disassociate for the color transform
if (has_alpha && video_stream->premultiplied_alpha()) {
ColorManager::DisassociateAlpha(frame);
}
// Perform color transform
color_processor->ConvertFrame(frame);
// Associate alpha
if (has_alpha) {
if (video_stream->premultiplied_alpha()) {
ColorManager::ReassociateAlpha(frame);
} else {
ColorManager::AssociateAlpha(frame);
}
}
}
if (!footage_tex_ref) {
OpenGLColorProcessorPtr color_processor = std::static_pointer_cast<OpenGLColorProcessor>(color_cache_.Get(colorspace_match));
OpenGLTextureCache::ReferencePtr footage_tex_ref = texture_cache_.Get(ctx_, frame);
if (!color_processor) {
color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager(),
video_stream->colorspace(),
video_stream->footage()->project()->color_manager()->GetReferenceColorSpace());
color_cache_.Add(colorspace_match, color_processor);
if (ocio_method == ColorManager::kOCIOFast) {
if (!color_processor->IsEnabled()) {
color_processor->Enable(ctx_, video_stream->premultiplied_alpha());
}
ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode());
VideoRenderingParams frame_params = frame->video_params();
// OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU
if (ocio_method == ColorManager::kOCIOAccurate) {
bool has_alpha = PixelFormat::FormatHasAlphaChannel(frame->format());
// Check frame aspect ratio
if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) {
int new_width = frame_params.width();
int new_height = frame_params.height();
// Convert frame to float for OCIO
frame = PixelFormat::ConvertPixelFormat(frame,
has_alpha
? PixelFormat::PIX_FMT_RGBA32F
: PixelFormat::PIX_FMT_RGB32F);
// If alpha is associated, disassociate for the color transform
if (has_alpha && video_stream->premultiplied_alpha()) {
ColorManager::DisassociateAlpha(frame);
// Scale the frame in a way that does not reduce the resolution
if (frame->sample_aspect_ratio() > 1) {
// Make wider
new_width = qRound(static_cast<double>(new_width) * frame->sample_aspect_ratio().toDouble());
} else {
// Make taller
new_height = qRound(static_cast<double>(new_height) / frame->sample_aspect_ratio().toDouble());
}
// Perform color transform
color_processor->ConvertFrame(frame);
// Associate alpha
if (has_alpha) {
if (video_stream->premultiplied_alpha()) {
ColorManager::ReassociateAlpha(frame);
} else {
ColorManager::AssociateAlpha(frame);
}
}
frame_params = VideoRenderingParams(new_width,
new_height,
frame_params.format(),
frame_params.divider());
}
footage_tex_ref = texture_cache_.Get(ctx_, frame);
VideoRenderingParams dest_params(frame_params.width(),
frame_params.height(),
video_params_.format(),
frame_params.divider());
if (ocio_method == ColorManager::kOCIOFast) {
if (!color_processor->IsEnabled()) {
color_processor->Enable(ctx_, video_stream->premultiplied_alpha());
}
// Create destination texture
OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, dest_params);
VideoRenderingParams frame_params = frame->video_params();
buffer_.Attach(associated_tex_ref->texture(), true);
buffer_.Bind();
footage_tex_ref->texture()->Bind();
// Check frame aspect ratio
if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) {
int new_width = frame_params.width();
int new_height = frame_params.height();
// Set viewport for texture size
functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height());
// Scale the frame in a way that does not reduce the resolution
if (frame->sample_aspect_ratio() > 1) {
// Make wider
new_width = qRound(static_cast<double>(new_width) * frame->sample_aspect_ratio().toDouble());
} else {
// Make taller
new_height = qRound(static_cast<double>(new_height) / frame->sample_aspect_ratio().toDouble());
}
// Blit old texture to new texture through OCIO shader
color_processor->ProcessOpenGL();
frame_params = VideoRenderingParams(new_width,
new_height,
frame_params.format(),
frame_params.divider());
}
footage_tex_ref->texture()->Release();
buffer_.Release();
buffer_.Detach();
VideoRenderingParams dest_params(frame_params.width(),
frame_params.height(),
video_params_.format(),
frame_params.divider());
// Create destination texture
OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, dest_params);
buffer_.Attach(associated_tex_ref->texture(), true);
buffer_.Bind();
footage_tex_ref->texture()->Bind();
// Set viewport for texture size
functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height());
// Blit old texture to new texture through OCIO shader
color_processor->ProcessOpenGL();
footage_tex_ref->texture()->Release();
buffer_.Release();
buffer_.Detach();
footage_tex_ref = associated_tex_ref;
}
if (stream->type() == Stream::kImage) {
// Since this is a still image, we could likely optimize this
still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha(), video_params_.divider()});
}
footage_tex_ref = associated_tex_ref;
}
table->Push(NodeParam::kTexture, QVariant::fromValue(footage_tex_ref));
return NodeValue(NodeParam::kTexture, QVariant::fromValue(footage_tex_ref));
}
void OpenGLProxy::Close()
+6 -13
View File
@@ -31,7 +31,8 @@
OLIVE_NAMESPACE_ENTER
class OpenGLProxy : public QObject {
class OpenGLProxy : public QObject
{
Q_OBJECT
public:
OpenGLProxy(QObject* parent = nullptr);
@@ -63,13 +64,14 @@ public:
void Close();
void FrameToValue(FramePtr frame, StreamPtr stream, NodeValueTable* table);
void SetParameters(const VideoRenderingParams& params);
void RunNodeAccelerated(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable& output_params);
public slots:
void RunNodeAccelerated(const OLIVE_NAMESPACE::Node *node, const OLIVE_NAMESPACE::TimeRange &range, OLIVE_NAMESPACE::NodeValueDatabase &input_params, OLIVE_NAMESPACE::NodeValueTable& output_params);
void TextureToBuffer(const QVariant& texture, int width, int height, const QMatrix4x4& matrix, void *buffer, int linesize);
void SetParameters(const VideoRenderingParams& params);
OLIVE_NAMESPACE::NodeValue FrameToValue(OLIVE_NAMESPACE::FramePtr frame, OLIVE_NAMESPACE::StreamPtr stream);
private:
QOpenGLContext* ctx_;
@@ -89,15 +91,6 @@ private:
OpenGLTextureCache texture_cache_;
struct CachedStill {
OpenGLTextureCache::ReferencePtr texture;
QString colorspace;
bool alpha_is_associated;
int divider;
};
RenderCache<Stream*, CachedStill> still_image_cache_;
private slots:
void FinishInit();
+30 -6
View File
@@ -31,30 +31,54 @@
OLIVE_NAMESPACE_ENTER
OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, QObject *parent) :
VideoRenderWorker(frame_cache, parent)
OpenGLWorker::OpenGLWorker(VideoRenderFrameCache *frame_cache, OpenGLProxy *proxy, QObject *parent) :
VideoRenderWorker(frame_cache, parent),
proxy_(proxy)
{
}
void OpenGLWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable *table)
NodeValue OpenGLWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range)
{
FramePtr frame = decoder->RetrieveVideo(range.in(),
video_params().divider(),
video_params().mode() == RenderMode::kOffline);
NodeValue value;
if (frame) {
emit RequestFrameToValue(frame, stream, table);
QMetaObject::invokeMethod(proxy_,
"FrameToValue",
Qt::BlockingQueuedConnection,
OLIVE_NS_RETURN_ARG(NodeValue, value),
OLIVE_NS_ARG(FramePtr, frame),
OLIVE_NS_ARG(StreamPtr, stream));
}
return value;
}
void OpenGLWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params)
{
emit RequestRunNodeAccelerated(node, range, input_params, output_params);
QMetaObject::invokeMethod(proxy_,
"RunNodeAccelerated",
Qt::BlockingQueuedConnection,
OLIVE_NS_CONST_ARG(Node*, node),
OLIVE_NS_CONST_ARG(TimeRange&, range),
OLIVE_NS_ARG(NodeValueDatabase&, input_params),
OLIVE_NS_ARG(NodeValueTable&, output_params));
}
void OpenGLWorker::TextureToBuffer(const QVariant &tex_in, int width, int height, const QMatrix4x4& matrix, void *buffer, int linesize)
{
emit RequestTextureToBuffer(tex_in, width, height, matrix, buffer, linesize);
QMetaObject::invokeMethod(proxy_,
"TextureToBuffer",
Qt::BlockingQueuedConnection,
Q_ARG(const QVariant&, tex_in),
Q_ARG(int, width),
Q_ARG(int, height),
Q_ARG(const QMatrix4x4&, matrix),
Q_ARG(void*, buffer),
Q_ARG(int, linesize));
}
OLIVE_NAMESPACE_EXIT
+6 -8
View File
@@ -26,6 +26,7 @@
#include "../videorenderworker.h"
#include "openglframebuffer.h"
#include "openglproxy.h"
#include "openglshadercache.h"
#include "opengltexturecache.h"
@@ -35,22 +36,19 @@ class OpenGLWorker : public VideoRenderWorker {
Q_OBJECT
public:
OpenGLWorker(VideoRenderFrameCache* frame_cache,
OpenGLProxy* proxy,
QObject* parent = nullptr);
signals:
void RequestFrameToValue(FramePtr frame, StreamPtr stream, NodeValueTable* table);
void RequestRunNodeAccelerated(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable& output_params);
void RequestTextureToBuffer(const QVariant& texture, int width, int height, const QMatrix4x4& matrix, void *buffer, int linesize);
protected:
virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) override;
virtual NodeValue FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range) override;
virtual void RunNodeAccelerated(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable& output_params) override;
virtual void TextureToBuffer(const QVariant& texture, int width, int height, const QMatrix4x4& matrix, void *buffer, int linesize) override;
private:
OpenGLProxy* proxy_;
};
OLIVE_NAMESPACE_EXIT
+9 -7
View File
@@ -98,18 +98,20 @@ DecoderPtr RenderWorker::ResolveDecoderFromInput(StreamPtr stream)
return decoder;
}
bool RenderWorker::IsStarted()
{
return started_;
}
void RenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange& input_time, NodeValueTable *table)
NodeValue RenderWorker::GetDataFromStream(StreamPtr stream, const TimeRange &input_time)
{
DecoderPtr decoder = ResolveDecoderFromInput(stream);
if (decoder) {
FrameToValue(decoder, stream, input_time, table);
return FrameToValue(decoder, stream, input_time);
}
return NodeValue();
}
bool RenderWorker::IsStarted()
{
return started_;
}
void RenderWorker::ProcessNodeEvent(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params)
+3 -3
View File
@@ -57,14 +57,14 @@ protected:
virtual void RunNodeAccelerated(const Node *node, const TimeRange& range, NodeValueDatabase &input_params, NodeValueTable &output_params);
virtual void FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range, NodeValueTable* table) = 0;
virtual void FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable* table) override;
virtual NodeValue FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeRange &range) = 0;
virtual void ProcessNodeEvent(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params) override;
DecoderPtr ResolveDecoderFromInput(StreamPtr stream);
NodeValue GetDataFromStream(StreamPtr stream, const TimeRange& input_time);
const NodeDependency& CurrentPath() const;
private:
+42
View File
@@ -116,6 +116,48 @@ NodeValueTable VideoRenderWorker::RenderInternal(const NodeDependency& path, con
return value;
}
void VideoRenderWorker::FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable *table)
{
if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) {
return;
}
ImageStreamPtr video_stream = std::static_pointer_cast<ImageStream>(stream);
rational time_match = (stream->type() == Stream::kImage) ? rational() : input_time.in();
QString colorspace_match = video_stream->get_colorspace_match_string();
NodeValue value;
bool found_cache = false;
if (still_image_cache_.Has(stream.get())) {
CachedStill cs = still_image_cache_.Get(stream.get());
if (cs.colorspace == colorspace_match
&& cs.alpha_is_associated == video_stream->premultiplied_alpha()
&& cs.divider == video_params_.divider()
&& cs.time == time_match) {
value = cs.texture;
found_cache = true;
} else {
still_image_cache_.Remove(stream.get());
}
}
if (!found_cache) {
value = GetDataFromStream(stream, input_time);
still_image_cache_.Add(stream.get(), {value,
colorspace_match,
video_stream->premultiplied_alpha(),
video_params_.divider(),
time_match});
}
table->Push(value);
}
void VideoRenderWorker::SetParameters(const VideoRenderingParams &video_params)
{
video_params_ = video_params;
+12
View File
@@ -98,6 +98,8 @@ protected:
virtual NodeValueTable RenderInternal(const NodeDependency& CurrentPath, const qint64& job_time) override;
virtual void FootageProcessingEvent(StreamPtr stream, const TimeRange &input_time, NodeValueTable* table) override;
ColorProcessorCache* color_cache();
private:
@@ -119,6 +121,16 @@ private:
OperatingMode operating_mode_;
struct CachedStill {
NodeValue texture;
QString colorspace;
bool alpha_is_associated;
int divider;
rational time;
};
RenderCache<Stream*, CachedStill> still_image_cache_;
private slots:
};
+1 -1
View File
@@ -328,7 +328,7 @@ void ViewerDisplayWidget::paintGL()
int cross = qMin(w, h) / 32;
QLine lines[] = {QLine(rect().center().x() - cross, rect().center().y(), rect().center().x() + cross, rect().center().y()),
QLine lines[] = {QLine(rect().center().x() - cross, rect().center().y(),rect().center().x() + cross, rect().center().y()),
QLine(rect().center().x(), rect().center().y() - cross, rect().center().x(), rect().center().y() + cross)};
p.drawLines(lines, 2);