/*** Olive - Non-Linear Video Editor Copyright (C) 2019 Olive Team This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ***/ #include "openglproxy.h" #include #include "common/clamp.h" #include "core.h" #include "node/block/transition/transition.h" #include "node/node.h" #include "openglcolorprocessor.h" #include "openglrenderfunctions.h" #include "render/colormanager.h" #include "render/pixelformat.h" OLIVE_NAMESPACE_ENTER OpenGLProxy* OpenGLProxy::instance_ = nullptr; OpenGLProxy::OpenGLProxy(QObject *parent) : QObject(parent), ctx_(nullptr), functions_(nullptr) { surface_.create(); } OpenGLProxy::~OpenGLProxy() { Close(); surface_.destroy(); } void OpenGLProxy::CreateInstance() { instance_ = new OpenGLProxy(); QThread* proxy_thread = new QThread(); proxy_thread->start(QThread::IdlePriority); instance_->moveToThread(proxy_thread); if (!instance_->Init()) { DestroyInstance(); } } void OpenGLProxy::DestroyInstance() { if (instance_) { instance_->thread()->quit(); instance_->thread()->wait(); instance_->thread()->deleteLater(); instance_->deleteLater(); instance_ = nullptr; } } bool OpenGLProxy::Init() { // Create context object ctx_ = new QOpenGLContext(); // Create OpenGL context (automatically destroys any existing if there is one) if (!ctx_->create()) { qWarning() << "Failed to create OpenGL context in thread" << thread(); return false; } ctx_->moveToThread(this->thread()); // The rest of the initialization needs to occur in the other thread, so we signal for it to start QMetaObject::invokeMethod(this, "FinishInit", Qt::QueuedConnection); return true; } QVariant OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream, const VideoParams& params, const RenderMode::Mode& mode) { ImageStreamPtr video_stream = std::static_pointer_cast(stream); // Set up OCIO context QString colorspace_match = video_stream->get_colorspace_match_string(); OpenGLColorProcessorPtr color_processor = std::static_pointer_cast(color_cache_.value(colorspace_match)); if (!color_processor) { color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager(), video_stream->colorspace(), video_stream->footage()->project()->color_manager()->GetReferenceColorSpace()); color_cache_.insert(colorspace_match, color_processor); } ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(mode); // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU if (ocio_method == ColorManager::kOCIOAccurate) { bool has_alpha = PixelFormat::FormatHasAlphaChannel(frame->format()); // Convert frame to float for OCIO frame = PixelFormat::ConvertPixelFormat(frame, has_alpha ? PixelFormat::PIX_FMT_RGBA32F : PixelFormat::PIX_FMT_RGB32F); // If alpha is associated, disassociate for the color transform if (has_alpha && video_stream->premultiplied_alpha()) { ColorManager::DisassociateAlpha(frame); } // Perform color transform color_processor->ConvertFrame(frame); // Associate alpha if (has_alpha) { if (video_stream->premultiplied_alpha()) { ColorManager::ReassociateAlpha(frame); } else { ColorManager::AssociateAlpha(frame); } } } OpenGLTextureCache::ReferencePtr footage_tex_ref = texture_cache_.Get(ctx_, frame); if (ocio_method == ColorManager::kOCIOFast) { if (!color_processor->IsEnabled()) { color_processor->Enable(ctx_, video_stream->premultiplied_alpha()); } VideoParams frame_params = frame->video_params(); // Check frame aspect ratio if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) { int new_width = frame_params.width(); int new_height = frame_params.height(); // Scale the frame in a way that does not reduce the resolution if (frame->sample_aspect_ratio() > 1) { // Make wider new_width = qRound(static_cast(new_width) * frame->sample_aspect_ratio().toDouble()); } else { // Make taller new_height = qRound(static_cast(new_height) / frame->sample_aspect_ratio().toDouble()); } frame_params = VideoParams(new_width, new_height, frame_params.format(), frame_params.divider()); } PixelFormat::Format texture_fmt; if (PixelFormat::FormatHasAlphaChannel(frame_params.format())) { texture_fmt = PixelFormat::GetFormatWithAlphaChannel(params.format()); } else { texture_fmt = PixelFormat::GetFormatWithoutAlphaChannel(params.format()); } VideoParams dest_params(frame_params.width(), frame_params.height(), texture_fmt, frame_params.divider()); // Create destination texture OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, dest_params); buffer_.Attach(associated_tex_ref->texture(), true); buffer_.Bind(); footage_tex_ref->texture()->Bind(); // Set viewport for texture size functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height()); // Blit old texture to new texture through OCIO shader color_processor->ProcessOpenGL(); footage_tex_ref->texture()->Release(); buffer_.Release(); buffer_.Detach(); footage_tex_ref = associated_tex_ref; } return QVariant::fromValue(footage_tex_ref); } QVariant OpenGLProxy::PreCachedFrameToValue(FramePtr frame) { return QVariant::fromValue(texture_cache_.Get(ctx_, frame)); } OpenGLShaderPtr OpenGLProxy::ResolveShaderFromCache(const Node *node, const QString &shader_id) { // Make a composite of the node ID and the shader ID (if applicable) QString full_shader_id = QStringLiteral("%1:%2").arg(node->id(), shader_id); OpenGLShaderPtr shader = shader_cache_.value(full_shader_id); if (!shader) { // Since we have shader code, compile it now ShaderCode code = node->GetShaderCode(shader_id); QString vert_code = code.vert_code(); QString frag_code = code.frag_code(); if (frag_code.isEmpty() && vert_code.isEmpty()) { qWarning() << "No shader code found for" << node->id() << "- operation will be a no-op"; } if (frag_code.isEmpty()) { frag_code = OpenGLShader::CodeDefaultFragment(); } if (vert_code.isEmpty()) { vert_code = OpenGLShader::CodeDefaultVertex(); } shader = OpenGLShader::Create(); if (shader && shader->create() && shader->addShaderFromSourceCode(QOpenGLShader::Fragment, frag_code) && shader->addShaderFromSourceCode(QOpenGLShader::Vertex, vert_code) && shader->link()) { shader_cache_.insert(full_shader_id, shader); } else { qWarning() << "Failed to compile shader for" << node->id(); shader = nullptr; } } return shader; } void OpenGLProxy::Close() { shader_cache_.clear(); buffer_.Destroy(); copy_pipeline_ = nullptr; functions_ = nullptr; delete ctx_; ctx_ = nullptr; } QVariant OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, const ShaderJob &job, const VideoParams& params) { // If this node is iterative, we'll pick up which input here GLuint iterative_input = 0; QList textures_to_bind; bool input_textures_have_alpha = false; OpenGLShaderPtr shader = ResolveShaderFromCache(node, job.GetShaderID()); if (!shader) { return QVariant(); } shader->bind(); NodeValueMap::const_iterator i; for (i=job.GetValues().constBegin(); i!=job.GetValues().constEnd(); i++) { // See if the shader has takes this parameter as an input int variable_location = shader->uniformLocation(i.key()->id()); if (variable_location == -1) { continue; } // This variable is used in the shader, let's set it const QVariant& value = i.value().data(); const NodeParam::DataType& data_type = (i.value().type() != NodeParam::kNone) ? i.value().type() : i.key()->data_type(); switch (data_type) { case NodeInput::kInt: shader->setUniformValue(variable_location, value.toInt()); break; case NodeInput::kFloat: shader->setUniformValue(variable_location, value.toFloat()); break; case NodeInput::kVec2: if (i.key()->IsArray()) { QVector nv = value.value< QVector >(); QVector a(nv.size()); for (int j=0;j(); } shader->setUniformValueArray(variable_location, a.constData(), a.size()); int count_location = shader->uniformLocation(QStringLiteral("%1_count").arg(i.key()->id())); if (count_location > -1) { shader->setUniformValue(count_location, a.size()); } } else { shader->setUniformValue(variable_location, value.value()); } break; case NodeInput::kVec3: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kVec4: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kMatrix: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kCombo: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kColor: { Color color = value.value(); shader->setUniformValue(variable_location, color.red(), color.green(), color.blue(), color.alpha()); break; } case NodeInput::kBoolean: shader->setUniformValue(variable_location, value.toBool()); break; case NodeInput::kTexture: { OpenGLTextureCache::ReferencePtr texture = value.value(); if (texture) { if (PixelFormat::FormatHasAlphaChannel(texture->texture()->format())) { input_textures_have_alpha = true; } } // Set value to bound texture shader->setUniformValue(variable_location, textures_to_bind.size()); // If this texture binding is the iterative input, set it here if (i.key() == job.GetIterativeInput()) { iterative_input = textures_to_bind.size(); } GLuint tex_id = texture ? texture->texture()->texture() : 0; textures_to_bind.append(tex_id); // Set enable flag if shader wants it int enable_param_location = shader->uniformLocation(QStringLiteral("%1_enabled").arg(i.key()->id())); if (enable_param_location > -1) { shader->setUniformValue(enable_param_location, tex_id > 0); } if (tex_id > 0) { // Set texture resolution if shader wants it int res_param_location = shader->uniformLocation(QStringLiteral("%1_resolution").arg(i.key()->id())); if (res_param_location > -1) { shader->setUniformValue(res_param_location, static_cast(texture->texture()->width() * texture->texture()->divider()), static_cast(texture->texture()->height() * texture->texture()->divider())); } } break; } case NodeInput::kSamples: case NodeInput::kText: case NodeInput::kRational: case NodeInput::kFont: case NodeInput::kFile: case NodeInput::kDecimal: case NodeInput::kNumber: case NodeInput::kString: case NodeInput::kVector: case NodeInput::kShaderJob: case NodeInput::kSampleJob: case NodeInput::kGenerateJob: case NodeInput::kFootage: case NodeInput::kBuffer: case NodeInput::kNone: case NodeInput::kAny: break; } } // Provide some standard args shader->setUniformValue("ove_resolution", static_cast(params.width()), static_cast(params.height())); if (node->IsBlock() && static_cast(node)->type() == Block::kTransition) { const TransitionBlock* transition_node = static_cast(node); // Provides total transition progress from 0.0 (start) - 1.0 (end) shader->setUniformValue("ove_tprog_all", static_cast(transition_node->GetTotalProgress(range.in()))); // Provides progress of out section from 1.0 (start) - 0.0 (end) shader->setUniformValue("ove_tprog_out", static_cast(transition_node->GetOutProgress(range.in()))); // Provides progress of in section from 0.0 (start) - 1.0 (end) shader->setUniformValue("ove_tprog_in", static_cast(transition_node->GetInProgress(range.in()))); } shader->release(); // Create the output textures PixelFormat::Format output_format = (input_textures_have_alpha || job.GetAlphaChannelRequired()) ? PixelFormat::GetFormatWithAlphaChannel(params.format()) : PixelFormat::GetFormatWithoutAlphaChannel(params.format()); VideoParams output_params(params.width(), params.height(), params.time_base(), output_format, params.divider()); int real_iteration_count; if (job.GetIterationCount() > 1 && job.GetIterativeInput()) { real_iteration_count = job.GetIterationCount(); } else { real_iteration_count = 1; } OpenGLTextureCache::ReferencePtr dst_refs[2]; dst_refs[0] = texture_cache_.Get(ctx_, output_params); // If this node requires multiple iterations, get a texture for it too if (real_iteration_count > 1) { dst_refs[1] = texture_cache_.Get(ctx_, output_params); } // Some nodes use multiple iterations for optimization OpenGLTextureCache::ReferencePtr input_tex, output_tex; // Set up OpenGL parameters as necessary functions_->glViewport(0, 0, params.effective_width(), params.effective_height()); // Bind all textures for (int i=0; iglActiveTexture(GL_TEXTURE0 + i); functions_->glBindTexture(GL_TEXTURE_2D, textures_to_bind.at(i)); OpenGLRenderFunctions::PrepareToDraw(functions_); } for (int iteration=0; iterationbind(); shader->setUniformValue("ove_iteration", iteration); shader->release(); // Replace iterative input if (iteration == 0) { output_tex = dst_refs[0]; } else { input_tex = dst_refs[(iteration+1)%2]; output_tex = dst_refs[iteration%2]; functions_->glActiveTexture(GL_TEXTURE0 + iterative_input); functions_->glBindTexture(GL_TEXTURE_2D, input_tex->texture()->texture()); OpenGLRenderFunctions::PrepareToDraw(functions_); } buffer_.Attach(output_tex->texture(), true); buffer_.Bind(); // Blit this texture through this shader OpenGLRenderFunctions::Blit(shader); buffer_.Release(); buffer_.Detach(); } // Release any textures we bound before for (int i=textures_to_bind.size()-1; i>=0; i--) { functions_->glActiveTexture(GL_TEXTURE0 + i); functions_->glBindTexture(GL_TEXTURE_2D, 0); } return QVariant::fromValue(output_tex); } void OpenGLProxy::TextureToBuffer(const QVariant& tex_in, FramePtr frame, const QMatrix4x4& matrix) { OpenGLTextureCache::ReferencePtr texture = tex_in.value(); if (!texture) { return; } OpenGLTextureCache::ReferencePtr download_tex; functions_->glViewport(0, 0, frame->width(), frame->height()); if (frame->width() != texture->texture()->width() || frame->height() != texture->texture()->height()) { // Resize the texture if necessary OpenGLTextureCache::ReferencePtr resized = texture_cache_.Get(ctx_, frame->video_params()); buffer_.Attach(resized->texture(), true); buffer_.Bind(); texture->texture()->Bind(); // Blit to this new texture OpenGLRenderFunctions::Blit(copy_pipeline_, false, matrix); texture->texture()->Release(); buffer_.Release(); buffer_.Detach(); download_tex = resized; } else { download_tex = texture; } buffer_.Attach(download_tex->texture()); buffer_.Bind(); functions_->glPixelStorei(GL_PACK_ROW_LENGTH, frame->linesize_pixels()); functions_->glReadPixels(0, 0, frame->width(), frame->height(), OpenGLRenderFunctions::GetPixelFormat(frame->format()), OpenGLRenderFunctions::GetPixelType(frame->format()), frame->data()); functions_->glPixelStorei(GL_PACK_ROW_LENGTH, 0); buffer_.Release(); buffer_.Detach(); } void OpenGLProxy::FinishInit() { // Make context current on that surface if (!ctx_->makeCurrent(&surface_)) { qWarning() << "Failed to makeCurrent() on offscreen surface in thread" << thread(); return; } // Store OpenGL functions instance functions_ = ctx_->functions(); functions_->glBlendFunc(GL_ONE, GL_ZERO); buffer_.Create(ctx_); copy_pipeline_ = OpenGLShader::CreateDefault(); } OLIVE_NAMESPACE_EXIT