/*** Olive - Non-Linear Video Editor Copyright (C) 2019 Olive Team This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ***/ #include "openglproxy.h" #include #include "common/clamp.h" #include "core.h" #include "node/block/transition/transition.h" #include "node/node.h" #include "openglcolorprocessor.h" #include "openglrenderfunctions.h" #include "render/colormanager.h" #include "render/pixelformat.h" OLIVE_NAMESPACE_ENTER OpenGLProxy::OpenGLProxy(QObject *parent) : QObject(parent), ctx_(nullptr), functions_(nullptr) { surface_.create(); } OpenGLProxy::~OpenGLProxy() { Close(); surface_.destroy(); } bool OpenGLProxy::Init() { // Create context object ctx_ = new QOpenGLContext(); // Create OpenGL context (automatically destroys any existing if there is one) if (!ctx_->create()) { qWarning() << "Failed to create OpenGL context in thread" << thread(); return false; } ctx_->moveToThread(this->thread()); // The rest of the initialization needs to occur in the other thread, so we signal for it to start QMetaObject::invokeMethod(this, "FinishInit", Qt::QueuedConnection); return true; } void OpenGLProxy::FrameToValue(FramePtr frame, StreamPtr stream, NodeValueTable* table) { // Ensure stream is video or image type if (stream->type() != Stream::kVideo && stream->type() != Stream::kImage) { return; } ImageStreamPtr video_stream = std::static_pointer_cast(stream); // Set up OCIO context QString colorspace_match = QStringLiteral("%1:%2").arg(video_stream->footage()->project()->color_manager()->GetConfigFilename(), video_stream->colorspace()); OpenGLTextureCache::ReferencePtr footage_tex_ref = nullptr; if (stream->type() == Stream::kImage && still_image_cache_.Has(stream.get())) { CachedStill cs = still_image_cache_.Get(stream.get()); if (cs.colorspace == colorspace_match && cs.alpha_is_associated == video_stream->premultiplied_alpha() && cs.divider == video_params_.divider()) { footage_tex_ref = cs.texture; } else { still_image_cache_.Remove(stream.get()); } } if (!footage_tex_ref) { OpenGLColorProcessorPtr color_processor = std::static_pointer_cast(color_cache_.Get(colorspace_match)); if (!color_processor) { color_processor = OpenGLColorProcessor::Create(video_stream->footage()->project()->color_manager(), video_stream->colorspace(), video_stream->footage()->project()->color_manager()->GetReferenceColorSpace()); color_cache_.Add(colorspace_match, color_processor); } ColorManager::OCIOMethod ocio_method = ColorManager::GetOCIOMethodForMode(video_params_.mode()); // OCIO's CPU conversion is more accurate, so for online we render on CPU but offline we render GPU if (ocio_method == ColorManager::kOCIOAccurate) { bool has_alpha = PixelFormat::FormatHasAlphaChannel(frame->format()); // Convert frame to float for OCIO frame = PixelFormat::ConvertPixelFormat(frame, has_alpha ? PixelFormat::PIX_FMT_RGBA32F : PixelFormat::PIX_FMT_RGB32F); // If alpha is associated, disassociate for the color transform if (has_alpha && video_stream->premultiplied_alpha()) { ColorManager::DisassociateAlpha(frame); } // Perform color transform color_processor->ConvertFrame(frame); // Associate alpha if (has_alpha) { if (video_stream->premultiplied_alpha()) { ColorManager::ReassociateAlpha(frame); } else { ColorManager::AssociateAlpha(frame); } } } footage_tex_ref = texture_cache_.Get(ctx_, frame); if (ocio_method == ColorManager::kOCIOFast) { if (!color_processor->IsEnabled()) { color_processor->Enable(ctx_, video_stream->premultiplied_alpha()); } VideoRenderingParams frame_params = frame->video_params(); // Check frame aspect ratio if (frame->sample_aspect_ratio() != 1 && frame->sample_aspect_ratio() != 0) { int new_width = frame_params.width(); int new_height = frame_params.height(); // Scale the frame in a way that does not reduce the resolution if (frame->sample_aspect_ratio() > 1) { // Make wider new_width = qRound(static_cast(new_width) * frame->sample_aspect_ratio().toDouble()); } else { // Make taller new_height = qRound(static_cast(new_height) / frame->sample_aspect_ratio().toDouble()); } frame_params = VideoRenderingParams(new_width, new_height, frame_params.format(), frame_params.divider()); } VideoRenderingParams dest_params(frame_params.width(), frame_params.height(), video_params_.format(), frame_params.divider()); // Create destination texture OpenGLTextureCache::ReferencePtr associated_tex_ref = texture_cache_.Get(ctx_, dest_params); buffer_.Attach(associated_tex_ref->texture(), true); buffer_.Bind(); footage_tex_ref->texture()->Bind(); // Set viewport for texture size functions_->glViewport(0, 0, associated_tex_ref->texture()->width(), associated_tex_ref->texture()->height()); // Blit old texture to new texture through OCIO shader color_processor->ProcessOpenGL(); footage_tex_ref->texture()->Release(); buffer_.Release(); buffer_.Detach(); footage_tex_ref = associated_tex_ref; } if (stream->type() == Stream::kImage) { // Since this is a still image, we could likely optimize this still_image_cache_.Add(stream.get(), {footage_tex_ref, colorspace_match, video_stream->premultiplied_alpha(), video_params_.divider()}); } } table->Push(NodeParam::kTexture, QVariant::fromValue(footage_tex_ref)); } void OpenGLProxy::Close() { shader_cache_.Clear(); buffer_.Destroy(); copy_pipeline_ = nullptr; functions_ = nullptr; delete ctx_; ctx_ = nullptr; } void OpenGLProxy::RunNodeAccelerated(const Node *node, const TimeRange &range, NodeValueDatabase &input_params, NodeValueTable &output_params) { if (!(node->GetCapabilities(input_params) & Node::kShader)) { return; } OpenGLShaderPtr shader = shader_cache_.Get(node->ShaderID(input_params)); if (!shader) { // Since we have shader code, compile it now QString vert_code = node->ShaderVertexCode(input_params); QString frag_code = node->ShaderFragmentCode(input_params); if (frag_code.isEmpty()) { frag_code = OpenGLShader::CodeDefaultFragment(); } if (vert_code.isEmpty()) { vert_code = OpenGLShader::CodeDefaultVertex(); } shader = OpenGLShader::Create(); shader->create(); shader->addShaderFromSourceCode(QOpenGLShader::Fragment, frag_code); shader->addShaderFromSourceCode(QOpenGLShader::Vertex, vert_code); shader->link(); shader_cache_.Add(node->id(), shader); } // Create the output textures QList dst_refs; dst_refs.append(texture_cache_.Get(ctx_, video_params_)); GLuint iterative_input = 0; // If this node requires multiple iterations, get a texture for it too if (node->ShaderIterations() > 1 && node->ShaderIterativeInput()) { dst_refs.append(texture_cache_.Get(ctx_, video_params_)); } // Lock the shader so no other thread interferes as we set parameters and draw (and we don't interfere with any others) shader->bind(); unsigned int input_texture_count = 0; foreach (NodeParam* param, node->parameters()) { if (param->type() == NodeParam::kInput) { // See if the shader has takes this parameter as an input int variable_location = shader->uniformLocation(param->id()); if (variable_location > -1) { // This variable is used in the shader, let's set it to our value NodeInput* input = static_cast(param); // Get value from database at this input NodeValue meta_value = node->InputValueFromTable(input, input_params, false); const QVariant& value = meta_value.data(); NodeParam::DataType data_type; if (meta_value.type() != NodeParam::kNone) { // Use value's data type data_type = meta_value.type(); } else { // Fallback on null value, send the null to the parameter data_type = input->data_type(); } switch (data_type) { case NodeInput::kInt: shader->setUniformValue(variable_location, value.toInt()); break; case NodeInput::kFloat: shader->setUniformValue(variable_location, value.toFloat()); break; case NodeInput::kVec2: if (input->IsArray()) { NodeInputArray* array = static_cast(input); QVector a(array->GetSize()); for (int i=0;iAt(i)].Get(NodeParam::kVec2).value(); } shader->setUniformValueArray(variable_location, a.constData(), a.size()); int count_location = shader->uniformLocation(QStringLiteral("%1_count").arg(input->id())); if (count_location > -1) { shader->setUniformValue(count_location, array->GetSize()); } } else { shader->setUniformValue(variable_location, value.value()); } break; case NodeInput::kVec3: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kVec4: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kMatrix: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kCombo: shader->setUniformValue(variable_location, value.value()); break; case NodeInput::kColor: { Color color = value.value(); shader->setUniformValue(variable_location, color.red(), color.green(), color.blue(), color.alpha()); break; } case NodeInput::kBoolean: shader->setUniformValue(variable_location, value.toBool()); break; case NodeInput::kFootage: case NodeInput::kTexture: case NodeInput::kBuffer: { OpenGLTextureCache::ReferencePtr texture = value.value(); functions_->glActiveTexture(GL_TEXTURE0 + input_texture_count); GLuint tex_id = texture ? texture->texture()->texture() : 0; functions_->glBindTexture(GL_TEXTURE_2D, tex_id); // Set value to bound texture shader->setUniformValue(variable_location, input_texture_count); // Set enable flag if shader wants it int enable_param_location = shader->uniformLocation(QStringLiteral("%1_enabled").arg(input->id())); if (enable_param_location > -1) { shader->setUniformValue(enable_param_location, tex_id > 0); } if (tex_id > 0) { // Set texture resolution if shader wants it int res_param_location = shader->uniformLocation(QStringLiteral("%1_resolution").arg(input->id())); if (res_param_location > -1) { shader->setUniformValue(res_param_location, static_cast(texture->texture()->width() * texture->texture()->divider()), static_cast(texture->texture()->height() * texture->texture()->divider())); } } // If this texture binding is the iterative input, set it here if (input == node->ShaderIterativeInput()) { iterative_input = input_texture_count; } OpenGLRenderFunctions::PrepareToDraw(functions_); input_texture_count++; break; } case NodeInput::kSamples: case NodeInput::kText: case NodeInput::kRational: case NodeInput::kFont: case NodeInput::kFile: case NodeInput::kDecimal: case NodeInput::kNumber: case NodeInput::kString: case NodeInput::kVector: case NodeInput::kNone: case NodeInput::kAny: break; } } } } // Set up OpenGL parameters as necessary functions_->glViewport(0, 0, video_params_.effective_width(), video_params_.effective_height()); // Provide some standard args shader->setUniformValue("ove_resolution", static_cast(video_params_.width()), static_cast(video_params_.height())); if (node->IsBlock() && static_cast(node)->type() == Block::kTransition) { const TransitionBlock* transition_node = static_cast(node); // Provides total transition progress from 0.0 (start) - 1.0 (end) shader->setUniformValue("ove_tprog_all", static_cast(transition_node->GetTotalProgress(range.in()))); // Provides progress of out section from 1.0 (start) - 0.0 (end) shader->setUniformValue("ove_tprog_out", static_cast(transition_node->GetOutProgress(range.in()))); // Provides progress of in section from 0.0 (start) - 1.0 (end) shader->setUniformValue("ove_tprog_in", static_cast(transition_node->GetInProgress(range.in()))); } // Some nodes use multiple iterations for optimization OpenGLTextureCache::ReferencePtr output_tex; for (int iteration=0;iterationShaderIterations();iteration++) { // If this is not the first iteration, set the parameter that will receive the last iteration's texture OpenGLTextureCache::ReferencePtr source_tex = dst_refs.at((iteration+1)%dst_refs.size()); OpenGLTextureCache::ReferencePtr destination_tex = dst_refs.at(iteration%dst_refs.size()); // Set iteration number shader->bind(); shader->setUniformValue("ove_iteration", iteration); shader->release(); if (iteration > 0) { functions_->glActiveTexture(GL_TEXTURE0 + iterative_input); functions_->glBindTexture(GL_TEXTURE_2D, source_tex->texture()->texture()); } buffer_.Attach(destination_tex->texture(), true); buffer_.Bind(); // Blit this texture through this shader OpenGLRenderFunctions::Blit(shader); buffer_.Release(); buffer_.Detach(); // Update output reference to the last texture we wrote to output_tex = destination_tex; } // Release any textures we bound before while (input_texture_count > 0) { input_texture_count--; // Release texture here functions_->glActiveTexture(GL_TEXTURE0 + input_texture_count); functions_->glBindTexture(GL_TEXTURE_2D, 0); } shader->release(); output_params.Push(NodeParam::kTexture, QVariant::fromValue(output_tex)); } void OpenGLProxy::TextureToBuffer(const QVariant &tex_in, int width, int height, const QMatrix4x4 &matrix, void *buffer, int linesize) { OpenGLTextureCache::ReferencePtr texture = tex_in.value(); if (!texture) { return; } QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions(); OpenGLTextureCache::ReferencePtr download_tex; if (width != texture->texture()->width() || height != texture->texture()->height()) { // Resize the texture if necessary OpenGLTextureCache::ReferencePtr resized = texture_cache_.Get(ctx_, VideoRenderingParams(width, height, texture->texture()->format())); buffer_.Attach(resized->texture(), true); buffer_.Bind(); texture->texture()->Bind(); f->glViewport(0, 0, width, height); // Blit to this new texture OpenGLRenderFunctions::Blit(copy_pipeline_, false, matrix); texture->texture()->Release(); buffer_.Release(); buffer_.Detach(); download_tex = resized; } else { download_tex = texture; } buffer_.Attach(download_tex->texture()); buffer_.Bind(); f->glPixelStorei(GL_PACK_ROW_LENGTH, linesize); f->glReadPixels(0, 0, width, height, OpenGLRenderFunctions::GetPixelFormat(video_params_.format()), OpenGLRenderFunctions::GetPixelType(video_params_.format()), buffer); f->glPixelStorei(GL_PACK_ROW_LENGTH, 0); buffer_.Release(); buffer_.Detach(); } void OpenGLProxy::SetParameters(const VideoRenderingParams ¶ms) { video_params_ = params; if (functions_ != nullptr && video_params_.is_valid()) { functions_->glViewport(0, 0, video_params_.effective_width(), video_params_.effective_height()); } } void OpenGLProxy::FinishInit() { // Make context current on that surface if (!ctx_->makeCurrent(&surface_)) { qWarning() << "Failed to makeCurrent() on offscreen surface in thread" << thread(); return; } // Store OpenGL functions instance functions_ = ctx_->functions(); functions_->glBlendFunc(GL_ONE, GL_ZERO); SetParameters(video_params_); buffer_.Create(ctx_); copy_pipeline_ = OpenGLShader::CreateDefault(); } OLIVE_NAMESPACE_EXIT