/*** Olive - Non-Linear Video Editor Copyright (C) 2020 Olive Team This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ***/ #include "renderprocessor.h" #include #include #include #include #include "project/project.h" #include "rendermanager.h" namespace olive { RenderProcessor::RenderProcessor(RenderTicketPtr ticket, Renderer *render_ctx, StillImageCache* still_image_cache, DecoderCache* decoder_cache, ShaderCache *shader_cache, QVariant default_shader) : ticket_(ticket), render_ctx_(render_ctx), still_image_cache_(still_image_cache), decoder_cache_(decoder_cache), shader_cache_(shader_cache), default_shader_(default_shader) { } void RenderProcessor::Run() { // Depending on the render ticket type, start a job RenderManager::TicketType type = ticket_->property("type").value(); ticket_->Start(); if (ticket_->WasCancelled()) { return; } switch (type) { case RenderManager::kTypeVideo: { ViewerOutput* viewer = Node::ValueToPtr(ticket_->property("viewer")); const VideoParams& video_params = ticket_->property("vparam").value(); rational time = ticket_->property("time").value(); NodeValueTable table = ProcessInput(viewer->texture_input(), TimeRange(time, time + video_params.time_base())); TexturePtr texture = table.Get(NodeValue::kTexture).value(); // Set up output frame parameters VideoParams frame_params = ticket_->property("vparam").value(); QSize frame_size = ticket_->property("size").value(); if (!frame_size.isNull()) { frame_params.set_width(frame_size.width()); frame_params.set_height(frame_size.height()); } VideoParams::Format frame_format = static_cast(ticket_->property("format").toInt()); if (frame_format != VideoParams::kFormatInvalid) { frame_params.set_format(frame_format); } if (RenderManager::instance()->backend() == RenderManager::kOpenGL && QOpenGLContext::openGLModuleType() == QOpenGLContext::LibGLES) { // HACK: From what I can tell, ANGLE only supports texture reading to RGBA frame_params.set_channel_count(VideoParams::kRGBAChannelCount); } else if (texture) { frame_params.set_channel_count(texture->channel_count()); } FramePtr frame = Frame::Create(); frame->set_timestamp(time); frame->set_video_params(frame_params); frame->allocate(); if (!texture) { // Blank frame out memset(frame->data(), 0, frame->allocated_size()); } else { // Dump texture contents to frame ColorProcessorPtr output_color_transform = ticket_->property("coloroutput").value(); const VideoParams& tex_params = texture->params(); if (tex_params.effective_width() != frame_params.effective_width() || tex_params.effective_height() != frame_params.effective_height() || tex_params.format() != frame_params.format() || output_color_transform) { TexturePtr blit_tex = render_ctx_->CreateTexture(frame_params); QMatrix4x4 matrix = ticket_->property("matrix").value(); if (output_color_transform) { // Yes color transform, blit color managed render_ctx_->BlitColorManaged(output_color_transform, texture, true, blit_tex.get(), true, matrix); } else { // No color transform, just blit ShaderJob job; job.InsertValue(QStringLiteral("ove_maintex"), NodeValue(NodeValue::kTexture, QVariant::fromValue(texture))); job.InsertValue(QStringLiteral("ove_mvpmat"), NodeValue(NodeValue::kMatrix, matrix)); render_ctx_->BlitToTexture(default_shader_, job, blit_tex.get()); } // Replace texture that we're going to download in the next step texture = blit_tex; } render_ctx_->DownloadFromTexture(texture.get(), frame->data(), frame->linesize_pixels()); } ticket_->Finish(QVariant::fromValue(frame), IsCancelled()); break; } case RenderManager::kTypeAudio: { ViewerOutput* viewer = Node::ValueToPtr(ticket_->property("viewer")); TimeRange time = ticket_->property("time").value(); NodeValueTable table = ProcessInput(viewer->samples_input(), time); ticket_->Finish(table.Get(NodeValue::kSamples), IsCancelled()); break; } case RenderManager::kTypeVideoDownload: { FrameHashCache* cache = Node::ValueToPtr(ticket_->property("cache")); FramePtr frame = ticket_->property("frame").value(); QByteArray hash = ticket_->property("hash").toByteArray(); ticket_->Finish(cache->SaveCacheFrame(hash, frame), false); break; } default: // Fail ticket_->Cancel(); } } DecoderPtr RenderProcessor::ResolveDecoderFromInput(Stream *stream) { if (!stream) { qWarning() << "Attempted to resolve the decoder of a null stream"; return nullptr; } QMutexLocker locker(decoder_cache_->mutex()); DecoderPtr decoder = decoder_cache_->value(stream); if (!decoder) { // No decoder decoder = Decoder::CreateFromID(stream->footage()->decoder()); if (decoder->Open(stream)) { decoder_cache_->insert(stream, decoder); } else { qWarning() << "Failed to open decoder for" << stream->footage()->filename() << "::" << stream->index(); return nullptr; } } return decoder; } void RenderProcessor::Process(RenderTicketPtr ticket, Renderer *render_ctx, StillImageCache *still_image_cache, DecoderCache *decoder_cache, ShaderCache *shader_cache, QVariant default_shader) { RenderProcessor p(ticket, render_ctx, still_image_cache, decoder_cache, shader_cache, default_shader); p.Run(); } NodeValueTable RenderProcessor::GenerateBlockTable(const TrackOutput *track, const TimeRange &range) { if (track->track_type() == Timeline::kTrackTypeAudio) { const AudioParams& audio_params = ticket_->property("aparam").value(); QList active_blocks = track->BlocksAtTimeRange(range); // All these blocks will need to output to a buffer so we create one here SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params, audio_params.time_to_samples(range.length())); block_range_buffer->fill(0); NodeValueTable merged_table; // Loop through active blocks retrieving their audio foreach (Block* b, active_blocks) { TimeRange range_for_block(qMax(b->in(), range.in()), qMin(b->out(), range.out())); int destination_offset = audio_params.time_to_samples(range_for_block.in() - range.in()); int max_dest_sz = audio_params.time_to_samples(range_for_block.length()); // Destination buffer NodeValueTable table = GenerateTable(b, range_for_block); SampleBufferPtr samples_from_this_block = table.Take(NodeValue::kSamples).value(); if (!samples_from_this_block) { // If we retrieved no samples from this block, do nothing continue; } // FIXME: Doesn't handle reversing if (b->speed_input()->IsKeyframing() || b->speed_input()->IsConnected()) { // FIXME: We'll need to calculate the speed hoo boy } else { double speed_value = b->speed_input()->GetStandardValue().toDouble(); if (qIsNull(speed_value)) { // Just silence, don't think there's any other practical application of 0 speed audio samples_from_this_block->fill(0); } else if (!qFuzzyCompare(speed_value, 1.0)) { // Multiply time samples_from_this_block->speed(speed_value); } } int copy_length = qMin(max_dest_sz, samples_from_this_block->sample_count()); // Copy samples into destination buffer block_range_buffer->set(samples_from_this_block->const_data(), destination_offset, copy_length); NodeValueTable::Merge({merged_table, table}); } if (ticket_->property("enablewaveforms").toBool()) { // Generate a visual waveform and send it back to the main thread AudioVisualWaveform visual_waveform; visual_waveform.set_channel_count(audio_params.channel_count()); visual_waveform.OverwriteSamples(block_range_buffer, audio_params.sample_rate()); RenderedWaveform waveform_info = {track, visual_waveform, range}; QVector waveform_list = ticket_->property("waveforms").value< QVector >(); waveform_list.append(waveform_info); ticket_->setProperty("waveforms", QVariant::fromValue(waveform_list)); } merged_table.Push(NodeValue::kSamples, QVariant::fromValue(block_range_buffer), track); return merged_table; } else { return NodeTraverser::GenerateBlockTable(track, range); } } QVariant RenderProcessor::ProcessVideoFootage(VideoStream *video_stream, const rational &input_time) { TexturePtr value = nullptr; // Check the still frame cache. On large frames such as high resolution still images, uploading // and color managing them for every frame is a waste of time, so we implement a small cache here // to optimize such a situation const VideoParams& video_params = ticket_->property("vparam").value(); ColorManager* color_manager = Node::ValueToPtr(ticket_->property("colormanager")); // See if we can make this divider larger (i.e. if the fooage is smaller) int footage_divider = video_params.divider(); while (footage_divider > 1 && VideoParams::GetScaledDimension(video_stream->width(), footage_divider-1) < video_params.effective_width() && VideoParams::GetScaledDimension(video_stream->height(), footage_divider-1) < video_params.effective_height()) { footage_divider--; } StillImageCache::EntryPtr want_entry = std::make_shared( nullptr, video_stream, ColorProcessor::GenerateID(color_manager, video_stream->colorspace(), color_manager->GetReferenceColorSpace()), video_stream->premultiplied_alpha(), footage_divider, (video_stream->video_type() == VideoStream::kVideoTypeStill) ? 0 : input_time, true); bool found_existing = false; still_image_cache_->mutex()->lock(); foreach (StillImageCache::EntryPtr e, still_image_cache_->entries()) { if (StillImageCache::CompareEntryMetadata(want_entry, e)) { // Found an exact match of the texture we want in the cache. See if it's working or if it's // ready. want_entry = e; found_existing = true; while (want_entry->working) { still_image_cache_->wait_cond()->wait(still_image_cache_->mutex()); } value = want_entry->texture; break; } } if (value) { // Found the texture, we can release the cache now still_image_cache_->mutex()->unlock(); } else { // Wasn't in still image cache, so we'll have to retrieve it from the decoder // Let other processors know we're getting this texture (want_entry's `working` field is // already set to true in the initializer above) if (!found_existing) { still_image_cache_->PushEntry(want_entry); } still_image_cache_->mutex()->unlock(); DecoderPtr decoder = ResolveDecoderFromInput(video_stream); if (decoder) { FramePtr frame = decoder->RetrieveVideo(input_time, footage_divider); if (frame) { // Return a texture from the derived class TexturePtr unmanaged_texture = render_ctx_->CreateTexture(frame->video_params(), frame->data(), frame->linesize_pixels()); // We convert to our rendering pixel format, since that will always be float-based which // is necessary for correct color conversion VideoParams managed_params = frame->video_params(); managed_params.set_format(video_params.format()); value = render_ctx_->CreateTexture(managed_params); ColorProcessorPtr processor = ColorProcessor::Create(color_manager, video_stream->colorspace(), color_manager->GetReferenceColorSpace()); render_ctx_->BlitColorManaged(processor, unmanaged_texture, video_stream->premultiplied_alpha(), value.get()); still_image_cache_->mutex()->lock(); // Put this into the image cache instead want_entry->texture = value; want_entry->working = false; still_image_cache_->wait_cond()->wakeAll(); still_image_cache_->mutex()->unlock(); } } } return QVariant::fromValue(value); } QVariant RenderProcessor::ProcessAudioFootage(AudioStream *stream, const TimeRange &input_time) { QVariant value; DecoderPtr decoder = ResolveDecoderFromInput(stream); if (decoder) { const AudioParams& audio_params = ticket_->property("aparam").value(); SampleBufferPtr frame = decoder->RetrieveAudio(input_time, audio_params, &IsCancelled()); if (frame) { value = QVariant::fromValue(frame); } } return value; } QVariant RenderProcessor::ProcessShader(const Node *node, const TimeRange &range, const ShaderJob &job) { Q_UNUSED(range) QString full_shader_id = QStringLiteral("%1:%2").arg(node->id(), job.GetShaderID()); QMutexLocker locker(shader_cache_->mutex()); QVariant shader = shader_cache_->value(full_shader_id); if (shader.isNull()) { // Since we have shader code, compile it now shader = render_ctx_->CreateNativeShader(node->GetShaderCode(job.GetShaderID())); if (shader.isNull()) { // Couldn't find or build the shader required return QVariant(); } } VideoParams tex_params = ticket_->property("vparam").value(); bool input_textures_have_alpha = false; for (auto it=job.GetValues().cbegin(); it!=job.GetValues().cend(); it++) { if (it.value().type() == NodeValue::kTexture) { TexturePtr tex = it.value().data().value(); if (tex && tex->channel_count() == VideoParams::kRGBAChannelCount) { input_textures_have_alpha = true; break; } } } if (input_textures_have_alpha || job.GetAlphaChannelRequired()) { tex_params.set_channel_count(VideoParams::kRGBAChannelCount); } else { tex_params.set_channel_count(VideoParams::kRGBChannelCount); } TexturePtr destination = render_ctx_->CreateTexture(tex_params); // Run shader render_ctx_->BlitToTexture(shader, job, destination.get()); return QVariant::fromValue(destination); } QVariant RenderProcessor::ProcessSamples(const Node *node, const TimeRange &range, const SampleJob &job) { if (!job.samples() || !job.samples()->is_allocated()) { return QVariant(); } SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(job.samples()->audio_params(), job.samples()->sample_count()); NodeValueDatabase value_db; const AudioParams& audio_params = ticket_->property("aparam").value(); for (int i=0;isample_count();i++) { // Calculate the exact rational time at this sample double sample_to_second = static_cast(i) / static_cast(audio_params.sample_rate()); rational this_sample_time = rational::fromDouble(range.in().toDouble() + sample_to_second); // Update all non-sample and non-footage inputs NodeValueMap::const_iterator j; for (j=job.GetValues().constBegin(); j!=job.GetValues().constEnd(); j++) { NodeValueTable value; NodeInput* corresponding_input = node->GetInputWithID(j.key()); if (corresponding_input) { value = ProcessInput(corresponding_input, TimeRange(this_sample_time, this_sample_time)); } else { value.Push(j.value()); } value_db.Insert(j.key(), value); } AddGlobalsToDatabase(value_db, TimeRange(this_sample_time, this_sample_time)); node->ProcessSamples(value_db, job.samples(), output_buffer, i); } return QVariant::fromValue(output_buffer); } QVariant RenderProcessor::ProcessFrameGeneration(const Node *node, const GenerateJob &job) { FramePtr frame = Frame::Create(); VideoParams frame_params = ticket_->property("vparam").value(); if (job.GetAlphaChannelRequired()) { frame_params.set_channel_count(VideoParams::kRGBAChannelCount); } else { frame_params.set_channel_count(VideoParams::kRGBChannelCount); } frame->set_video_params(frame_params); frame->allocate(); node->GenerateFrame(frame, job); TexturePtr texture = render_ctx_->CreateTexture(frame->video_params(), frame->data(), frame->linesize_pixels()); return QVariant::fromValue(texture); } QVariant RenderProcessor::GetCachedFrame(const Node *node, const rational &time) { if (!ticket_->property("cache").toString().isEmpty() && node->id() == QStringLiteral("org.olivevideoeditor.Olive.videoinput")) { const VideoParams& video_params = ticket_->property("vparam").value(); QByteArray hash = RenderManager::Hash(node, video_params, time); FramePtr f = FrameHashCache::LoadCacheFrame(ticket_->property("cache").toString(), hash); if (f) { // The cached frame won't load with the correct divider by default, so we enforce it here VideoParams p = f->video_params(); p.set_width(f->width() * video_params.divider()); p.set_height(f->height() * video_params.divider()); p.set_divider(video_params.divider()); f->set_video_params(p); TexturePtr texture = render_ctx_->CreateTexture(f->video_params(), f->data(), f->linesize_pixels()); return QVariant::fromValue(texture); } } return QVariant(); } QVector2D RenderProcessor::GenerateResolution() const { // Set resolution to the destination to the "logical" resolution of the destination const VideoParams& video_params = ticket_->property("vparam").value(); return QVector2D(video_params.width() * video_params.pixel_aspect_ratio().toDouble(), video_params.height()); } }