oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

forked MediaInput into two derived classes for video and audio

Browse Source 
Rather than convoluting the MediaInput node, te MediaInput is now an abstract
base class that provides access to a Decoder and derived classes are
responsible for handling it
This commit is contained in:
itsmattkc
2019-10-24 17:03:33 +11:00
parent 2fdf78df4a
commit af702835da
12 changed files with 401 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/decoder/ffmpeg/ffmpegdecoder.cpp
+7 -9
View File
@@ -41,8 +41,7 @@ FFmpegDecoder::FFmpegDecoder() :
opts_(nullptr),
frame_(nullptr),
pkt_(nullptr),
scale_ctx_(nullptr),
resample_ctx_(nullptr)
scale_ctx_(nullptr)
{
}
@@ -283,11 +282,6 @@ void FFmpegDecoder::Close()
scale_ctx_ = nullptr;
}
if (resample_ctx_ != nullptr) {
swr_free(&resample_ctx_);
resample_ctx_ = nullptr;
}
if (pkt_ != nullptr) {
av_packet_free(&pkt_);
pkt_ = nullptr;
@@ -574,14 +568,18 @@ void FFmpegDecoder::Index()
if (resampler != nullptr) {
// We must need to resample this (mainly just convert from planar to packed if necessary)
resampler_output = new uint8_t[buffer_size];
swr_convert(resampler, &resampler_output, frame_->nb_samples, frame_->data, frame_->nb_samples);
swr_convert(resampler,
&resampler_output,
frame_->nb_samples,
const_cast<const uint8_t **>(frame_->data),
frame_->nb_samples);
} else {
// No resampling required, we can write directly from te frame buffer
resampler_output = frame_->data[0];
}
// Write packed WAV data to the disk cache
wave_out.write(resampler_output, buffer_size);
wave_out.write(reinterpret_cast<char*>(resampler_output), buffer_size);
// If we allocated an output for the resampler, delete it here
if (resampler_output != frame_->data[0]) {
app/decoder/ffmpeg/ffmpegdecoder.h
-1
View File
@@ -141,7 +141,6 @@ private:
AVPacket* pkt_;
SwsContext* scale_ctx_;
SwrContext* resample_ctx_;
int output_fmt_;
QVector<int64_t> frame_index_;
app/node/input/media/CMakeLists.txt
+3
View File
@@ -14,6 +14,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
add_subdirectory(audio)
add_subdirectory(video)
set(OLIVE_SOURCES
${OLIVE_SOURCES}
node/input/media/media.h
app/node/input/media/audio/CMakeLists.txt
+22
View File
@@ -0,0 +1,22 @@
# Olive - Non-Linear Video Editor
# Copyright (C) 2019 Olive Team
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set(OLIVE_SOURCES
${OLIVE_SOURCES}
node/input/media/audio/audio.h
node/input/media/audio/audio.cpp
PARENT_SCOPE
)
app/node/input/media/audio/audio.cpp
+26
View File
@@ -0,0 +1,26 @@
#include "audio.h"
AudioInput::AudioInput()
{
}
QString AudioInput::Name()
{
return tr("Audio Input");
}
QString AudioInput::id()
{
return "org.olivevideoeditor.Olive.audioinput";
}
QString AudioInput::Category()
{
return tr("Input");
}
QString AudioInput::Description()
{
return tr("Import an audio footage stream.");
}
app/node/input/media/audio/audio.h
+17
View File
@@ -0,0 +1,17 @@
#ifndef AUDIOINPUT_H
#define AUDIOINPUT_H
#include "../media.h"
class AudioInput : public MediaInput
{
public:
AudioInput();
virtual QString Name() override;
virtual QString id() override;
virtual QString Category() override;
virtual QString Description() override;
};
#endif // AUDIOINPUT_H
app/node/input/media/media.cpp
-242
View File
@@ -20,80 +20,19 @@
#include "media.h"
#include <QDebug>
#include <QOpenGLPixelTransferOptions>
#include "core.h"
#include "decoder/ffmpeg/ffmpegdecoder.h"
#include "project/item/footage/footage.h"
#include "render/gl/shadergenerators.h"
#include "render/gl/functions.h"
#include "render/pixelservice.h"
#include "render/video/videorenderer.h"
MediaInput::MediaInput() :
decoder_(nullptr),
color_processor_(nullptr),
pipeline_(nullptr),
ocio_texture_(0),
frame_(nullptr)
{
footage_input_ = new NodeInput("footage_in");
footage_input_->add_data_input(NodeInput::kFootage);
AddParameter(footage_input_);
matrix_input_ = new NodeInput("matrix_in");
matrix_input_->add_data_input(NodeInput::kMatrix);
AddParameter(matrix_input_);
texture_output_ = new NodeOutput("tex_out");
texture_output_->set_data_type(NodeOutput::kTexture);
texture_output_->SetValueCachingEnabled(false);
AddParameter(texture_output_);
}
QString MediaInput::Name()
{
return tr("Media");
}
QString MediaInput::id()
{
return "org.olivevideoeditor.Olive.mediainput";
}
QString MediaInput::Category()
{
return tr("Input");
}
QString MediaInput::Description()
{
return tr("Import a footage stream.");
}
void MediaInput::Release()
{
internal_tex_.Destroy();
frame_ = nullptr;
decoder_ = nullptr;
color_processor_ = nullptr;
pipeline_ = nullptr;
if (ocio_texture_ != 0) {
ocio_ctx_->functions()->glDeleteTextures(1, &ocio_texture_);
}
}
NodeInput *MediaInput::matrix_input()
{
return matrix_input_;
}
NodeOutput *MediaInput::texture_output()
{
return texture_output_;
}
StreamPtr MediaInput::Footage()
@@ -106,187 +45,6 @@ void MediaInput::SetFootage(StreamPtr f)
footage_input_->set_value(QVariant::fromValue(f));
}
void MediaInput::Hash(QCryptographicHash *hash, NodeOutput *from, const rational &time)
{
Node::Hash(hash, from, time);
// Use frame value from Decoder
if (from == texture_output_) {
if (!SetupDecoder()) {
qDebug() << "Failed to setup decoder for hashing";
return;
}
int64_t timestamp = decoder_->GetTimestampFromTime(time);
QByteArray pts_bytes;
pts_bytes.resize(sizeof(int64_t));
memcpy(pts_bytes.data(), &timestamp, sizeof(int64_t));
hash->addData(pts_bytes);
// FIXME: Add OCIO data
// FIXME: Add alpha association value
}
}
QVariant MediaInput::Value(NodeOutput *output, const rational &in, const rational &out)
{
Q_UNUSED(out)
// FIXME: Hardcoded value
bool alpha_is_associated = false;
if (output == texture_output_) {
// Find the current Renderer instance
RenderInstance* renderer = VideoRendererProcessor::CurrentInstance();
// If nothing is available, don't return a texture
if (renderer == nullptr) {
return 0;
}
// Make sure decoder is set up
if (!SetupDecoder()) {
return 0;
}
// Check if we need to get a frame or not
if (frame_ == nullptr || frame_->native_timestamp() != decoder_->GetTimestampFromTime(in)) {
// Get frame from Decoder
frame_ = decoder_->Retrieve(in);
if (frame_ == nullptr) {
qDebug() << "Received a null frame while time was" << in.toDouble();
return 0;
}
if (color_processor_ == nullptr) {
QString colorspace = std::static_pointer_cast<VideoStream>(Footage())->colorspace();
if (colorspace.isEmpty()) {
// FIXME: Should use Footage() to find the Project* it belongs to instead of this
colorspace = olive::core.GetActiveProject()->default_input_colorspace();
}
color_processor_ = ColorProcessor::Create(colorspace, OCIO::ROLE_SCENE_LINEAR);
}
// OpenColorIO v1's color transforms can be done on GPU, which improves performance but reduces accuracy. When
// online, we prefer accuracy over performance so we use the CPU path instead:
// NOTE: OCIO v2 boasts 1:1 results with the CPU and GPU path so this won't be necessary forever
if (renderer->params().mode() == olive::RenderMode::kOnline) {
// Convert to 32F, which is required for OpenColorIO's color transformation
frame_ = PixelService::ConvertPixelFormat(frame_, olive::PIX_FMT_RGBA32F);
if (alpha_is_associated) {
// Unassociate alpha here if associated
ColorManager::DisassociateAlpha(frame_);
}
// Transform color to reference space
color_processor_->ConvertFrame(frame_);
if (alpha_is_associated) {
// If alpha was associated, reassociate here
ColorManager::ReassociateAlpha(frame_);
} else {
// If alpha was not associated, associate here
ColorManager::AssociateAlpha(frame_);
}
}
// We use an internal texture to bring the texture into GPU space before performing transformations
// Ensure the texture is the accurate to the frame
if (internal_tex_.width() != frame_->width()
|| internal_tex_.height() != frame_->height()
|| internal_tex_.format() != frame_->format()) {
internal_tex_.Destroy();
}
// Create or upload the new data to the texture
if (!internal_tex_.IsCreated()) {
internal_tex_.Create(renderer->context(),
frame_->width(),
frame_->height(),
static_cast<olive::PixelFormat>(frame_->format()),
frame_->data());
} else {
internal_tex_.Upload(frame_->data());
}
}
// Create new texture in reference space to send throughout the rest of the graph
RenderTexturePtr output_texture = std::make_shared<RenderTexture>();
output_texture->Create(renderer->context(),
renderer->params().effective_width(),
renderer->params().effective_height(),
renderer->params().format(),
RenderTexture::kDoubleBuffer);
// Using the transformation matrix, blit our internal texture (in frame format) to our output texture (in
// reference format)
if (renderer->params().mode() == olive::RenderMode::kOffline) {
// For offline rendering, OCIO's GPU path is acceptable:
// NOTE: OCIO v2 boasts 1:1 results with the CPU and GPU path so this won't be necessary forever
// Use an OCIO pipeline shader (which wraps in a default pipeline and will also handle alpha association)
if (pipeline_ == nullptr) {
pipeline_ = olive::ShaderGenerator::OCIOPipeline(renderer->context(),
ocio_texture_, // FIXME: A raw GLuint texture, should wrap this up
color_processor_->GetProcessor(),
alpha_is_associated);
// Used for cleanup later
ocio_ctx_ = renderer->context();
}
} else if (pipeline_ == nullptr) {
// In online, the color transformation was performed on the CPU (see above), so we only need to blit
pipeline_ = olive::ShaderGenerator::DefaultPipeline();
}
renderer->context()->functions()->glBlendFunc(GL_ONE, GL_ZERO);
// Draw onto the output texture using the renderer's framebuffer
renderer->buffer()->Attach(output_texture);
renderer->buffer()->Bind();
// Draw with the internal texture
internal_tex_.Bind();
QMatrix4x4 transform;
// Scale texture to a square for incoming matrix transformation
transform.scale(2.0f / static_cast<float>(renderer->params().width()),
2.0f / static_cast<float>(renderer->params().height()));
// Multiply by input transformation
transform *= matrix_input_->get_value(in).value<QMatrix4x4>();
// Scale texture to the media size
transform.scale(static_cast<float>(frame_->width()), static_cast<float>(frame_->height()));
transform.scale(0.5f, 0.5f);
// Use pipeline to blit using transformation matrix from input
if (renderer->params().mode() == olive::RenderMode::kOffline) {
olive::gl::OCIOBlit(pipeline_, ocio_texture_, false, transform);
} else {
olive::gl::Blit(pipeline_, false, transform);
}
// Release everything
internal_tex_.Release();
renderer->buffer()->Detach();
renderer->buffer()->Release();
return QVariant::fromValue(output_texture);
}
return 0;
}
bool MediaInput::SetupDecoder()
{
if (decoder_ != nullptr) {
app/node/input/media/media.h
+3 -35
View File
@@ -18,16 +18,11 @@
***/
#ifndef IMAGE_H
#define IMAGE_H
#include <QOpenGLTexture>
#ifndef MEDIAINPUT_H
#define MEDIAINPUT_H
#include "decoder/decoder.h"
#include "node/node.h"
#include "render/colormanager.h"
#include "render/rendertexture.h"
#include "render/gl/shadergenerators.h"
/**
* @brief A node that imports an image
@@ -38,47 +33,20 @@ class MediaInput : public Node
public:
MediaInput();
virtual QString Name() override;
virtual QString id() override;
virtual QString Category() override;
virtual QString Description() override;
virtual void Release() override;
NodeInput* matrix_input();
NodeOutput* texture_output();
StreamPtr Footage();
void SetFootage(StreamPtr f);
virtual void Hash(QCryptographicHash *hash, NodeOutput* from, const rational &time) override;
protected:
virtual QVariant Value(NodeOutput* output, const rational& in, const rational& out) override;
private:
bool SetupDecoder();
NodeInput* footage_input_;
NodeInput* matrix_input_;
NodeOutput* texture_output_;
RenderTexture internal_tex_;
DecoderPtr decoder_;
ColorProcessorPtr color_processor_;
ShaderPtr pipeline_;
QOpenGLContext* ocio_ctx_;
GLuint ocio_texture_;
FramePtr frame_;
};
#endif // IMAGE_H
#endif // MEDIAINPUT_H
app/node/input/media/video/CMakeLists.txt
+22
View File
@@ -0,0 +1,22 @@
# Olive - Non-Linear Video Editor
# Copyright (C) 2019 Olive Team
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set(OLIVE_SOURCES
${OLIVE_SOURCES}
node/input/media/video/video.h
node/input/media/video/video.cpp
PARENT_SCOPE
)
app/node/input/media/video/video.cpp
+251
View File
@@ -0,0 +1,251 @@
#include "video.h"
#include <QDebug>
#include <QOpenGLPixelTransferOptions>
#include "core.h"
#include "decoder/ffmpeg/ffmpegdecoder.h"
#include "project/item/footage/footage.h"
#include "render/gl/shadergenerators.h"
#include "render/gl/functions.h"
#include "render/pixelservice.h"
#include "render/video/videorenderer.h"
VideoInput::VideoInput() :
color_processor_(nullptr),
pipeline_(nullptr),
ocio_texture_(0)
{
matrix_input_ = new NodeInput("matrix_in");
matrix_input_->add_data_input(NodeInput::kMatrix);
AddParameter(matrix_input_);
texture_output_ = new NodeOutput("tex_out");
texture_output_->set_data_type(NodeOutput::kTexture);
texture_output_->SetValueCachingEnabled(false);
AddParameter(texture_output_);
}
QString VideoInput::Name()
{
return tr("Video Input");
}
QString VideoInput::id()
{
return "org.olivevideoeditor.Olive.videoinput";
}
QString VideoInput::Category()
{
return tr("Input");
}
QString VideoInput::Description()
{
return tr("Import a video footage stream.");
}
void VideoInput::Release()
{
MediaInput::Release();
internal_tex_.Destroy();
color_processor_ = nullptr;
pipeline_ = nullptr;
if (ocio_texture_ != 0) {
ocio_ctx_->functions()->glDeleteTextures(1, &ocio_texture_);
}
}
NodeInput *VideoInput::matrix_input()
{
return matrix_input_;
}
NodeOutput *VideoInput::texture_output()
{
return texture_output_;
}
void VideoInput::Hash(QCryptographicHash *hash, NodeOutput *from, const rational &time)
{
Node::Hash(hash, from, time);
// Use frame value from Decoder
if (from == texture_output_) {
if (!SetupDecoder()) {
qDebug() << "Failed to setup decoder for hashing";
return;
}
int64_t timestamp = decoder_->GetTimestampFromTime(time);
QByteArray pts_bytes;
pts_bytes.resize(sizeof(int64_t));
memcpy(pts_bytes.data(), &timestamp, sizeof(int64_t));
hash->addData(pts_bytes);
// FIXME: Add OCIO data
// FIXME: Add alpha association value
}
}
QVariant VideoInput::Value(NodeOutput *output, const rational &in, const rational &out)
{
Q_UNUSED(out)
// FIXME: Hardcoded value
bool alpha_is_associated = false;
if (output == texture_output_) {
// Find the current Renderer instance
RenderInstance* renderer = VideoRendererProcessor::CurrentInstance();
// If nothing is available, don't return a texture
if (renderer == nullptr) {
return 0;
}
// Make sure decoder is set up
if (!SetupDecoder()) {
return 0;
}
// Check if we need to get a frame or not
if (frame_ == nullptr || frame_->native_timestamp() != decoder_->GetTimestampFromTime(in)) {
// Get frame from Decoder
frame_ = decoder_->Retrieve(in);
if (frame_ == nullptr) {
qDebug() << "Received a null frame while time was" << in.toDouble();
return 0;
}
if (color_processor_ == nullptr) {
QString colorspace = std::static_pointer_cast<VideoStream>(Footage())->colorspace();
if (colorspace.isEmpty()) {
// FIXME: Should use Footage() to find the Project* it belongs to instead of this
colorspace = olive::core.GetActiveProject()->default_input_colorspace();
}
color_processor_ = ColorProcessor::Create(colorspace, OCIO::ROLE_SCENE_LINEAR);
}
// OpenColorIO v1's color transforms can be done on GPU, which improves performance but reduces accuracy. When
// online, we prefer accuracy over performance so we use the CPU path instead:
// NOTE: OCIO v2 boasts 1:1 results with the CPU and GPU path so this won't be necessary forever
if (renderer->params().mode() == olive::RenderMode::kOnline) {
// Convert to 32F, which is required for OpenColorIO's color transformation
frame_ = PixelService::ConvertPixelFormat(frame_, olive::PIX_FMT_RGBA32F);
if (alpha_is_associated) {
// Unassociate alpha here if associated
ColorManager::DisassociateAlpha(frame_);
}
// Transform color to reference space
color_processor_->ConvertFrame(frame_);
if (alpha_is_associated) {
// If alpha was associated, reassociate here
ColorManager::ReassociateAlpha(frame_);
} else {
// If alpha was not associated, associate here
ColorManager::AssociateAlpha(frame_);
}
}
// We use an internal texture to bring the texture into GPU space before performing transformations
// Ensure the texture is the accurate to the frame
if (internal_tex_.width() != frame_->width()
|| internal_tex_.height() != frame_->height()
|| internal_tex_.format() != frame_->format()) {
internal_tex_.Destroy();
}
// Create or upload the new data to the texture
if (!internal_tex_.IsCreated()) {
internal_tex_.Create(renderer->context(),
frame_->width(),
frame_->height(),
static_cast<olive::PixelFormat>(frame_->format()),
frame_->data());
} else {
internal_tex_.Upload(frame_->data());
}
}
// Create new texture in reference space to send throughout the rest of the graph
RenderTexturePtr output_texture = std::make_shared<RenderTexture>();
output_texture->Create(renderer->context(),
renderer->params().effective_width(),
renderer->params().effective_height(),
renderer->params().format(),
RenderTexture::kDoubleBuffer);
// Using the transformation matrix, blit our internal texture (in frame format) to our output texture (in
// reference format)
if (renderer->params().mode() == olive::RenderMode::kOffline) {
// For offline rendering, OCIO's GPU path is acceptable:
// NOTE: OCIO v2 boasts 1:1 results with the CPU and GPU path so this won't be necessary forever
// Use an OCIO pipeline shader (which wraps in a default pipeline and will also handle alpha association)
if (pipeline_ == nullptr) {
pipeline_ = olive::ShaderGenerator::OCIOPipeline(renderer->context(),
ocio_texture_, // FIXME: A raw GLuint texture, should wrap this up
color_processor_->GetProcessor(),
alpha_is_associated);
// Used for cleanup later
ocio_ctx_ = renderer->context();
}
} else if (pipeline_ == nullptr) {
// In online, the color transformation was performed on the CPU (see above), so we only need to blit
pipeline_ = olive::ShaderGenerator::DefaultPipeline();
}
renderer->context()->functions()->glBlendFunc(GL_ONE, GL_ZERO);
// Draw onto the output texture using the renderer's framebuffer
renderer->buffer()->Attach(output_texture);
renderer->buffer()->Bind();
// Draw with the internal texture
internal_tex_.Bind();
QMatrix4x4 transform;
// Scale texture to a square for incoming matrix transformation
transform.scale(2.0f / static_cast<float>(renderer->params().width()),
2.0f / static_cast<float>(renderer->params().height()));
// Multiply by input transformation
transform *= matrix_input_->get_value(in).value<QMatrix4x4>();
// Scale texture to the media size
transform.scale(static_cast<float>(frame_->width()), static_cast<float>(frame_->height()));
transform.scale(0.5f, 0.5f);
// Use pipeline to blit using transformation matrix from input
if (renderer->params().mode() == olive::RenderMode::kOffline) {
olive::gl::OCIOBlit(pipeline_, ocio_texture_, false, transform);
} else {
olive::gl::Blit(pipeline_, false, transform);
}
// Release everything
internal_tex_.Release();
renderer->buffer()->Detach();
renderer->buffer()->Release();
return QVariant::fromValue(output_texture);
}
return 0;
}
app/node/input/media/video/video.h
+48
View File
@@ -0,0 +1,48 @@
#ifndef VIDEOINPUT_H
#define VIDEOINPUT_H
#include <QOpenGLTexture>
#include "../media.h"
#include "render/colormanager.h"
#include "render/rendertexture.h"
#include "render/gl/shadergenerators.h"
class VideoInput : public MediaInput
{
public:
VideoInput();
virtual QString Name() override;
virtual QString id() override;
virtual QString Category() override;
virtual QString Description() override;
virtual void Release() override;
NodeInput* matrix_input();
NodeOutput* texture_output();
virtual void Hash(QCryptographicHash *hash, NodeOutput* from, const rational &time) override;
protected:
virtual QVariant Value(NodeOutput* output, const rational& in, const rational& out) override;
private:
NodeInput* matrix_input_;
NodeOutput* texture_output_;
RenderTexture internal_tex_;
ColorProcessorPtr color_processor_;
ShaderPtr pipeline_;
QOpenGLContext* ocio_ctx_;
GLuint ocio_texture_;
};
#endif // VIDEOINPUT_H
app/widget/timelinewidget/tool/import.cpp
+2 -2
View File
@@ -28,7 +28,7 @@
#include "core.h"
#include "node/distort/transform/transform.h"
#include "node/color/opacity/opacity.h"
#include "node/input/media/media.h"
#include "node/input/media/video/video.h"
TrackType TrackTypeFromStreamType(Stream::Type stream_type)
{
@@ -217,7 +217,7 @@ void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event)
TimelineViewGhostItem* ghost = parent()->ghost_items_.at(i);
ClipBlock* clip = new ClipBlock();
MediaInput* media = new MediaInput();
VideoInput* media = new VideoInput();
TransformDistort* transform = new TransformDistort();
OpacityNode* opacity = new OpacityNode();