From d133ee98325a377a54187cb186b83b50c16191e2 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Wed, 26 Jun 2019 22:52:44 -0700 Subject: [PATCH] finished merging earlier decoder code --- app/decoder/CMakeLists.txt | 28 +++++ app/decoder/decoder.cpp | 28 +++++ app/decoder/decoder.h | 109 ++++++++++++++++++ app/decoder/ffmpeg/CMakeLists.txt | 22 ++++ app/decoder/ffmpeg/ffmpegdecoder.cpp | 138 +++++++++++++++++++++++ app/decoder/ffmpeg/ffmpegdecoder.h | 31 +++++ app/decoder/frame.cpp | 94 +++++++++++++++ app/decoder/frame.h | 100 ++++++++++++++++ app/decoder/probeserver.cpp | 8 ++ app/decoder/probeserver.h | 12 ++ app/project/item/footage/audiostream.cpp | 1 - 11 files changed, 570 insertions(+), 1 deletion(-) create mode 100644 app/decoder/CMakeLists.txt create mode 100644 app/decoder/decoder.cpp create mode 100644 app/decoder/decoder.h create mode 100644 app/decoder/ffmpeg/CMakeLists.txt create mode 100644 app/decoder/ffmpeg/ffmpegdecoder.cpp create mode 100644 app/decoder/ffmpeg/ffmpegdecoder.h create mode 100644 app/decoder/frame.cpp create mode 100644 app/decoder/frame.h create mode 100644 app/decoder/probeserver.cpp create mode 100644 app/decoder/probeserver.h diff --git a/app/decoder/CMakeLists.txt b/app/decoder/CMakeLists.txt new file mode 100644 index 000000000..00c3f9a67 --- /dev/null +++ b/app/decoder/CMakeLists.txt @@ -0,0 +1,28 @@ +# Olive - Non-Linear Video Editor +# Copyright (C) 2019 Olive Team +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +add_subdirectory(ffmpeg) + +set(OLIVE_SOURCES + ${OLIVE_SOURCES} + decoder/decoder.h + decoder/decoder.cpp + decoder/frame.h + decoder/frame.cpp + decoder/probeserver.h + decoder/probeserver.cpp + PARENT_SCOPE +) diff --git a/app/decoder/decoder.cpp b/app/decoder/decoder.cpp new file mode 100644 index 000000000..2b278d334 --- /dev/null +++ b/app/decoder/decoder.cpp @@ -0,0 +1,28 @@ +#include "decoder.h" + +Decoder::Decoder() : + open_(false) +{ +} + +Decoder::Decoder(Stream *fs) : + open_(false), + stream_(fs) +{ +} + +Decoder::~Decoder() +{ +} + +Stream *Decoder::stream() +{ + return stream_; +} + +void Decoder::set_stream(Stream *fs) +{ + Close(); + + stream_ = fs; +} diff --git a/app/decoder/decoder.h b/app/decoder/decoder.h new file mode 100644 index 000000000..f9a1591ed --- /dev/null +++ b/app/decoder/decoder.h @@ -0,0 +1,109 @@ +#ifndef DECODER_H +#define DECODER_H + +#include +#include + +#include "rational.h" +#include "project/item/footage/footage.h" +#include "decoder/frame.h" + +/** + * @brief The Decoder class + * + * A decoder's is the main class for bringing external media into Olive. Its responsibilities are to serve as + * abstraction from codecs/decoders and provide complete frames. These frames can be video or audio data and are + * provided as Frame objects in shared pointers to alleviate the responsibility of memory handling. + * + * The main function in a decoder is Retrieve() which should return complete image/audio data. A decoder should + * alleviate all the complexities of codec compression from the rest of the application (i.e. a decoder should never + * return a partial frame or require other parts of the system to interface directly with the codec). Often this will + * necessitate pre-emptively caching, indexing, or even fully transcoding media before using it which can be implemented + * through the Analyze() function. + * + * A decoder does NOT perform any pixel/sample format conversion. Frames should pass through the PixelFormatConverter + * (olive::pix_fmt_conv) to be utilized in the rest of the rendering pipeline. + */ +class Decoder : public QObject +{ + Q_OBJECT +public: + Decoder(); + + Decoder(Stream* fs); + + virtual ~Decoder(); + + Stream* stream(); + void set_stream(Stream* fs); + + /** + * @brief Open media/allocate memory + * + * Any file handles or memory allocation that needs to be done before this instance of a Decoder can return data + * should be done here. + * + * @return + * + * TRUE if successful and ready to return data, FALSE if failed to open and unable to retrieve data. If the function + * fails, any memory allocated should be free'd before returning FALSE, possibly by calling Close(). + */ + virtual bool Open() = 0; + + /** + * @brief Retrieve frame/data + * + * The main function for retrieving data from the Decoder. This function should always provide complete frame data + * (i.e. no partial frames or missing samples) at the timecode provided. The Decoder should perform any steps + * required to retrieve a complete frame separate from the rest of the program, using any form of caching/indexing + * to keep this as performant as possible. + * + * It's acceptable for this function to check whether the Decoder is open, and call Open() if not. If Open() returns + * false, this function should return nullptr. + * + * @param timecode + * + * The timecode (a rational in seconds) to retrieve the data at. + * + * @param length + * + * Audio only - ignored for video decoders. The total length of audio data to retrieve (a rational in seconds). + * + * @return + * + * A FramePtr of valid data at this timecode (of the requested length if this is audio media), or nullptr if there + * was nothing to retrieve at the provided timecode or the media could not be opened. + */ + virtual FramePtr Retrieve(const rational& timecode, const rational& length = 0) = 0; + + /** + * @brief Close media/deallocate memory + * + * Any file handles or memory allocations opened in Open() should be cleaned up here. + * + * As the main memory freeing function, it's good practice to call this in Open() if there's an error that prevents + * correct function before Open() returns. As such, Close() should be prepared for not all memory/file handles to + * have been opened successfully. + */ + virtual void Close() = 0; + + /** + * @brief Prepare footage for use by a Decoder later + * + * Needs fleshing out. This functions purpose will be to perform initial analyses of a video file. Any caching, + * indexing, or transcoding to help make this media performant and reliable should be done here. + */ + //virtual void Analyze();// = 0; + + + +protected: + bool open_; + +private: + Stream* stream_; +}; + +using DecoderPtr = std::shared_ptr; + +#endif // DECODER_H diff --git a/app/decoder/ffmpeg/CMakeLists.txt b/app/decoder/ffmpeg/CMakeLists.txt new file mode 100644 index 000000000..27512870b --- /dev/null +++ b/app/decoder/ffmpeg/CMakeLists.txt @@ -0,0 +1,22 @@ +# Olive - Non-Linear Video Editor +# Copyright (C) 2019 Olive Team +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set(OLIVE_SOURCES + ${OLIVE_SOURCES} + decoder/ffmpeg/ffmpegdecoder.h + decoder/ffmpeg/ffmpegdecoder.cpp + PARENT_SCOPE +) diff --git a/app/decoder/ffmpeg/ffmpegdecoder.cpp b/app/decoder/ffmpeg/ffmpegdecoder.cpp new file mode 100644 index 000000000..6ff55993d --- /dev/null +++ b/app/decoder/ffmpeg/ffmpegdecoder.cpp @@ -0,0 +1,138 @@ +#include "ffmpegdecoder.h" + +extern "C" { +#include +#include +} + +#include +#include +#include +#include + +FFmpegDecoder::FFmpegDecoder() : + fmt_ctx_(nullptr), + codec_ctx_(nullptr), + opts_(nullptr) +{ +} + +bool FFmpegDecoder::Open() +{ + if (open_) { + return true; + } + + int error_code; + + // Convert QString to a C strng + QByteArray ba = stream()->footage()->filename().toUtf8(); + const char* filename = ba.constData(); + + // Open file in a format context + error_code = avformat_open_input(&fmt_ctx_, filename, nullptr, nullptr); + + // Handle format context error + if (error_code != 0) { + FFmpegErr(error_code); + return false; + } + + // Get stream information from format + error_code = avformat_find_stream_info(fmt_ctx_, nullptr); + + // Handle get stream information error + if (error_code < 0) { + FFmpegErr(error_code); + return false; + } + + // Dump format information + av_dump_format(fmt_ctx_, stream()->index(), filename, 0); + + // Get reference to correct AVStream + avstream_ = fmt_ctx_->streams[stream()->index()]; + + // Find decoder + AVCodec* codec = avcodec_find_decoder(avstream_->codecpar->codec_id); + + // Handle failure to find decoder + if (codec == nullptr) { + Error(tr("Failed to find appropriate decoder for this codec (%1 :: %2)") + .arg(stream()->footage()->filename(), avstream_->codecpar->codec_id)); + return false; + } + + // Allocate context for the decoder + codec_ctx_ = avcodec_alloc_context3(codec); + if (codec_ctx_ == nullptr) { + Error(tr("Failed to allocate codec context (%1 :: %2)").arg(stream()->footage()->filename(), stream()->index())); + return false; + } + + // Copy parameters from the AVStream to the AVCodecContext + error_code = avcodec_parameters_to_context(codec_ctx_, avstream_->codecpar); + + // Handle failure to copy parameters + if (error_code < 0) { + FFmpegErr(error_code); + return false; + } + + // enable multithreading on decoding + error_code = av_dict_set(&opts_, "threads", "auto", 0); + + // Handle failure to set multithreaded decoding + if (error_code < 0) { + FFmpegErr(error_code); + return false; + } + + // Open codec + error_code = avcodec_open2(codec_ctx_, codec, &opts_); + if (error_code < 0) { + FFmpegErr(error_code); + return false; + } + + open_ = true; + + return true; +} + +void FFmpegDecoder::Close() +{ + if (opts_ != nullptr) { + av_dict_free(&opts_); + opts_ = nullptr; + } + + if (codec_ctx_ != nullptr) { + avcodec_free_context(&codec_ctx_); + codec_ctx_ = nullptr; + } + + if (fmt_ctx_ != nullptr) { + avformat_close_input(&fmt_ctx_); + fmt_ctx_ = nullptr; + } + + open_ = false; +} + +void FFmpegDecoder::FFmpegErr(int error_code) +{ + char err[1024]; + av_strerror(error_code, err, 1024); + + Error(tr("Error decoding %1 - %2 %3").arg(stream()->footage()->filename(), + QString::number(error_code), + err)); +} + +void FFmpegDecoder::Error(const QString &s) +{ + qWarning() << s; + + Close(); +} diff --git a/app/decoder/ffmpeg/ffmpegdecoder.h b/app/decoder/ffmpeg/ffmpegdecoder.h new file mode 100644 index 000000000..c690a2fdd --- /dev/null +++ b/app/decoder/ffmpeg/ffmpegdecoder.h @@ -0,0 +1,31 @@ +#ifndef FFMPEGDECODER_H +#define FFMPEGDECODER_H + +#include + +#include "decoder/decoder.h" + +class FFmpegDecoder : public Decoder +{ +public: + FFmpegDecoder(); + + virtual bool Open() override; + virtual void Close() override; + +protected: + void FFmpegErr(int error_code); + void Error(const QString& s); + + AVFormatContext* fmt_ctx_; + AVCodecContext* codec_ctx_; + AVStream* avstream_; + AVPacket* pkt_; + AVFrame* frame_; + AVDictionary* opts_; + + QVector frame_index_; + +}; + +#endif // FFMPEGDECODER_H diff --git a/app/decoder/frame.cpp b/app/decoder/frame.cpp new file mode 100644 index 000000000..a1e9ae757 --- /dev/null +++ b/app/decoder/frame.cpp @@ -0,0 +1,94 @@ +#include "frame.h" + +#include + +Frame::Frame() : + frame_(nullptr) +{ +} + +Frame::Frame(AVFrame *f) : + frame_(f) +{ +} + +Frame::Frame(const Frame &f) : + frame_(nullptr) +{ + Q_UNUSED(f) +} + +Frame::Frame(Frame &&f) : + frame_(f.frame_) +{ + f.frame_ = nullptr; +} + +Frame &Frame::operator=(const Frame &f) +{ + Q_UNUSED(f) + + frame_ = nullptr; + return *this; +} + +Frame &Frame::operator=(Frame &&f) +{ + if (&f != this) { + frame_ = f.frame_; + f.frame_ = nullptr; + } + + return *this; +} + +Frame::~Frame() +{ + FreeChild(); +} + +void Frame::SetAVFrame(AVFrame *f, AVRational timebase) +{ + FreeChild(); + + f = frame_; + timestamp_ = rational(timebase.num*f->pts, timebase.den); +} + +const int &Frame::width() +{ + return frame_->width; +} + +const int &Frame::height() +{ + return frame_->height; +} + +const rational &Frame::timestamp() +{ + return timestamp_; +} + +const int &Frame::format() +{ + return frame_->format; +} + +uint8_t **Frame::data() +{ + return frame_->data; +} + +int *Frame::linesize() +{ + return frame_->linesize; +} + +void Frame::FreeChild() +{ + if (frame_ != nullptr) { + av_frame_free(&frame_); + frame_ = nullptr; + } +} diff --git a/app/decoder/frame.h b/app/decoder/frame.h new file mode 100644 index 000000000..77cf65273 --- /dev/null +++ b/app/decoder/frame.h @@ -0,0 +1,100 @@ +#ifndef FRAME_H +#define FRAME_H + +extern "C" { +#include +} + +#include + +#include "rational.h" + +/** + * @brief The Frame class + * + * Abstraction from AVFrame. Currently a simple AVFrame wrapper. + * + * This class does not support copying at this time. + */ +class Frame +{ +public: + // Normal constructor + Frame(); + + // AVFrame constructor + Frame(AVFrame* f); + + // Copy constructor + Frame(const Frame& f); + + // Move constructor + Frame(Frame&& f); + + // Copy assignment operator + Frame& operator=(const Frame& f); + + // Move assignment operator + Frame& operator=(Frame&& f); + + // Destructor + ~Frame(); + + /** + * @brief Set frame child + * + * This class currently primarily functions as a wrapper for AVFrame for use outside of the Decoder classes. + * The internal AVFrame is set here. This class will also take ownership of the AVFrame and automatically + * clear it when deconstructed. + * + * @param f + */ + void SetAVFrame(AVFrame* f, AVRational timebase); + + /** + * @brief Get frame's width in pixels + */ + const int& width(); + + /** + * @brief Get frame's height in pixels + */ + const int& height(); + + /** + * @brief Get frame's timestamp. + * + * This timestamp is always a rational that will equate to the time in seconds. + */ + const rational& timestamp(); + + /** + * @brief Get frame's format + * + * @return + * + * Currently this will either be an AVPixelFormat (video) or an AVSampleFormat (audio). + */ + const int& format(); + + /** + * @brief Get the data buffer of this frame + */ + uint8_t** data(); + + /** + * @brief Get the linesize information for this frame + */ + int* linesize(); + +private: + + void FreeChild(); + + AVFrame* frame_; + rational timestamp_; +}; + +using FramePtr = std::shared_ptr; + +#endif // FRAME_H diff --git a/app/decoder/probeserver.cpp b/app/decoder/probeserver.cpp new file mode 100644 index 000000000..1de734185 --- /dev/null +++ b/app/decoder/probeserver.cpp @@ -0,0 +1,8 @@ +#include "probeserver.h" + +#include "decoder/ffmpeg/ffmpegdecoder.h" + +bool olive::Probe(Footage *f) +{ + +} diff --git a/app/decoder/probeserver.h b/app/decoder/probeserver.h new file mode 100644 index 000000000..9ae67eb05 --- /dev/null +++ b/app/decoder/probeserver.h @@ -0,0 +1,12 @@ +#ifndef PROBESERVER_H +#define PROBESERVER_H + +#include "project/item/footage/footage.h" + +namespace olive { + +bool Probe(Footage* f); + +} + +#endif // PROBESERVER_H diff --git a/app/project/item/footage/audiostream.cpp b/app/project/item/footage/audiostream.cpp index a4ccb9b65..349824544 100644 --- a/app/project/item/footage/audiostream.cpp +++ b/app/project/item/footage/audiostream.cpp @@ -22,7 +22,6 @@ AudioStream::AudioStream() { - } Stream::Type AudioStream::type()