From d76ac6ff381661d929fc8d682ea39b21b67d3790 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Thu, 23 May 2019 02:22:45 +1000 Subject: [PATCH] began custom pix fmt converter --- CMakeLists.txt | 4 + decoders/decoder.cpp | 8 +- decoders/decoder.h | 19 +++- decoders/ffmpegdecoder.cpp | 26 +++++ decoders/ffmpegdecoder.h | 17 ++- decoders/ffmpegvideodecoder.cpp | 6 ++ decoders/ffmpegvideodecoder.h | 14 +++ decoders/frame.cpp | 6 ++ decoders/frame.h | 11 ++ decoders/pixelformatconverter.cpp | 168 ++++++++++++++++++++++++++++++ decoders/pixelformatconverter.h | 140 +++++++++++++++++++++++++ nodes/nodes/nodemedia.cpp | 53 ++-------- nodes/nodes/nodemedia.h | 23 ++++ ts/olive_ar.ts | 6 +- ts/olive_bs.ts | 6 +- ts/olive_de.ts | 6 +- ts/olive_es.ts | 6 +- ts/olive_fr.ts | 6 +- ts/olive_id.ts | 6 +- ts/olive_it.ts | 6 +- ts/olive_ru.ts | 6 +- ts/olive_sr.ts | 6 +- 22 files changed, 472 insertions(+), 77 deletions(-) create mode 100644 decoders/ffmpegvideodecoder.cpp create mode 100644 decoders/ffmpegvideodecoder.h create mode 100644 decoders/frame.cpp create mode 100644 decoders/frame.h create mode 100644 decoders/pixelformatconverter.cpp create mode 100644 decoders/pixelformatconverter.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d2b0ffec..fca5644f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,10 @@ set(OLIVE_SOURCES decoders/decoder.h decoders/ffmpegdecoder.cpp decoders/ffmpegdecoder.h + decoders/ffmpegvideodecoder.cpp + decoders/ffmpegvideodecoder.h + decoders/pixelformatconverter.cpp + decoders/pixelformatconverter.h dialogs/aboutdialog.cpp dialogs/aboutdialog.h dialogs/actionsearch.cpp diff --git a/decoders/decoder.cpp b/decoders/decoder.cpp index 8de99a7b3..ba9646722 100644 --- a/decoders/decoder.cpp +++ b/decoders/decoder.cpp @@ -1,6 +1,12 @@ #include "decoder.h" -Decoder::Decoder() +Decoder::Decoder() : + open_(false) +{ + +} + +Decoder::~Decoder() { } diff --git a/decoders/decoder.h b/decoders/decoder.h index 8c2bd04d0..67a034899 100644 --- a/decoders/decoder.h +++ b/decoders/decoder.h @@ -1,11 +1,28 @@ #ifndef DECODER_H #define DECODER_H +#include + +#include "global/rational.h" class Decoder { public: Decoder(); + virtual ~Decoder(); + + virtual bool Open() = 0; + virtual void Request(const rational& timecode) = 0; + virtual void Retrieve(uint8_t** buffer, int* linesize) = 0; + virtual void Close() = 0; + + // For video decoding + virtual int width() = 0; + virtual int height() = 0; + +protected: + bool open_; + }; -#endif // DECODER_H \ No newline at end of file +#endif // DECODER_H diff --git a/decoders/ffmpegdecoder.cpp b/decoders/ffmpegdecoder.cpp index 26cf3c579..bcd4234ed 100644 --- a/decoders/ffmpegdecoder.cpp +++ b/decoders/ffmpegdecoder.cpp @@ -4,3 +4,29 @@ FFmpegDecoder::FFmpegDecoder() { } + +bool FFmpegDecoder::Open() +{ + if (open_) { + return true; + } + + +} + +void FFmpegDecoder::Request(const rational &timecode) +{ + +} + +void FFmpegDecoder::Retrieve(uint8_t **buffer, int *linesize) +{ + +} + +void FFmpegDecoder::Close() +{ + if (!open_) { + return; + } +} diff --git a/decoders/ffmpegdecoder.h b/decoders/ffmpegdecoder.h index 52f614f40..bd016b83f 100644 --- a/decoders/ffmpegdecoder.h +++ b/decoders/ffmpegdecoder.h @@ -1,11 +1,24 @@ #ifndef FFMPEGDECODER_H #define FFMPEGDECODER_H +#include "decoder.h" -class FFmpegDecoder +class FFmpegDecoder : public Decoder { public: FFmpegDecoder(); + + virtual bool Open(); + virtual void Request(const rational& timecode); + virtual void Retrieve(uint8_t** buffer, int* linesize); + virtual void Close(); + +private: + AVFormatContext* fmt_ctx_; + AVCodecContext* codec_ctx_; + AVStream* stream_; + AVPacket* pkt_; + AVFrame* frame_; }; -#endif // FFMPEGDECODER_H \ No newline at end of file +#endif // FFMPEGDECODER_H diff --git a/decoders/ffmpegvideodecoder.cpp b/decoders/ffmpegvideodecoder.cpp new file mode 100644 index 000000000..65985bc8f --- /dev/null +++ b/decoders/ffmpegvideodecoder.cpp @@ -0,0 +1,6 @@ +#include "ffmpegvideodecoder.h" + +FFmpegVideoDecoder::FFmpegVideoDecoder() +{ + +} diff --git a/decoders/ffmpegvideodecoder.h b/decoders/ffmpegvideodecoder.h new file mode 100644 index 000000000..7fd4dc632 --- /dev/null +++ b/decoders/ffmpegvideodecoder.h @@ -0,0 +1,14 @@ +#ifndef FFMPEGVIDEODECODER_H +#define FFMPEGVIDEODECODER_H + +#include "ffmpegdecoder.h" + +class FFmpegVideoDecoder : public FFmpegDecoder +{ +public: + FFmpegVideoDecoder(); + + +}; + +#endif // FFMPEGVIDEODECODER_H diff --git a/decoders/frame.cpp b/decoders/frame.cpp new file mode 100644 index 000000000..27fbdb46f --- /dev/null +++ b/decoders/frame.cpp @@ -0,0 +1,6 @@ +#include "frame.h" + +Frame::Frame() +{ + +} diff --git a/decoders/frame.h b/decoders/frame.h new file mode 100644 index 000000000..d6852c659 --- /dev/null +++ b/decoders/frame.h @@ -0,0 +1,11 @@ +#ifndef FRAME_H +#define FRAME_H + + +class Frame +{ +public: + Frame(); +}; + +#endif // FRAME_H diff --git a/decoders/pixelformatconverter.cpp b/decoders/pixelformatconverter.cpp new file mode 100644 index 000000000..ef5d73863 --- /dev/null +++ b/decoders/pixelformatconverter.cpp @@ -0,0 +1,168 @@ +#include "pixelformatconverter.h" + +#include +#include + +PixelFormatConverter olive::pix_fmt_conv; + +PixelFormatConverter::PixelFormatConverter() +{ + threads_.resize(qMax(1, QThread::idealThreadCount()-1)); + + for (int i=0;istart(QThread::HighPriority); + } +} + +PixelFormatConverter::~PixelFormatConverter() +{ + for (int i=0;iConvert(input_buffer, + input_linesize, + width, + line_start, + qMin(lines_per_thread, height - line_start), + input_fmt, + output_buffer, + output_fmt); + } + + // Wait for each thread to complete + for (int i=0;iwait(); + } + + + mutex_.unlock(); +} + +int PixelFormatConverter::GetBufferSize(olive::PixelFormat format, const int &width, const int &height) +{ + switch (format) { + case olive::PIX_FMT_RGBA8: + return width * height; + case olive::PIX_FMT_RGBA16: + case olive::PIX_FMT_RGBA16F: + return 2 * width * height; + case olive::PIX_FMT_RGBA32F: + return 4 * width * height; + default: + return 0; + } +} + +PixFmtConvertThread::PixFmtConvertThread() : + cancelled_(false) +{ +} + +void PixFmtConvertThread::run() +{ + while (!cancelled_) { + wait_cond_.wait(&mutex_); + if (cancelled_) break; + + Process(); + } +} + +void PixFmtConvertThread::Convert(uint8_t **input_buffer, + int *input_linesize, + const int &width, + const int &line_start, + const int &line_count, + AVPixelFormat input_fmt, + void *output_buffer, + olive::PixelFormat output_fmt) +{ + mutex_.lock(); + + input_buffer_ = input_buffer; + input_linesize_ = input_linesize; + width_ = width; + line_start_ = line_start; + line_count_ = line_count; + input_fmt_ = input_fmt; + output_buffer_ = output_buffer; + output_fmt_ = output_fmt; + + wait_cond_.wakeAll(); + + mutex_.unlock(); +} + +void PixFmtConvertThread::Cancel() +{ + cancelled_ = true; + wait_cond_.wakeAll(); + wait(); +} + +void PixFmtConvertThread::Process() +{ + switch (input_fmt_) { + case AV_PIX_FMT_RGBA: + { + int in_start = input_linesize_[0]*line_start_; + int out_start = width_*line_start_; + float f; + int channels = 4; // RGBA + + for (int i=0;i(output_buffer_)[out_line_start+j] = f; + + } + } + break; + } + default: + qWarning() << "PixFmtConvertThread doesn't yet know how to convert pixel format" << input_fmt_; + } +} diff --git a/decoders/pixelformatconverter.h b/decoders/pixelformatconverter.h new file mode 100644 index 000000000..75d196523 --- /dev/null +++ b/decoders/pixelformatconverter.h @@ -0,0 +1,140 @@ +#ifndef PIXELFORMATCONVERTER_H +#define PIXELFORMATCONVERTER_H + +extern "C" { +#include +} + +#include +#include +#include + +#include "rendering/pixelformats.h" + +class PixFmtConvertThread : public QThread { +public: + PixFmtConvertThread(); + + virtual void run() override; + + void Convert(uint8_t** input_buffer, + int* input_linesize, + const int &width, + const int &line_start, + const int &line_count, + AVPixelFormat input_fmt, + void* output_buffer, + olive::PixelFormat output_fmt); + void Cancel(); + +private: + // Main processing function + void Process(); + + // Threading variables + QWaitCondition wait_cond_; + QMutex mutex_; + bool cancelled_; + + // Input variables for conversion + uint8_t** input_buffer_; + int* input_linesize_; + int width_; + int line_start_; + int line_count_; + AVPixelFormat input_fmt_; + + // Output variables for conversion + void* output_buffer_; + olive::PixelFormat output_fmt_; +}; + +/** + * @brief The PixelFormatConverter class + * + * A background process for optimized conversion of frames to Olive's internal pipeline format. It is designed as a + * replacement of FFmpeg's swscale for all pixel format conversions. It's main advantages over swscale are: + * + * * Support for floating-point pixel formats + * * Better control over the conversion from YUV to RGB + * * Multithreaded by design + * + * In the old system, frame pixel formats were converted twice - once to RGBA32 or RGBA64 by swscale, then a second + * time to RGBA16F or RGBA32F by OpenGL. In addition to the aforementioned advantages, a future goal of this class is to + * reduce those two steps to one. + */ +class PixelFormatConverter +{ +public: + PixelFormatConverter(); + + ~PixelFormatConverter(); + + /** + * @brief Converts data from FFmpeg (or equivalent data) to data ready for Olive's buffer + * + * @param input_buffer + * + * The buffer of data to convert. This can be fed AVFrame->data directly. + * + * @param input_linesize + * + * The linesize of the data to convert. This can be fed AVFrame->linesize directly. + * + * @param width + * + * Width of the frame. This will be the same in the output buffer. + * + * @param height + * + * Height of the frame. This will be the same in the output buffer. + * + * @param input_fmt + * + * Expects a value from the AVPixelFormat enum. + * + * @param output_buffer + * + * The buffer to place the converted data. This function does NOT allocate data, this must be done beforehand. + * The helper function GetBufferSize() can be used to determine the minimum size (in bytes) that this buffer + * needs to have allocated for this function to be successful. + * + * @param output_fmt + * + * The format to convert to. Must be a member of the olive::PixelFormat enum. + */ + void AVFrameToPipeline(uint8_t** input_buffer, + int* input_linesize, + const int &width, + const int &height, + AVPixelFormat input_fmt, + void* output_buffer, + olive::PixelFormat output_fmt); + + /** + * @brief Returns the minimum buffer size (in bytes) necessary for a given format, width, and height. + * + * @param format + * + * The format of the data the buffer should contain. Must be a member of the olive::PixelFormat enum. + * + * @param width + * + * The width (in pixels) of the buffer. + * + * @param height + * + * The height (in pixels) of the buffer. + */ + static int GetBufferSize(olive::PixelFormat format, const int& width, const int& height); + +private: + QMutex mutex_; + QVector threads_; +}; + +namespace olive { +extern PixelFormatConverter pix_fmt_conv; +} + +#endif // PIXELFORMATCONVERTER_H diff --git a/nodes/nodes/nodemedia.cpp b/nodes/nodes/nodemedia.cpp index d7df4da39..34ce77fe6 100644 --- a/nodes/nodes/nodemedia.cpp +++ b/nodes/nodes/nodemedia.cpp @@ -6,6 +6,7 @@ NodeMedia::NodeMedia(NodeGraph* c) : Node(c), + media_(nullptr), buffer_(c->memory_cache()) { matrix_input_ = new NodeIO(this, "matrix", tr("Matrix"), true, false); @@ -25,65 +26,25 @@ QString NodeMedia::id() return "org.olivevideoeditor.Olive.media"; } -double H = 0.0; - void NodeMedia::Process(const rational &time) { Q_UNUSED(time) buffer_.buffer()->BindBuffer(); - // DEBUG CODE - we should see a solid rainbow color return from this function - H += 1; - double S = 1.0; - double V = 1.0; - double C = S * V; - double X = C * (1 - abs(fmod(H / 60.0, 2) - 1)); - double m = V - C; - double Rs, Gs, Bs; - - if(H >= 0 && H < 60) { - Rs = C; - Gs = X; - Bs = 0; - } - else if(H >= 60 && H < 120) { - Rs = X; - Gs = C; - Bs = 0; - } - else if(H >= 120 && H < 180) { - Rs = 0; - Gs = C; - Bs = X; - } - else if(H >= 180 && H < 240) { - Rs = 0; - Gs = X; - Bs = C; - } - else if(H >= 240 && H < 300) { - Rs = X; - Gs = 0; - Bs = C; - } - else { - Rs = C; - Gs = 0; - Bs = X; - } - - glClearColor(1.0, 0.0, 0.0, 1.0); + glClearColor(0.5, 0.0, 1.0, 1.0); glClear(GL_COLOR_BUFFER_BIT); - // END DEBUG CODE buffer_.buffer()->ReleaseBuffer(); - qDebug() << "Node Media has" << buffer_.buffer()->texture(); - texture_output_->SetValue(buffer_.buffer()->texture()); } +void NodeMedia::SetMedia(Media *f) +{ + +} + NodeIO *NodeMedia::matrix_input() { return matrix_input_; diff --git a/nodes/nodes/nodemedia.h b/nodes/nodes/nodemedia.h index 708a36934..594aa74d6 100644 --- a/nodes/nodes/nodemedia.h +++ b/nodes/nodes/nodemedia.h @@ -3,7 +3,20 @@ #include "nodes/node.h" #include "rendering/memorycache.h" +#include "project/media.h" +/** + * @brief The NodeMedia class + * + * The NodeMedia node is the main entry point for external video/audio files into the Olive pipeline. Its + * responsibilities are as follows: + * + * * Use a Decoder to retrieve the correct raw frame data + * * Convert the frame data to the pipeline format (usually RGBA16F or RGBA32F) + * * Convert the frame to scene linear colorspace with OpenColorIO + * + * Naturally this Node is resource intensive and effort should be spent here to make this as performant as possible. + */ class NodeMedia : public Node { Q_OBJECT @@ -15,13 +28,23 @@ public: virtual void Process(const rational& time) override; + void SetMedia(Media* f); + NodeIO* matrix_input(); NodeIO* texture_output(); private: + // Matrix input for the transformation to apply to this media (if any) NodeIO* matrix_input_; + + // Texture output NodeIO* texture_output_; + // Media object to display + Media* media_; + + // Texture buffer + // TODO Probable cache point MemoryCache::Reference buffer_; }; diff --git a/ts/olive_ar.ts b/ts/olive_ar.ts index 2ed7f0291..698b218ad 100644 --- a/ts/olive_ar.ts +++ b/ts/olive_ar.ts @@ -2171,17 +2171,17 @@ Audio Layout: %6 NodeMedia - + Matrix - + Texture - + Media الوسائط diff --git a/ts/olive_bs.ts b/ts/olive_bs.ts index 7240e3c7c..2ee0d9b25 100644 --- a/ts/olive_bs.ts +++ b/ts/olive_bs.ts @@ -1975,17 +1975,17 @@ Audio Layout: %6 NodeMedia - + Matrix - + Texture - + Media diff --git a/ts/olive_de.ts b/ts/olive_de.ts index 43920e1da..b52f5e5d3 100644 --- a/ts/olive_de.ts +++ b/ts/olive_de.ts @@ -2191,17 +2191,17 @@ Audio Layout: %6 NodeMedia - + Matrix - + Texture - + Media Medien diff --git a/ts/olive_es.ts b/ts/olive_es.ts index 63011d122..868243769 100644 --- a/ts/olive_es.ts +++ b/ts/olive_es.ts @@ -1826,17 +1826,17 @@ Audio Layout: %6 NodeMedia - + Matrix - + Texture - + Media diff --git a/ts/olive_fr.ts b/ts/olive_fr.ts index 8799477a8..f34c9aa9e 100644 --- a/ts/olive_fr.ts +++ b/ts/olive_fr.ts @@ -2155,17 +2155,17 @@ Canaux audio : %6 NodeMedia - + Matrix - + Texture - + Media Média diff --git a/ts/olive_id.ts b/ts/olive_id.ts index 23f620c74..668e79061 100644 --- a/ts/olive_id.ts +++ b/ts/olive_id.ts @@ -2016,17 +2016,17 @@ Tata Audio: %6 NodeMedia - + Matrix - + Texture - + Media diff --git a/ts/olive_it.ts b/ts/olive_it.ts index 9595a392c..ff70116e6 100644 --- a/ts/olive_it.ts +++ b/ts/olive_it.ts @@ -2009,17 +2009,17 @@ Disposizione audio: %6 NodeMedia - + Matrix - + Texture - + Media Media diff --git a/ts/olive_ru.ts b/ts/olive_ru.ts index f460db0f5..4a22a0197 100644 --- a/ts/olive_ru.ts +++ b/ts/olive_ru.ts @@ -1906,17 +1906,17 @@ Audio Layout: %6 NodeMedia - + Matrix - + Texture - + Media Файл diff --git a/ts/olive_sr.ts b/ts/olive_sr.ts index ff7a54c50..bb2781e40 100644 --- a/ts/olive_sr.ts +++ b/ts/olive_sr.ts @@ -1962,17 +1962,17 @@ Audio Layout: %6 NodeMedia - + Matrix - + Texture - + Media