began custom pix fmt converter

CMakeLists.txt

@@ -77,6 +77,10 @@ set(OLIVE_SOURCES
   decoders/decoder.h
   decoders/ffmpegdecoder.cpp
   decoders/ffmpegdecoder.h
+  decoders/ffmpegvideodecoder.cpp
+  decoders/ffmpegvideodecoder.h
+  decoders/pixelformatconverter.cpp
+  decoders/pixelformatconverter.h
   dialogs/aboutdialog.cpp
   dialogs/aboutdialog.h
   dialogs/actionsearch.cpp

decoders/decoder.cpp

@@ -1,6 +1,12 @@
 #include "decoder.h"
 
-Decoder::Decoder()
+Decoder::Decoder() :
+  open_(false)
+{
+
+}
+
+Decoder::~Decoder()
 {
 
 }

decoders/decoder.h

@@ -1,11 +1,28 @@
 #ifndef DECODER_H
 #define DECODER_H
 
+#include <stdint.h>
+
+#include "global/rational.h"
 
 class Decoder
 {
 public:
   Decoder();
+  virtual ~Decoder();
+
+  virtual bool Open() = 0;
+  virtual void Request(const rational& timecode) = 0;
+  virtual void Retrieve(uint8_t** buffer, int* linesize) = 0;
+  virtual void Close() = 0;
+
+  // For video decoding
+  virtual int width() = 0;
+  virtual int height() = 0;
+
+protected:
+  bool open_;
+
 };
 
-#endif // DECODER_H
+#endif // DECODER_H

decoders/ffmpegdecoder.cpp

@@ -4,3 +4,29 @@ FFmpegDecoder::FFmpegDecoder()
 {
 
 }
+
+bool FFmpegDecoder::Open()
+{
+  if (open_) {
+    return true;
+  }
+
+
+}
+
+void FFmpegDecoder::Request(const rational &timecode)
+{
+
+}
+
+void FFmpegDecoder::Retrieve(uint8_t **buffer, int *linesize)
+{
+
+}
+
+void FFmpegDecoder::Close()
+{
+  if (!open_) {
+    return;
+  }
+}

decoders/ffmpegdecoder.h

@@ -1,11 +1,24 @@
 #ifndef FFMPEGDECODER_H
 #define FFMPEGDECODER_H
 
+#include "decoder.h"
 
-class FFmpegDecoder
+class FFmpegDecoder : public Decoder
 {
 public:
   FFmpegDecoder();
+
+  virtual bool Open();
+  virtual void Request(const rational& timecode);
+  virtual void Retrieve(uint8_t** buffer, int* linesize);
+  virtual void Close();
+
+private:
+  AVFormatContext* fmt_ctx_;
+  AVCodecContext* codec_ctx_;
+  AVStream* stream_;
+  AVPacket* pkt_;
+  AVFrame* frame_;
 };
 
-#endif // FFMPEGDECODER_H
+#endif // FFMPEGDECODER_H

decoders/ffmpegvideodecoder.cpp

@@ -0,0 +1,6 @@
+#include "ffmpegvideodecoder.h"
+
+FFmpegVideoDecoder::FFmpegVideoDecoder()
+{
+
+}

decoders/ffmpegvideodecoder.h

@@ -0,0 +1,14 @@
+#ifndef FFMPEGVIDEODECODER_H
+#define FFMPEGVIDEODECODER_H
+
+#include "ffmpegdecoder.h"
+
+class FFmpegVideoDecoder : public FFmpegDecoder
+{
+public:
+  FFmpegVideoDecoder();
+
+
+};
+
+#endif // FFMPEGVIDEODECODER_H

decoders/frame.cpp

@@ -0,0 +1,6 @@
+#include "frame.h"
+
+Frame::Frame()
+{
+
+}

decoders/frame.h

@@ -0,0 +1,11 @@
+#ifndef FRAME_H
+#define FRAME_H
+
+
+class Frame
+{
+public:
+  Frame();
+};
+
+#endif // FRAME_H

decoders/pixelformatconverter.cpp

@@ -0,0 +1,168 @@
+#include "pixelformatconverter.h"
+
+#include <QtMath>
+#include <QDebug>
+
+PixelFormatConverter olive::pix_fmt_conv;
+
+PixelFormatConverter::PixelFormatConverter()
+{
+  threads_.resize(qMax(1, QThread::idealThreadCount()-1));
+
+  for (int i=0;i<threads_.size();i++) {
+    threads_[i] = new PixFmtConvertThread();
+    threads_[i]->start(QThread::HighPriority);
+  }
+}
+
+PixelFormatConverter::~PixelFormatConverter()
+{
+  for (int i=0;i<threads_.size();i++) {
+    delete threads_[i];
+  }
+}
+
+void PixelFormatConverter::AVFrameToPipeline(uint8_t **input_buffer,
+                                             int *input_linesize,
+                                             const int& width,
+                                             const int& height,
+                                             AVPixelFormat input_fmt,
+                                             void *output_buffer,
+                                             olive::PixelFormat output_fmt)
+{
+  // TODO: Support all AVPixelFormats. There are many and this will be an ongoing process (though many will likely
+  //       not need much effort (or any support at all). In the time-being this function will continue to use swscale
+  //       (functioning as abstraction from the rest of the code) while remaining support is added.
+
+
+  // The main focus of this function is to convert to float. There's currently no intention to support converting to
+  // integer formats.
+  Q_ASSERT(output_fmt == olive::PIX_FMT_RGBA16F || output_fmt == olive::PIX_FMT_RGBA32F);
+
+
+  // Wait til other frame conversions are done
+  mutex_.lock();
+
+
+  // Split job between threads
+  int lines_per_thread = qCeil(qreal(height)/qreal(threads_.size()));
+
+  // Send conversion job to threads
+  for (int i=0;i<threads_.size();i++) {
+
+    int line_start = lines_per_thread*i;
+
+    threads_.at(i)->Convert(input_buffer,
+                            input_linesize,
+                            width,
+                            line_start,
+                            qMin(lines_per_thread, height - line_start),
+                            input_fmt,
+                            output_buffer,
+                            output_fmt);
+  }
+
+  // Wait for each thread to complete
+  for (int i=0;i<threads_.size();i++) {
+    threads_.at(i)->wait();
+  }
+
+
+  mutex_.unlock();
+}
+
+int PixelFormatConverter::GetBufferSize(olive::PixelFormat format, const int &width, const int &height)
+{
+  switch (format) {
+  case olive::PIX_FMT_RGBA8:
+    return width * height;
+  case olive::PIX_FMT_RGBA16:
+  case olive::PIX_FMT_RGBA16F:
+    return 2 * width * height;
+  case olive::PIX_FMT_RGBA32F:
+    return 4 * width * height;
+  default:
+    return 0;
+  }
+}
+
+PixFmtConvertThread::PixFmtConvertThread() :
+  cancelled_(false)
+{
+}
+
+void PixFmtConvertThread::run()
+{
+  while (!cancelled_) {
+    wait_cond_.wait(&mutex_);
+    if (cancelled_) break;
+
+    Process();
+  }
+}
+
+void PixFmtConvertThread::Convert(uint8_t **input_buffer,
+                                  int *input_linesize,
+                                  const int &width,
+                                  const int &line_start,
+                                  const int &line_count,
+                                  AVPixelFormat input_fmt,
+                                  void *output_buffer,
+                                  olive::PixelFormat output_fmt)
+{
+  mutex_.lock();
+
+  input_buffer_ = input_buffer;
+  input_linesize_ = input_linesize;
+  width_ = width;
+  line_start_ = line_start;
+  line_count_ = line_count;
+  input_fmt_ = input_fmt;
+  output_buffer_ = output_buffer;
+  output_fmt_ = output_fmt;
+
+  wait_cond_.wakeAll();
+
+  mutex_.unlock();
+}
+
+void PixFmtConvertThread::Cancel()
+{
+  cancelled_ = true;
+  wait_cond_.wakeAll();
+  wait();
+}
+
+void PixFmtConvertThread::Process()
+{
+  switch (input_fmt_) {
+  case AV_PIX_FMT_RGBA:
+  {
+    int in_start = input_linesize_[0]*line_start_;
+    int out_start = width_*line_start_;
+    float f;
+    int channels = 4; // RGBA
+
+    for (int i=0;i<line_count_;i++) {
+
+      int in_line_start = in_start + input_linesize_[0]*i;
+      int in_line_end = in_line_start + width_*channels;
+      int out_line_start = out_start + width_*i;
+
+      for (int j=in_line_start;j<in_line_end;j++) {
+
+        // Convert 8-bit integer to float
+        f = input_buffer_[0][in_line_start+j] / 255.0f;
+
+        // Place float into output buffer
+        // TODO only float support, no half float support yet
+        static_cast<float*>(output_buffer_)[out_line_start+j] = f;
+
+      }
+    }
+    break;
+  }
+  default:
+    qWarning() << "PixFmtConvertThread doesn't yet know how to convert pixel format" << input_fmt_;
+  }
+}

decoders/pixelformatconverter.h

@@ -0,0 +1,140 @@
+#ifndef PIXELFORMATCONVERTER_H
+#define PIXELFORMATCONVERTER_H
+
+extern "C" {
+#include <libavutil/pixfmt.h>
+}
+
+#include <QMutex>
+#include <QThread>
+#include <QWaitCondition>
+
+#include "rendering/pixelformats.h"
+
+class PixFmtConvertThread : public QThread {
+public:
+  PixFmtConvertThread();
+
+  virtual void run() override;
+
+  void Convert(uint8_t** input_buffer,
+               int* input_linesize,
+               const int &width,
+               const int &line_start,
+               const int &line_count,
+               AVPixelFormat input_fmt,
+               void* output_buffer,
+               olive::PixelFormat output_fmt);
+  void Cancel();
+
+private:
+  // Main processing function
+  void Process();
+
+  // Threading variables
+  QWaitCondition wait_cond_;
+  QMutex mutex_;
+  bool cancelled_;
+
+  // Input variables for conversion
+  uint8_t** input_buffer_;
+  int* input_linesize_;
+  int width_;
+  int line_start_;
+  int line_count_;
+  AVPixelFormat input_fmt_;
+
+  // Output variables for conversion
+  void* output_buffer_;
+  olive::PixelFormat output_fmt_;
+};
+
+/**
+ * @brief The PixelFormatConverter class
+ *
+ * A background process for optimized conversion of frames to Olive's internal pipeline format. It is designed as a
+ * replacement of FFmpeg's swscale for all pixel format conversions. It's main advantages over swscale are:
+ *
+ * * Support for floating-point pixel formats
+ * * Better control over the conversion from YUV to RGB
+ * * Multithreaded by design
+ *
+ * In the old system, frame pixel formats were converted twice - once to RGBA32 or RGBA64 by swscale, then a second
+ * time to RGBA16F or RGBA32F by OpenGL. In addition to the aforementioned advantages, a future goal of this class is to
+ * reduce those two steps to one.
+ */
+class PixelFormatConverter
+{
+public:
+  PixelFormatConverter();
+
+  ~PixelFormatConverter();
+
+  /**
+   * @brief Converts data from FFmpeg (or equivalent data) to data ready for Olive's buffer
+   *
+   * @param input_buffer
+   *
+   * The buffer of data to convert. This can be fed AVFrame->data directly.
+   *
+   * @param input_linesize
+   *
+   * The linesize of the data to convert. This can be fed AVFrame->linesize directly.
+   *
+   * @param width
+   *
+   * Width of the frame. This will be the same in the output buffer.
+   *
+   * @param height
+   *
+   * Height of the frame. This will be the same in the output buffer.
+   *
+   * @param input_fmt
+   *
+   * Expects a value from the AVPixelFormat enum.
+   *
+   * @param output_buffer
+   *
+   * The buffer to place the converted data. This function does NOT allocate data, this must be done beforehand.
+   * The helper function GetBufferSize() can be used to determine the minimum size (in bytes) that this buffer
+   * needs to have allocated for this function to be successful.
+   *
+   * @param output_fmt
+   *
+   * The format to convert to. Must be a member of the olive::PixelFormat enum.
+   */
+  void AVFrameToPipeline(uint8_t** input_buffer,
+                         int* input_linesize,
+                         const int &width,
+                         const int &height,
+                         AVPixelFormat input_fmt,
+                         void* output_buffer,
+                         olive::PixelFormat output_fmt);
+
+  /**
+   * @brief Returns the minimum buffer size (in bytes) necessary for a given format, width, and height.
+   *
+   * @param format
+   *
+   * The format of the data the buffer should contain. Must be a member of the olive::PixelFormat enum.
+   *
+   * @param width
+   *
+   * The width (in pixels) of the buffer.
+   *
+   * @param height
+   *
+   * The height (in pixels) of the buffer.
+   */
+  static int GetBufferSize(olive::PixelFormat format, const int& width, const int& height);
+
+private:
+  QMutex mutex_;
+  QVector<PixFmtConvertThread*> threads_;
+};
+
+namespace olive {
+extern PixelFormatConverter pix_fmt_conv;
+}
+
+#endif // PIXELFORMATCONVERTER_H

nodes/nodes/nodemedia.cpp

@@ -6,6 +6,7 @@
 
 NodeMedia::NodeMedia(NodeGraph* c) :
   Node(c),
+  media_(nullptr),
   buffer_(c->memory_cache())
 {
   matrix_input_ = new NodeIO(this, "matrix", tr("Matrix"), true, false);
@@ -25,65 +26,25 @@ QString NodeMedia::id()
   return "org.olivevideoeditor.Olive.media";
 }
 
-double H = 0.0;
-
 void NodeMedia::Process(const rational &time)
 {
   Q_UNUSED(time)
 
   buffer_.buffer()->BindBuffer();
 
-  // DEBUG CODE - we should see a solid rainbow color return from this function
-  H += 1;
-  double S = 1.0;
-  double V = 1.0;
-  double C = S * V;
-  double X = C * (1 - abs(fmod(H / 60.0, 2) - 1));
-  double m = V - C;
-  double Rs, Gs, Bs;
-
-  if(H >= 0 && H < 60) {
-    Rs = C;
-    Gs = X;
-    Bs = 0;
-  }
-  else if(H >= 60 && H < 120) {
-    Rs = X;
-    Gs = C;
-    Bs = 0;
-  }
-  else if(H >= 120 && H < 180) {
-    Rs = 0;
-    Gs = C;
-    Bs = X;
-  }
-  else if(H >= 180 && H < 240) {
-    Rs = 0;
-    Gs = X;
-    Bs = C;
-  }
-  else if(H >= 240 && H < 300) {
-    Rs = X;
-    Gs = 0;
-    Bs = C;
-  }
-  else {
-    Rs = C;
-    Gs = 0;
-    Bs = X;
-  }
-
-  glClearColor(1.0, 0.0, 0.0, 1.0);
+  glClearColor(0.5, 0.0, 1.0, 1.0);
   glClear(GL_COLOR_BUFFER_BIT);
-  // END DEBUG CODE
 
   buffer_.buffer()->ReleaseBuffer();
 
-  qDebug() << "Node Media has" << buffer_.buffer()->texture();
-
   texture_output_->SetValue(buffer_.buffer()->texture());
 }
 
+void NodeMedia::SetMedia(Media *f)
+{
+
+}
+
 NodeIO *NodeMedia::matrix_input()
 {
   return matrix_input_;

nodes/nodes/nodemedia.h

@@ -3,7 +3,20 @@
 
 #include "nodes/node.h"
 #include "rendering/memorycache.h"
+#include "project/media.h"
 
+/**
+ * @brief The NodeMedia class
+ *
+ * The NodeMedia node is the main entry point for external video/audio files into the Olive pipeline. Its
+ * responsibilities are as follows:
+ *
+ * * Use a Decoder to retrieve the correct raw frame data
+ * * Convert the frame data to the pipeline format (usually RGBA16F or RGBA32F)
+ * * Convert the frame to scene linear colorspace with OpenColorIO
+ *
+ * Naturally this Node is resource intensive and effort should be spent here to make this as performant as possible.
+ */
 class NodeMedia : public Node
 {
   Q_OBJECT
@@ -15,13 +28,23 @@ public:
 
   virtual void Process(const rational& time) override;
 
+  void SetMedia(Media* f);
+
   NodeIO* matrix_input();
   NodeIO* texture_output();
 
 private:
+  // Matrix input for the transformation to apply to this media (if any)
   NodeIO* matrix_input_;
+
+  // Texture output
   NodeIO* texture_output_;
 
+  // Media object to display
+  Media* media_;
+
+  // Texture buffer
+  // TODO Probable cache point
   MemoryCache::Reference buffer_;
 };
 

ts/olive_ar.ts

@@ -2171,17 +2171,17 @@ Audio Layout: %6</source>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished">الوسائط</translation>
     </message>

ts/olive_bs.ts

@@ -1975,17 +1975,17 @@ Audio Layout: %6</source>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished"></translation>
     </message>

ts/olive_de.ts

@@ -2191,17 +2191,17 @@ Audio Layout: %6</translation>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished">Medien</translation>
     </message>

ts/olive_es.ts

@@ -1826,17 +1826,17 @@ Audio Layout: %6</source>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished"></translation>
     </message>

ts/olive_fr.ts

@@ -2155,17 +2155,17 @@ Canaux audio : %6</translation>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished">Média</translation>
     </message>

ts/olive_id.ts

@@ -2016,17 +2016,17 @@ Tata Audio: %6</translation>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished"></translation>
     </message>

ts/olive_it.ts

@@ -2009,17 +2009,17 @@ Disposizione audio: %6</translation>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished">Media</translation>
     </message>

ts/olive_ru.ts

@@ -1906,17 +1906,17 @@ Audio Layout: %6</source>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished">Файл</translation>
     </message>

ts/olive_sr.ts

@@ -1962,17 +1962,17 @@ Audio Layout: %6</source>
 <context>
     <name>NodeMedia</name>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="11"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="12"/>
         <source>Matrix</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="14"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="15"/>
         <source>Texture</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <location filename="../nodes/nodes/nodemedia.cpp" line="20"/>
+        <location filename="../nodes/nodes/nodemedia.cpp" line="21"/>
         <source>Media</source>
         <translation type="unfinished"></translation>
     </message>