work on decoder classes

CMakeLists.txt

@@ -79,8 +79,12 @@ set(OLIVE_SOURCES
   decoders/ffmpegdecoder.h
   decoders/ffmpegvideodecoder.cpp
   decoders/ffmpegvideodecoder.h
+  decoders/frame.cpp
+  decoders/frame.h
   decoders/pixelformatconverter.cpp
   decoders/pixelformatconverter.h
+  decoders/audio/ffmpegaudiodecoder.cpp
+  decoders/audio/ffmpegaudiodecoder.h
   dialogs/aboutdialog.cpp
   dialogs/aboutdialog.h
   dialogs/actionsearch.cpp

decoders/audio/ffmpegaudiodecoder.cpp

@@ -0,0 +1,6 @@
+#include "ffmpegaudiodecoder.h"
+
+FFmpegAudioDecoder::FFmpegAudioDecoder()
+{
+
+}

decoders/audio/ffmpegaudiodecoder.h

@@ -0,0 +1,17 @@
+#ifndef FFMPEGAUDIODECODER_H
+#define FFMPEGAUDIODECODER_H
+
+#include "decoders/ffmpegdecoder.h"
+
+/**
+ * @brief The FFmpegAudioDecoder class
+ *
+ * The role of an audio decoder is to simply convert audio to
+ */
+class FFmpegAudioDecoder : public FFmpegDecoder
+{
+public:
+  FFmpegAudioDecoder();
+};
+
+#endif // FFMPEGAUDIODECODER_H

decoders/decoder.cpp

@@ -1,12 +1,29 @@
 #include "decoder.h"
 
-Decoder::Decoder() :
+Decoder::Decoder(QObject *parent) :
+  QObject(parent),
   open_(false)
 {
+}
 
+Decoder::Decoder(FootageStream *fs) :
+  open_(false),
+  stream_(fs)
+{
 }
 
 Decoder::~Decoder()
 {
+}
 
+FootageStream *Decoder::stream()
+{
+  return stream_;
+}
+
+void Decoder::set_stream(FootageStream *fs)
+{
+  Close();
+
+  stream_ = fs;
 }

decoders/decoder.h

@@ -4,16 +4,33 @@
 #include <stdint.h>
 
 #include "global/rational.h"
+#include "project/footage.h"
+#include "frame.h"
 
-class Decoder
+/**
+ * @brief The Decoder class
+ *
+ * A decoder's is the main class for bringing external media into Olive. Its responsibilities are to serve as
+ * abstraction from codecs/decoders and provide complete frames. These frames can be video or audio data and are
+ * provided as Frame objects in shared pointers to alleviate the responsibility of memory handling.
+ *
+ * A decoder does NOT perform any pixel format conversion
+ */
+class Decoder : public QObject
 {
+  Q_OBJECT
 public:
-  Decoder();
+  Decoder(QObject* parent = nullptr);
+
+  Decoder(FootageStream* fs, QObject *parent = nullptr);
+
   virtual ~Decoder();
 
+  FootageStream* stream();
+  void set_stream(FootageStream* fs);
+
   virtual bool Open() = 0;
-  virtual void Request(const rational& timecode) = 0;
-  virtual void Retrieve(uint8_t** buffer, int* linesize) = 0;
+  virtual FramePtr Retrieve(const rational& timecode, const rational& length = 0) = 0;
   virtual void Close() = 0;
 
   // For video decoding
@@ -23,6 +40,10 @@ public:
 protected:
   bool open_;
 
+private:
+  FootageStream* stream_;
 };
 
+using DecoderPtr = std::shared_ptr<Decoder>;
+
 #endif // DECODER_H

decoders/ffmpegdecoder.cpp

@@ -1,8 +1,23 @@
 #include "ffmpegdecoder.h"
 
-FFmpegDecoder::FFmpegDecoder()
-{
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+}
 
+#include <QStatusBar>
+#include <QString>
+#include <QtMath>
+#include <QDebug>
+
+#include "ui/mainwindow.h"
+
+FFmpegDecoder::FFmpegDecoder(QObject *parent) :
+  Decoder(parent),
+  fmt_ctx_(nullptr),
+  codec_ctx_(nullptr),
+  opts_(nullptr)
+{
 }
 
 bool FFmpegDecoder::Open()
@@ -11,22 +26,116 @@ bool FFmpegDecoder::Open()
     return true;
   }
 
+  int error_code;
 
-}
+  QByteArray ba = stream()->footage->url.toUtf8();
+  const char* filename = ba.constData();
 
-void FFmpegDecoder::Request(const rational &timecode)
-{
+  // Open file in a format context
+  error_code = avformat_open_input(&fmt_ctx_, filename, nullptr, nullptr);
 
-}
+  // Handle format context error
+  if (error_code != 0) {
+    FFmpegErr(error_code);
+    return false;
+  }
 
-void FFmpegDecoder::Retrieve(uint8_t **buffer, int *linesize)
-{
+  // Get stream information from format
+  error_code = avformat_find_stream_info(fmt_ctx_, nullptr);
 
+  // Handle get stream information error
+  if (error_code < 0) {
+    FFmpegErr(error_code);
+    return false;
+  }
+
+  // Dump format information
+  av_dump_format(fmt_ctx_, stream()->file_index, filename, 0);
+
+  // Get reference to correct AVStream
+  avstream_ = fmt_ctx_->streams[stream()->file_index];
+
+  // Find decoder
+  AVCodec* codec = avcodec_find_decoder(avstream_->codecpar->codec_id);
+
+  // Handle failure to find decoder
+  if (codec == nullptr) {
+    Error(tr("Failed to find appropriate decoder for this codec (%1 :: %2)")
+          .arg(stream()->footage->url, avstream_->codecpar->codec_id));
+    return false;
+  }
+
+  // Allocate context for the decoder
+  codec_ctx_ = avcodec_alloc_context3(codec);
+  if (codec_ctx_ == nullptr) {
+    Error(tr("Failed to allocate codec context (%1 :: %2)").arg(stream()->footage->url, stream()->file_index));
+    return false;
+  }
+
+  // Copy parameters from the AVStream to the AVCodecContext
+  error_code = avcodec_parameters_to_context(codec_ctx_, avstream_->codecpar);
+
+  // Handle failure to copy parameters
+  if (error_code < 0) {
+    FFmpegErr(error_code);
+    return false;
+  }
+
+  // enable multithreading on decoding
+  error_code = av_dict_set(&opts_, "threads", "auto", 0);
+
+  // Handle failure to set multithreaded decoding
+  if (error_code < 0) {
+    FFmpegErr(error_code);
+    return false;
+  }
+
+  // Open codec
+  error_code = avcodec_open2(codec_ctx_, codec, &opts_);
+  if (error_code < 0) {
+    FFmpegErr(error_code);
+    return false;
+  }
+
+  open_ = true;
+
+  return true;
 }
 
 void FFmpegDecoder::Close()
 {
-  if (!open_) {
-    return;
+  if (opts_ != nullptr) {
+    av_dict_free(&opts_);
+    opts_ = nullptr;
   }
+
+  if (codec_ctx_ != nullptr) {
+    avcodec_free_context(&codec_ctx_);
+    codec_ctx_ = nullptr;
+  }
+
+  if (fmt_ctx_ != nullptr) {
+    avformat_close_input(&fmt_ctx_);
+    fmt_ctx_ = nullptr;
+  }
+
+  open_ = false;
+}
+
+void FFmpegDecoder::FFmpegErr(int error_code)
+{
+  char err[1024];
+  av_strerror(error_code, err, 1024);
+
+  Error(tr("Error decoding %1 - %2 %3").arg(stream()->footage->url,
+                                            QString::number(error_code),
+                                            err));
+}
+
+void FFmpegDecoder::Error(const QString &s)
+{
+  qWarning() << s;
+  olive::MainWindow->statusBar()->showMessage(s);
+
+  Close();
 }

decoders/ffmpegdecoder.h

@@ -8,17 +8,22 @@ class FFmpegDecoder : public Decoder
 public:
   FFmpegDecoder();
 
-  virtual bool Open();
-  virtual void Request(const rational& timecode);
-  virtual void Retrieve(uint8_t** buffer, int* linesize);
-  virtual void Close();
+  virtual bool Open() override;
+  virtual void Close() override;
+
+protected:
+  void FFmpegErr(int error_code);
+  void Error(const QString& s);
 
-private:
   AVFormatContext* fmt_ctx_;
   AVCodecContext* codec_ctx_;
-  AVStream* stream_;
+  AVStream* avstream_;
   AVPacket* pkt_;
   AVFrame* frame_;
+  AVDictionary* opts_;
+
+  QVector<int64_t> frame_index_;
+
 };
 
 #endif // FFMPEGDECODER_H

decoders/ffmpegvideodecoder.cpp

@@ -1,6 +1,63 @@
 #include "ffmpegvideodecoder.h"
 
-FFmpegVideoDecoder::FFmpegVideoDecoder()
+#include <QDebug>
+
+FFmpegVideoDecoder::FFmpegVideoDecoder(QObject *parent) :
+  FFmpegDecoder(parent)
 {
 
 }
+
+FramePtr FFmpegVideoDecoder::Retrieve(const rational &timecode, const rational &length)
+{
+  // TODO index frames
+
+  int receive_ret;
+
+  // Allocate new frame
+  AVFrame* frame = av_frame_alloc();
+  AVPacket* pkt = av_packet_alloc();
+
+  while ((receive_ret = avcodec_receive_frame(codec_ctx_, frame)) == AVERROR(EAGAIN)) {
+    int read_ret = 0;
+    do {
+
+      // Make sure any previous packet buffers are cleared before reading new ones
+      av_packet_unref(pkt);
+
+      // Read next packet from file
+      read_ret = av_read_frame(fmt_ctx_, pkt);
+
+    } while (read_ret >= 0 && pkt->stream_index != stream()->file_index);
+
+    if (read_ret >= 0) {
+      int send_ret = avcodec_send_packet(codec_ctx_, pkt);
+      if (send_ret < 0) {
+        qCritical() << "Failed to send packet to decoder." << send_ret;
+        return nullptr;
+      }
+    } else {
+      if (read_ret == AVERROR_EOF) {
+        int send_ret = avcodec_send_packet(codec_ctx_, nullptr);
+        if (send_ret < 0) {
+          qCritical() << "Failed to send packet to decoder." << send_ret;
+          return nullptr;
+        }
+      } else {
+        qCritical() << "Could not read frame." << read_ret;
+        return nullptr;
+      }
+    }
+  }
+  if (receive_ret < 0) {
+    if (receive_ret != AVERROR_EOF) {
+      qCritical() << "Failed to receive packet from decoder." << receive_ret;
+    }
+  }
+
+  // Free AVPacket
+  av_packet_free(&pkt);
+
+  // AVFrame is wrapped in Frame object and shared ptr to automatically clear it
+  return std::make_shared<Frame>(frame);
+}

decoders/ffmpegvideodecoder.h

@@ -8,7 +8,9 @@ class FFmpegVideoDecoder : public FFmpegDecoder
 public:
   FFmpegVideoDecoder();
 
+  virtual FramePtr Retrieve(const rational& timecode, const rational& length = 0);
 
+private:
 };
 
 #endif // FFMPEGVIDEODECODER_H

decoders/frame.cpp

@@ -1,6 +1,79 @@
-#include "frame.h"
-
-Frame::Frame()
-{
-
-}
+#include "frame.h"
+
+Frame::Frame() :
+  frame_(nullptr)
+{
+}
+
+Frame::Frame(AVFrame *f) :
+  frame_(f)
+{
+}
+
+Frame::Frame(const Frame &f) :
+  frame_(nullptr)
+{
+}
+
+Frame::Frame(Frame &&f) :
+  frame_(f.frame_)
+{
+  f.frame_ = nullptr;
+}
+
+Frame &Frame::operator=(const Frame &f)
+{
+  frame_ = nullptr;
+  return *this;
+}
+
+Frame &Frame::operator=(Frame &&f)
+{
+  if (&f != this) {
+    frame_ = f.frame_;
+    f.frame_ = nullptr;
+  }
+
+  return *this;
+}
+
+Frame::~Frame()
+{
+  FreeChild();
+}
+
+void Frame::SetAVFrame(AVFrame *f, AVRational timebase)
+{
+  FreeChild();
+
+  f = frame_;
+  timestamp_ = rational(timebase.num*f->pts, timebase.den);
+}
+
+const int &Frame::width()
+{
+  return frame_->width;
+}
+
+const int &Frame::height()
+{
+  return frame_->height;
+}
+
+const rational &Frame::timestamp()
+{
+  return timestamp_;
+}
+
+const int &Frame::format()
+{
+  return frame_->format;
+}
+
+void Frame::FreeChild()
+{
+  if (frame_ != nullptr) {
+    av_frame_free(&frame_);
+    frame_ = nullptr;
+  }
+}

decoders/frame.h

@@ -1,11 +1,86 @@
-#ifndef FRAME_H
-#define FRAME_H
-
-
-class Frame
-{
-public:
-  Frame();
-};
-
-#endif // FRAME_H
+#ifndef FRAME_H
+#define FRAME_H
+
+#include <memory>
+
+#include "global/rational.h"
+
+/**
+ * @brief The Frame class
+ *
+ * Abstraction from AVFrame. Currently a simple AVFrame wrapper.
+ *
+ * This class does not support copying at this time.
+ */
+class Frame
+{
+public:
+  // Normal constructor
+  Frame();
+
+  // AVFrame constructor
+  Frame(AVFrame* f);
+
+  // Copy constructor
+  Frame(const Frame& f);
+
+  // Move constructor
+  Frame(Frame&& f);
+
+  // Copy assignment operator
+  Frame& operator=(const Frame& f);
+
+  // Move assignment operator
+  Frame& operator=(Frame&& f);
+
+  // Destructor
+  ~Frame();
+
+  /**
+   * @brief Set frame child
+   *
+   * This class currently primarily functions as a wrapper for AVFrame for use outside of the Decoder classes.
+   * The internal AVFrame is set here. This class will also take ownership of the AVFrame and automatically
+   * clear it when deconstructed.
+   *
+   * @param f
+   */
+  void SetAVFrame(AVFrame* f, AVRational timebase);
+
+  /**
+   * @brief Get frame's width in pixels
+   */
+  const int& width();
+
+  /**
+   * @brief Get frame's height in pixels
+   */
+  const int& height();
+
+  /**
+   * @brief Get frame's timestamp.
+   *
+   * This timestamp is always a rational that will equate to the time in seconds.
+   */
+  const rational& timestamp();
+
+  /**
+   * @brief Get frame's format
+   *
+   * @return
+   *
+   * Currently this will either be an AVPixelFormat (video) or an AVSampleFormat (audio).
+   */
+  const int& format();
+
+private:
+
+  void FreeChild();
+
+  AVFrame* frame_;
+  rational timestamp_;
+};
+
+using FramePtr = std::shared_ptr<Frame>;
+
+#endif // FRAME_H

decoders/pixelformatconverter.cpp

@@ -1,9 +1,14 @@
 #include "pixelformatconverter.h"
 
+extern "C" {
+  #include <libavcodec/avcodec.h>
+  #include <libswscale/swscale.h>
+}
+
 #include <QtMath>
 #include <QDebug>
 
-PixelFormatConverter olive::pix_fmt_conv;
+PixelFormatConverter* olive::pix_fmt_conv;
 
 PixelFormatConverter::PixelFormatConverter()
 {
@@ -35,9 +40,41 @@ void PixelFormatConverter::AVFrameToPipeline(uint8_t **input_buffer,
   //       (functioning as abstraction from the rest of the code) while remaining support is added.
 
 
-  // The main focus of this function is to convert to float. There's currently no intention to support converting to
-  // integer formats.
-  Q_ASSERT(output_fmt == olive::PIX_FMT_RGBA16F || output_fmt == olive::PIX_FMT_RGBA32F);
+  // Currently we don't natively support anything other than RGBA and RGBA64 so if it isn't this, we'll need to
+  // swscale it to one of those for the timebeing
+  uint8_t *converted_buffer = nullptr;
+  if (input_fmt != AV_PIX_FMT_RGBA && input_fmt != AV_PIX_FMT_RGBA64) {
+
+    converted_buffer = new uint8_t[input_linesize[0] * height];
+
+    // Determine whether this is an 8-bit image or higher
+    AVPixelFormat possible_pix_fmts[] = {
+      AV_PIX_FMT_RGBA,
+      AV_PIX_FMT_RGBA64,
+      AV_PIX_FMT_NONE
+    };
+
+    AVPixelFormat pix_fmt = avcodec_find_best_pix_fmt_of_list(possible_pix_fmts,
+                                                              input_fmt,
+                                                              1,
+                                                              nullptr);
+
+    SwsContext* sws_ctx = sws_getContext(width,
+                                         height,
+                                         input_fmt,
+                                         width,
+                                         height,
+                                         pix_fmt,
+                                         0,
+                                         nullptr,
+                                         nullptr,
+                                         nullptr);
+
+    sws_scale(sws_ctx, input_buffer, input_linesize, 0, height, &converted_buffer, input_linesize);
+
+    input_fmt = pix_fmt;
+
+  }
 
 
   // Wait til other frame conversions are done
@@ -69,6 +106,10 @@ void PixelFormatConverter::AVFrameToPipeline(uint8_t **input_buffer,
 
 
   mutex_.unlock();
+
+  if (converted_buffer != nullptr) {
+    delete [] converted_buffer;
+  }
 }
 
 int PixelFormatConverter::GetBufferSize(olive::PixelFormat format, const int &width, const int &height)
@@ -141,7 +182,7 @@ void PixFmtConvertThread::Process()
     int in_start = input_linesize_[0]*line_start_;
     int out_start = width_*line_start_;
     float f;
-    int channels = 4; // RGBA
+    int channels = 4; // FIXME RGBA magic number
 
     for (int i=0;i<line_count_;i++) {
 

decoders/pixelformatconverter.h

@@ -56,7 +56,7 @@ private:
  * replacement of FFmpeg's swscale for all pixel format conversions. It's main advantages over swscale are:
  *
  * * Support for floating-point pixel formats
- * * Better control over the conversion from YUV to RGB
+ * * Better control over the conversion from YCbCr to RGB
  * * Multithreaded by design
  *
  * In the old system, frame pixel formats were converted twice - once to RGBA32 or RGBA64 by swscale, then a second
@@ -134,7 +134,7 @@ private:
 };
 
 namespace olive {
-extern PixelFormatConverter pix_fmt_conv;
+extern PixelFormatConverter* pix_fmt_conv;
 }
 
 #endif // PIXELFORMATCONVERTER_H

global/rational.cpp

@@ -1,6 +1,6 @@
 #include "rational.h"
 
-rational::rational(const int &numerator) :
+rational::rational(const int64_t &numerator) :
   numerator_(numerator),
   denominator_(1)
 {
@@ -9,7 +9,7 @@ rational::rational(const int &numerator) :
   }
 }
 
-rational::rational(const int &numerator, const int &denominator) :
+rational::rational(const int64_t &numerator, const int64_t &denominator) :
   numerator_(numerator),
   denominator_(denominator)
 {
@@ -295,6 +295,15 @@ bool rational::operator!() const
   return !numerator_;
 }
 
+double rational::ToDouble() const
+{
+  if (denominator_ == 0) {
+    return 0;
+  } else {
+    return static_cast<double>(numerator_)/static_cast<double>(denominator_);
+  }
+}
+
 void rational::FixSigns()
 {
   // Ensures denominator is always positive (while numerator can be positive or negative)
@@ -312,7 +321,7 @@ void rational::FixSigns()
 
 void rational::Reduce()
 {
-  int d = 1;
+  int64_t d = 1;
 
   if(denominator_ != 0 && numerator_ != 0) {
     d = GreatestCommonDenominator(numerator_, denominator_);
@@ -324,12 +333,12 @@ void rational::Reduce()
   }
 }
 
-int rational::GreatestCommonDenominator(const int& x, const int& y)
+int64_t rational::GreatestCommonDenominator(const int64_t& x, const int64_t& y)
 {
   if (y == 0) {
     return x;
   } else {
-    int tmp = x % y;
+    int64_t tmp = x % y;
     return GreatestCommonDenominator(y, tmp);
   }
 }

global/rational.h

@@ -4,13 +4,16 @@
 // Adapted from https://github.com/angularadam/Qt-Class-rational used in compliance with the GNU General Public License
 
 #include <iostream>
-#include <libavformat/avformat.h>
+
+extern "C" {
+  #include <libavformat/avformat.h>
+}
 
 class rational {
 public:
   // Constructors
-  rational(const int& numerator = 0);
-  rational(const int& numerator, const int& denominator);
+  rational(const int64_t& numerator = 0);
+  rational(const int64_t& numerator, const int64_t& denominator);
   rational(const AVRational& r); // Auto-convert from an FFmpeg AVRational
   rational(const rational& r);
 
@@ -51,12 +54,12 @@ public:
   // Convert to double
   double ToDouble() const;
 private:
-  int numerator_;
-  int denominator_;
+  int64_t numerator_;
+  int64_t denominator_;
 
   void FixSigns();
   void Reduce();
-  int GreatestCommonDenominator(const int &x, const int &y);
+  int64_t GreatestCommonDenominator(const int64_t &x, const int64_t &y);
 };
 
 #endif // RATIONAL_H

nodes/nodes/nodemedia.cpp

@@ -7,7 +7,8 @@
 NodeMedia::NodeMedia(NodeGraph* c) :
   Node(c),
   media_(nullptr),
-  buffer_(c->memory_cache())
+  buffer_(c->memory_cache()),
+  decoder_(nullptr)
 {
   matrix_input_ = new NodeIO(this, "matrix", tr("Matrix"), true, false);
   matrix_input_->AddAcceptedNodeInput(olive::nodes::kMatrix);

nodes/nodes/nodemedia.h

@@ -4,6 +4,7 @@
 #include "nodes/node.h"
 #include "rendering/memorycache.h"
 #include "project/media.h"
+#include "decoders/decoder.h"
 
 /**
  * @brief The NodeMedia class
@@ -46,6 +47,9 @@ private:
   // Texture buffer
   // TODO Probable cache point
   MemoryCache::Reference buffer_;
+
+  // Decoder
+  DecoderPtr decoder_;
 };
 
 #endif // MEDIANODE_H

project/footage.h

@@ -45,18 +45,27 @@ enum VideoInterlacingMode {
 class Sequence;
 class Clip;
 class PreviewGenerator;
+class Footage;
 
 struct FootageStream {
+  // Format stream index
+  Footage* footage;
   int file_index;
+
+  // Video parameters
   int video_width;
   int video_height;
-  bool infinite_length;
   double video_frame_rate;
   int video_interlacing;
   int video_auto_interlacing;
+  bool infinite_length;
+
+  // Audio parameters
   int audio_channels;
   int audio_layout;
   int audio_frequency;
+
+  // Enabled
   bool enabled;
 
   // preview thumbnail/waveform

project/previewgenerator.cpp

@@ -127,6 +127,8 @@ void PreviewGenerator::parse_media() {
           }
         }
 
+        ms.footage = footage_;
+
         if (append) stream_list.append(ms);
       }
     }