implemented audio exporting

The encoder was moved to its own thread and will transcode the PCM from the
audio renderer into the chosen codec while the video frames are still
received. The implementation isn't perfect and could use some cleaning up, but
it is functional at the moment.

app/codec/encoder.cpp

@@ -3,8 +3,8 @@
 #include "ffmpeg/ffmpegencoder.h"
 
 Encoder::Encoder(const EncodingParams &params) :
-  open_(false),
-  params_(params)
+  params_(params),
+  open_(false)
 {
 }
 
@@ -73,7 +73,43 @@ const AudioRenderingParams &EncodingParams::audio_params() const
   return audio_params_;
 }
 
-EncoderPtr Encoder::CreateFromID(const QString &id, const EncodingParams& params)
+Encoder* Encoder::CreateFromID(const QString &id, const EncodingParams& params)
 {
-  return std::make_shared<FFmpegEncoder>(params);
+  return new FFmpegEncoder(params);
+}
+
+bool Encoder::IsOpen() const
+{
+  return open_;
+}
+
+void Encoder::Open()
+{
+  if (!open_) {
+    open_ = OpenInternal();
+  }
+
+  if (open_) {
+    emit OpenSucceeded();
+  } else {
+    emit OpenFailed();
+  }
+}
+
+void Encoder::Write(FramePtr frame)
+{
+  if (open_) {
+    WriteInternal(frame);
+  }
+}
+
+void Encoder::Close()
+{
+  if (open_) {
+    CloseInternal();
+
+    open_ = false;
+  }
+
+  emit Closed();
 }

app/codec/encoder.h

@@ -42,17 +42,12 @@ private:
   AudioRenderingParams audio_params_;
 };
 
-class Encoder
+class Encoder : public QObject
 {
+  Q_OBJECT
 public:
   Encoder(const EncodingParams& params);
 
-  DISABLE_COPY_MOVE(Encoder)
-
-  virtual bool Open() = 0;
-  virtual void Write(FramePtr frame) = 0;
-  virtual void Close() = 0;
-
   /**
    * @brief Create a Encoder instance using a Encoder ID
    *
@@ -60,15 +55,33 @@ public:
    *
    * A Encoder instance or nullptr if a Decoder with this ID does not exist
    */
-  static EncoderPtr CreateFromID(const QString& id, const EncodingParams &params);
+  static Encoder *CreateFromID(const QString& id, const EncodingParams &params);
 
-protected:
   const EncodingParams& params() const;
 
-  bool open_;
+public slots:
+  void Open();
+  void Write(FramePtr frame);
+  virtual void WriteAudio(const AudioRenderingParams& pcm_info, const QString& pcm_filename) = 0;
+  void Close();
+
+signals:
+  void OpenSucceeded();
+  void OpenFailed();
+
+  void Closed();
+
+protected:
+  virtual bool OpenInternal() = 0;
+  virtual void WriteInternal(FramePtr frame) = 0;
+  virtual void CloseInternal() = 0;
+
+  bool IsOpen() const;
 
 private:
   EncodingParams params_;
+
+  bool open_;
 };
 
 #endif // ENCODER_H

app/codec/ffmpeg/ffmpegencoder.cpp

@@ -1,5 +1,7 @@
 #include "ffmpegencoder.h"
 
+#include <QFile>
+
 #include "ffmpegcommon.h"
 #include "render/pixelservice.h"
 
@@ -15,12 +17,82 @@ FFmpegEncoder::FFmpegEncoder(const EncodingParams &params) :
 {
 }
 
-bool FFmpegEncoder::Open()
+void FFmpegEncoder::WriteAudio(const AudioRenderingParams &pcm_info, const QString &pcm_filename)
 {
-  if (open_) {
-    return true;
-  }
+  QFile pcm(pcm_filename);
+  if (pcm.open(QFile::ReadOnly)) {
+    // Divide PCM stream into AVFrames
 
+    // See if the codec defines a number of samples per frame
+    int maximum_frame_samples = audio_codec_ctx_->frame_size;
+    if (!maximum_frame_samples) {
+      // If not, use another frame size
+      if (params().video_enabled()) {
+        // If we're encoding video, use enough samples to cover roughly one frame of video
+        maximum_frame_samples = params().audio_params().time_to_samples(params().video_params().time_base());
+      } else {
+        // If no video, just use an arbitary number
+        maximum_frame_samples = 256;
+      }
+    }
+
+    SwrContext* swr_ctx = swr_alloc_set_opts(nullptr,
+                                             audio_codec_ctx_->channel_layout,
+                                             audio_codec_ctx_->sample_fmt,
+                                             audio_codec_ctx_->sample_rate,
+                                             pcm_info.channel_layout(),
+                                             FFmpegCommon::GetFFmpegSampleFormat(pcm_info.format()),
+                                             pcm_info.sample_rate(),
+                                             0,
+                                             nullptr);
+
+    swr_init(swr_ctx);
+
+    // Loop through PCM queueing write events
+    AVFrame* frame = av_frame_alloc();
+
+    // Set up frame
+    frame->channel_layout = audio_codec_ctx_->channel_layout;
+    frame->nb_samples = maximum_frame_samples;
+    frame->format = audio_codec_ctx_->sample_fmt;
+
+    // Allocate its buffers
+    av_frame_get_buffer(frame, 0);
+    int sample_counter = 0;
+
+    while (!pcm.atEnd()) {
+      int samples_needed = static_cast<int>(frame->nb_samples + swr_get_delay(swr_ctx, pcm_info.sample_rate()));
+
+      QByteArray input_data;
+      input_data = pcm.read(pcm_info.samples_to_bytes(samples_needed));
+      const char* input_data_array = input_data.constData();
+      samples_needed = pcm_info.bytes_to_samples(input_data.size());
+
+      // Use swresample to convert the data into the correct format/linesize
+      int converted = swr_convert(swr_ctx,
+                                  frame->data,
+                                  maximum_frame_samples,
+                                  reinterpret_cast<const uint8_t**>(&input_data_array),
+                                  samples_needed);
+
+      //qDebug() << "swr_convert returned" << converted << "samples";
+      frame->pts = sample_counter;
+
+      sample_counter += converted;
+
+      WriteAVFrame(frame, audio_codec_ctx_, audio_stream_);
+    }
+
+    av_frame_free(&frame);
+
+    swr_free(&swr_ctx);
+
+    pcm.close();
+  }
+}
+
+bool FFmpegEncoder::OpenInternal()
+{
   int error_code;
 
   // Convert QString to C string that FFmpeg expects
@@ -32,7 +104,7 @@ bool FFmpegEncoder::Open()
 
   // Check error code
   if (error_code < 0) {
-    FFmpegError(error_code);
+    FFmpegError("Failed to allocate output context", error_code);
     return false;
   }
 
@@ -42,8 +114,6 @@ bool FFmpegEncoder::Open()
       return false;
     }
 
-
-
     // This is the format we will expect frames received in Write() to be in
     olive::PixelFormat native_pixel_fmt = params().video_params().format();
 
@@ -81,112 +151,70 @@ bool FFmpegEncoder::Open()
   // Open output file for writing
   error_code = avio_open(&fmt_ctx_->pb, filename_c_str, AVIO_FLAG_WRITE);
   if (error_code < 0) {
-    FFmpegError(error_code);
+    FFmpegError("Failed to open IO context", error_code);
     return false;
   }
 
   // Write header
   error_code = avformat_write_header(fmt_ctx_, nullptr);
   if (error_code < 0) {
-    FFmpegError(error_code);
+    FFmpegError("Failed to write format header", error_code);
     return false;
   }
 
-  open_ = true;
-
   return true;
 }
 
-void FFmpegEncoder::Write(FramePtr frame)
+void FFmpegEncoder::WriteInternal(FramePtr frame)
 {
   AVFrame* encoded_frame = av_frame_alloc();
-  AVPacket* pkt = av_packet_alloc();
 
   int error_code;
+  const char* input_data;
+  int input_linesize;
 
-  AVCodecContext* codec_ctx;
-  AVStream* stream;
+  // Frame must be video
+  encoded_frame->width = frame->width();
+  encoded_frame->height = frame->height();
+  encoded_frame->format = video_codec_ctx_->pix_fmt;
 
-  if (frame->width() > 0) {
-    // Frame must be video
-    encoded_frame->width = frame->width();
-    encoded_frame->height = frame->height();
-    encoded_frame->format = video_codec_ctx_->pix_fmt;
-
-    error_code = av_frame_get_buffer(encoded_frame, 0);
-    if (error_code < 0) {
-      FFmpegError(error_code);
-      goto fail;
-    }
-
-    // We may need to convert this frame to a frame that swscale will understand
-    if (frame->format() != video_conversion_fmt_) {
-      frame = PixelService::ConvertPixelFormat(frame, video_conversion_fmt_);
-    }
-
-    // Use swscale context to convert formats/linesizes
-    const char* input_data = frame->const_data();
-    int input_linesize = frame->width() * PixelService::BytesPerPixel(video_conversion_fmt_);
-    error_code = sws_scale(video_scale_ctx_,
-                           reinterpret_cast<const uint8_t**>(&input_data),
-                           &input_linesize,
-                           0,
-                           frame->height(),
-                           encoded_frame->data,
-                           encoded_frame->linesize);
-    if (error_code < 0) {
-      goto fail;
-    }
-
-    codec_ctx = video_codec_ctx_;
-    stream = video_stream_;
-  } else {
-    // Frame must be audio
-    codec_ctx = audio_codec_ctx_;
-    stream = audio_stream_;
-  }
-
-  encoded_frame->pts = qRound(frame->timestamp().toDouble() / av_q2d(codec_ctx->time_base));
-
-  // Send raw frame to the encoder
-  error_code = avcodec_send_frame(codec_ctx, encoded_frame);
+  error_code = av_frame_get_buffer(encoded_frame, 0);
   if (error_code < 0) {
-    FFmpegError(error_code);
+    FFmpegError("Failed to create AVFrame buffer", error_code);
     goto fail;
   }
 
-  // Retrieve packets from encoder
-  while (error_code >= 0) {
-    error_code = avcodec_receive_packet(codec_ctx, pkt);
-
-    // EAGAIN just means the encoder wants another frame before encoding
-    if (error_code == AVERROR(EAGAIN)) {
-      break;
-    } else if (error_code < 0) {
-      FFmpegError(error_code);
-      goto fail;
-    }
-
-    // Set packet stream index
-    pkt->stream_index = stream->index;
-
-    av_packet_rescale_ts(pkt, codec_ctx->time_base, stream->time_base);
-
-    // Write packet to file
-    av_interleaved_write_frame(fmt_ctx_, pkt);
-
-    // Unref packet in case we're getting another
-    av_packet_unref(pkt);
+  // We may need to convert this frame to a frame that swscale will understand
+  if (frame->format() != video_conversion_fmt_) {
+    frame = PixelService::ConvertPixelFormat(frame, video_conversion_fmt_);
   }
 
+  // Use swscale context to convert formats/linesizes
+  input_data = frame->const_data();
+  input_linesize = frame->width() * PixelService::BytesPerPixel(video_conversion_fmt_);
+  error_code = sws_scale(video_scale_ctx_,
+                         reinterpret_cast<const uint8_t**>(&input_data),
+                         &input_linesize,
+                         0,
+                         frame->height(),
+                         encoded_frame->data,
+                         encoded_frame->linesize);
+  if (error_code < 0) {
+    FFmpegError("Failed to scale frame", error_code);
+    goto fail;
+  }
+
+  encoded_frame->pts = qRound(frame->timestamp().toDouble() / av_q2d(video_codec_ctx_->time_base));
+
+  WriteAVFrame(encoded_frame, video_codec_ctx_, video_stream_);
+
 fail:
-  av_packet_free(&pkt);
   av_frame_free(&encoded_frame);
 }
 
-void FFmpegEncoder::Close()
+void FFmpegEncoder::CloseInternal()
 {
-  if (open_) {
+  if (IsOpen()) {
     // Flush encoders
     FlushEncoders();
 
@@ -217,14 +245,54 @@ void FFmpegEncoder::Close()
   }
 }
 
-void FFmpegEncoder::FFmpegError(int error_code)
+void FFmpegEncoder::FFmpegError(const char* context, int error_code)
 {
-  char err[1024];
-  av_strerror(error_code, err, 1024);
+  char err[128];
+  av_strerror(error_code, err, 128);
 
-  Error(QStringLiteral("Error encoding %1 - %2 %3").arg(params().filename(),
-                                                        QString::number(error_code),
-                                                        err));
+  Error(QStringLiteral("%1 for %2 - %3 %4").arg(context,
+                                                params().filename(),
+                                                QString::number(error_code),
+                                                err));
+}
+
+void FFmpegEncoder::WriteAVFrame(AVFrame *frame, AVCodecContext* codec_ctx, AVStream* stream)
+{
+  // Send raw frame to the encoder
+  int error_code = avcodec_send_frame(codec_ctx, frame);
+  if (error_code < 0) {
+    FFmpegError("Failed to send frame to encoder", error_code);
+    return;
+  }
+
+  AVPacket* pkt = av_packet_alloc();
+
+  // Retrieve packets from encoder
+  while (error_code >= 0) {
+    error_code = avcodec_receive_packet(codec_ctx, pkt);
+
+    // EAGAIN just means the encoder wants another frame before encoding
+    if (error_code == AVERROR(EAGAIN)) {
+      break;
+    } else if (error_code < 0) {
+      FFmpegError("Failed to receive packet from decoder", error_code);
+      goto fail;
+    }
+
+    // Set packet stream index
+    pkt->stream_index = stream->index;
+
+    av_packet_rescale_ts(pkt, codec_ctx->time_base, stream->time_base);
+
+    // Write packet to file
+    av_interleaved_write_frame(fmt_ctx_, pkt);
+
+    // Unref packet in case we're getting another
+    av_packet_unref(pkt);
+  }
+
+fail:
+  av_packet_free(&pkt);
 }
 
 bool FFmpegEncoder::InitializeStream(AVMediaType type, AVStream** stream_ptr, AVCodecContext** codec_ctx_ptr, const QString& codec)
@@ -314,14 +382,14 @@ bool FFmpegEncoder::SetupCodecContext(AVStream* stream, AVCodecContext* codec_ct
   // Try to open encoder
   error_code = avcodec_open2(codec_ctx, codec, &codec_opts);
   if (error_code < 0) {
-    FFmpegError(error_code);
+    FFmpegError("Failed to open encoder", error_code);
     return false;
   }
 
   // Copy context settings to codecpar object
   error_code = avcodec_parameters_from_context(stream->codecpar, codec_ctx);
   if (error_code < 0) {
-    FFmpegError(error_code);
+    FFmpegError("Failed to copy codec parameters to stream", error_code);
     return false;
   }
 

app/codec/ffmpeg/ffmpegencoder.h

@@ -11,12 +11,17 @@ extern "C" {
 
 class FFmpegEncoder : public Encoder
 {
+  Q_OBJECT
 public:
   FFmpegEncoder(const EncodingParams &params);
 
-  virtual bool Open() override;
-  virtual void Write(FramePtr frame) override;
-  virtual void Close() override;
+public slots:
+  virtual void WriteAudio(const AudioRenderingParams& pcm_info, const QString& pcm_filename) override;
+
+protected:
+  virtual bool OpenInternal() override;
+  virtual void WriteInternal(FramePtr frame) override;
+  virtual void CloseInternal() override;
 
 private:
   /**
@@ -36,7 +41,9 @@ private:
    *
    * @param error_code
    */
-  void FFmpegError(int error_code);
+  void FFmpegError(const char *context, int error_code);
+
+  void WriteAVFrame(AVFrame* frame, AVCodecContext *codec_ctx, AVStream *stream);
 
   bool InitializeStream(enum AVMediaType type, AVStream** stream, AVCodecContext** codec_ctx, const QString& codec);
   bool InitializeCodecContext(AVStream** stream, AVCodecContext** codec_ctx, AVCodec* codec);

app/codec/frame.h

@@ -140,4 +140,6 @@ private:
 
 };
 
+Q_DECLARE_METATYPE(FramePtr)
+
 #endif // FRAME_H