implemented preliminary pre-cacher for audio

FFmpegDecoder now implements an "indexer" for audio, which actually just
extracts the raw PCM audio from an audio codec and saves it to disk

app/audio/sampleformat.h

@@ -28,11 +28,14 @@ namespace olive {
  */
 enum SampleFormat {
   SAMPLE_FMT_INVALID = -1,
+
   SAMPLE_FMT_U8,
   SAMPLE_FMT_S16,
   SAMPLE_FMT_S32,
+  SAMPLE_FMT_S64,
   SAMPLE_FMT_FLT,
   SAMPLE_FMT_DBL,
+
   SAMPLE_FMT_COUNT
 };
 

app/common/timecodefunctions.cpp

@@ -35,8 +35,24 @@ QString olive::timestamp_to_timecode(const int64_t &timestamp,
 {
   double timestamp_dbl = (rational(timestamp) * timebase).toDouble();
 
+  // Determine what symbol to separate frames (";" is used for drop frame, ":" is non-drop frame)
+  QString frame_token = ";";
+
   switch (display) {
-  case kTimecodeFrames:
+  case kTimecodeNonDropFrame:
+    frame_token = ":";
+
+    // Convert timestamp from drop frame to non-drop frame
+    // FIXME: There's probably a better way to do this
+    if (timebase == rational(1001, 30000)) {
+      timestamp_dbl = timestamp_dbl / (30000.0/1001.0) * 30.0;
+    } else if (timebase == rational(1001, 60000)) {
+      timestamp_dbl = timestamp_dbl / (60000.0/1001.0) * 60.0;
+    } else if (timebase == rational(1001, 24000)) {
+      timestamp_dbl = timestamp_dbl / (24000.0/1001.0) * 24.0;
+    }
+    /* fall-through */
+  case kTimecodeDropFrame:
   case kTimecodeSeconds:
   {
     QString prefix;
@@ -68,11 +84,12 @@ QString olive::timestamp_to_timecode(const int64_t &timestamp,
 
       int frames = qRound((timestamp_dbl - total_seconds) * frame_rate.toDouble());
 
-      return QString("%1%2:%3:%4;%5").arg(prefix,
-                                          padded(hours, 2),
-                                          padded(mins, 2),
-                                          padded(secs, 2),
-                                          padded(frames, 2));
+      return QString("%1%2:%3:%4%5%6").arg(prefix,
+                                           padded(hours, 2),
+                                           padded(mins, 2),
+                                           padded(secs, 2),
+                                           frame_token,
+                                           padded(frames, 2));
     }
   }
   case kFrames:

app/common/timecodefunctions.h

@@ -28,7 +28,8 @@
 namespace olive {
 
 enum TimecodeDisplay {
-  kTimecodeFrames,
+  kTimecodeDropFrame,
+  kTimecodeNonDropFrame,
   kTimecodeSeconds,
   kFrames,
   kMilliseconds

app/config/config.cpp

@@ -49,7 +49,7 @@ Config &Config::Current()
 void Config::SetDefaults()
 {
   config_map_.clear();
-  config_map_["TimecodeDisplay"] = olive::kTimecodeFrames;
+  config_map_["TimecodeDisplay"] = olive::kTimecodeNonDropFrame;
   config_map_["DefaultStillLength"] = QVariant::fromValue(rational(2));
   config_map_["HoverFocus"] = false;
   config_map_["AudioScrubbing"] = true;

app/decoder/CMakeLists.txt

@@ -23,5 +23,7 @@ set(OLIVE_SOURCES
   decoder/decoder.cpp
   decoder/frame.h
   decoder/frame.cpp
+  decoder/wave.h
+  decoder/wave.cpp
   PARENT_SCOPE
 )

app/decoder/ffmpeg/ffmpegdecoder.cpp

@@ -32,6 +32,7 @@ extern "C" {
 
 #include "common/filefunctions.h"
 #include "common/timecodefunctions.h"
+#include "decoder/wave.h"
 #include "render/pixelservice.h"
 
 FFmpegDecoder::FFmpegDecoder() :
@@ -412,7 +413,7 @@ bool FFmpegDecoder::Probe(Footage *f)
           str->set_type(Stream::kAttachment);
           break;
 
-        // We should never realistically get here, but we make an "invalid" stream just in case
+          // We should never realistically get here, but we make an "invalid" stream just in case
         default:
           str->set_type(Stream::kUnknown);
           break;
@@ -501,27 +502,104 @@ void FFmpegDecoder::Index()
   // Reset state
   Seek(0);
 
-  // This should be unnecessary, but just in case...
-  frame_index_.clear();
-
-  // Iterate through every single frame and get each timestamp
-  // NOTE: Expects no frames to have been read so far
-
   int ret = 0;
 
-  while (true) {
-    ret = GetFrame();
+  if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+    // This should be unnecessary, but just in case...
+    frame_index_.clear();
 
-    if (ret < 0) {
-      break;
+    // Iterate through every single frame and get each timestamp
+    // NOTE: Expects no frames to have been read so far
+
+    while (true) {
+      ret = GetFrame();
+
+      if (ret < 0) {
+        break;
+      } else {
+        frame_index_.append(frame_->pts);
+      }
+    }
+
+    // Save index to file
+    SaveFrameIndex();
+  } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+    // Iterate through each audio frame and extract the PCM data
+
+    WaveOutput wave_out("C:\\Users\\Matt\\AppData\\Local\\Temp\\temporary.wav", // FIXME: Hardcoded path
+                        AudioRenderingParams(avstream_->codecpar->sample_rate,
+                                             avstream_->codecpar->channel_layout,
+                                             GetNativeSampleRate(static_cast<AVSampleFormat>(avstream_->codecpar->format))));
+
+    SwrContext* resampler = nullptr;
+    AVSampleFormat src_sample_fmt = static_cast<AVSampleFormat>(avstream_->codecpar->format);
+    AVSampleFormat dst_sample_fmt;
+
+    // We don't use planar types internally, so if this is a planar format convert it now
+    if (av_sample_fmt_is_planar(src_sample_fmt)) {
+      dst_sample_fmt = av_get_packed_sample_fmt(src_sample_fmt);
+
+      // Bizarrely, swr_alloc_set_opts() uses a signed int64 while most of FFmpeg uses unsigned. We cast here.
+      int64_t channel_layout = static_cast<int64_t>(avstream_->codecpar->channel_layout);
+
+      resampler = swr_alloc_set_opts(nullptr,
+                                     channel_layout,
+                                     dst_sample_fmt,
+                                     avstream_->codecpar->sample_rate,
+                                     channel_layout,
+                                     src_sample_fmt,
+                                     avstream_->codecpar->sample_rate,
+                                     0,
+                                     nullptr);
     } else {
-      frame_index_.append(frame_->pts);
+      dst_sample_fmt = src_sample_fmt;
+    }
+
+    if (wave_out.open()) {
+      while (true) {
+        ret = GetFrame();
+
+        if (ret < 0) {
+          break;
+        } else {
+          // Calculate the byte size for this audio buffer
+          int buffer_size = av_samples_get_buffer_size(nullptr,
+                                                       avstream_->codecpar->channels,
+                                                       frame_->nb_samples,
+                                                       dst_sample_fmt,
+                                                       0); // FIXME: Documentation unclear - should this be 0 or 1?
+
+          uint8_t* resampler_output;
+
+          if (resampler != nullptr) {
+            // We must need to resample this (mainly just convert from planar to packed if necessary)
+            resampler_output = new uint8_t[buffer_size];
+            swr_convert(resampler, &resampler_output, frame_->nb_samples, frame_->data, frame_->nb_samples);
+          } else {
+            // No resampling required, we can write directly from te frame buffer
+            resampler_output = frame_->data[0];
+          }
+
+          // Write packed WAV data to the disk cache
+          wave_out.write(resampler_output, buffer_size);
+
+          // If we allocated an output for the resampler, delete it here
+          if (resampler_output != frame_->data[0]) {
+            delete [] resampler_output;
+          }
+        }
+      }
+
+      wave_out.close();
+    } else {
+      qWarning() << "Failed to open WAVE output for indexing";
+    }
+
+    if (resampler != nullptr) {
+      swr_free(&resampler);
     }
   }
 
-  // Save index to file
-  SaveFrameIndex();
-
   // Reset state
   Seek(0);
 }
@@ -529,7 +607,7 @@ void FFmpegDecoder::Index()
 QString FFmpegDecoder::GetIndexFilename()
 {
   if (!open_) {
-    qWarning() << tr("GetIndexFilename tried to run while decoder was closed");
+    qWarning() << "GetIndexFilename tried to run while decoder was closed";
     return QString();
   }
 
@@ -636,6 +714,35 @@ AVPixelFormat FFmpegDecoder::GetCompatiblePixelFormat(const AVPixelFormat &pix_f
                                            nullptr);
 }
 
+olive::SampleFormat FFmpegDecoder::GetNativeSampleRate(const AVSampleFormat &smp_fmt)
+{
+  switch (smp_fmt) {
+  case AV_SAMPLE_FMT_U8:
+    return olive::SAMPLE_FMT_U8;
+  case AV_SAMPLE_FMT_S16:
+    return olive::SAMPLE_FMT_S16;
+  case AV_SAMPLE_FMT_S32:
+    return olive::SAMPLE_FMT_S32;
+  case AV_SAMPLE_FMT_S64:
+    return olive::SAMPLE_FMT_S64;
+  case AV_SAMPLE_FMT_FLT:
+    return olive::SAMPLE_FMT_FLT;
+  case AV_SAMPLE_FMT_DBL:
+    return olive::SAMPLE_FMT_DBL;
+  case AV_SAMPLE_FMT_U8P :
+  case AV_SAMPLE_FMT_S16P:
+  case AV_SAMPLE_FMT_S32P:
+  case AV_SAMPLE_FMT_S64P:
+  case AV_SAMPLE_FMT_FLTP:
+  case AV_SAMPLE_FMT_DBLP:
+  case AV_SAMPLE_FMT_NONE:
+  case AV_SAMPLE_FMT_NB:
+    break;
+  }
+
+  return olive::SAMPLE_FMT_INVALID;
+}
+
 int64_t FFmpegDecoder::GetClosestTimestampInIndex(const int64_t &ts)
 {
   // Index now if we haven't already

app/decoder/ffmpeg/ffmpegdecoder.h

@@ -30,6 +30,7 @@ extern "C" {
 #include <QVector>
 
 #include "decoder/decoder.h"
+#include "audio/sampleformat.h"
 
 /**
  * @brief A Decoder derivative that wraps FFmpeg functions as on Olive decoder
@@ -130,6 +131,8 @@ private:
    */
   AVPixelFormat GetCompatiblePixelFormat(const AVPixelFormat& pix_fmt);
 
+  olive::SampleFormat GetNativeSampleRate(const AVSampleFormat& smp_fmt);
+
   AVFormatContext* fmt_ctx_;
   AVCodecContext* codec_ctx_;
   AVStream* avstream_;

app/decoder/wave.cpp

@@ -0,0 +1,143 @@
+#include "wave.h"
+
+const int16_t kWAVIntegerFormat = 1;
+const int16_t kWAVFloatFormat = 3;
+
+WaveOutput::WaveOutput(const QString &f,
+                       const AudioRenderingParams& params) :
+  file_(f),
+  params_(params)
+{
+}
+
+WaveOutput::~WaveOutput()
+{
+  close();
+}
+
+bool WaveOutput::open()
+{
+  data_length_ = 0;
+
+  if (file_.open(QFile::WriteOnly)) {
+    // RIFF header
+    file_.write("RIFF");
+
+    // Total file size minus RIFF and this integer (minus 8 bytes, filled in later)
+    write_int<int32_t>(&file_, 0);
+
+    // File type header
+    file_.write("WAVE");
+
+    // Begin format descriptor chunk
+    file_.write("fmt ");
+
+    // Format chunk size
+    write_int<int32_t>(&file_, 16);
+
+    // Type of format
+    switch (params_.format()) {
+    case olive::SAMPLE_FMT_U8:
+    case olive::SAMPLE_FMT_S16:
+    case olive::SAMPLE_FMT_S32:
+    case olive::SAMPLE_FMT_S64:
+      write_int<int16_t>(&file_, kWAVIntegerFormat);
+      break;
+    case olive::SAMPLE_FMT_FLT:
+    case olive::SAMPLE_FMT_DBL:
+      write_int<int16_t>(&file_, kWAVFloatFormat);
+      break;
+    case olive::SAMPLE_FMT_INVALID:
+    case olive::SAMPLE_FMT_COUNT:
+      qWarning() << "Invalid sample format for WAVE audio";
+      file_.close();
+      return false;
+    }
+
+    // Number of channels
+    write_int<int16_t>(&file_, static_cast<int16_t>(params_.channel_count()));
+
+    // Sample rate
+    write_int<int32_t>(&file_, params_.sample_rate());
+
+    // Bytes per second
+    write_int<int32_t>(&file_, (params_.sample_rate() * params_.bits_per_sample() * params_.channel_count())/8);
+
+    // Bytes per sample
+    write_int<int16_t>(&file_, static_cast<int16_t>((params_.bits_per_sample() * params_.channel_count())/8));
+
+    // Bits per sample
+    write_int<int16_t>(&file_, static_cast<int16_t>(params_.bits_per_sample()));
+
+    // Data chunk header
+    file_.write("data");
+
+    // Size of data chunk (filled in later)
+    write_int<int32_t>(&file_, 0);
+
+    return true;
+  }
+
+  return false;
+}
+
+void WaveOutput::write(const QByteArray &bytes)
+{
+  if (file_.isOpen()) {
+    file_.write(bytes);
+
+    data_length_ += bytes.size();
+  }
+}
+
+void WaveOutput::write(const char *bytes, int length)
+{
+  if (file_.isOpen()) {
+    file_.write(bytes, length);
+
+    data_length_ += length;
+  }
+}
+
+void WaveOutput::close()
+{
+  if (file_.isOpen()) {
+
+    // Write file sizes
+    file_.seek(4);
+    write_int<int32_t>(&file_, data_length_ + 36);
+
+    file_.seek(40);
+    write_int<int32_t>(&file_, data_length_);
+
+    file_.close();
+  }
+}
+
+void WaveOutput::switch_endianness(QByteArray& array)
+{
+  int half_sz = array.size()/2;
+
+  for (int i=0;i<half_sz;i++) {
+    int oppose_index = array.size() - i - 1;
+
+    char temp = array[i];
+    array[i] = array[oppose_index];
+    array[oppose_index] = temp;
+  }
+}
+
+template<typename T>
+void WaveOutput::write_int(QFile *file, T integer)
+{
+  QByteArray bytes;
+  bytes.resize(sizeof(T));
+  memcpy(bytes.data(), &integer, static_cast<size_t>(bytes.size()));
+
+  // WAV expects little-endian, so if the integer is big endian we need to switch
+  if (QSysInfo::ByteOrder == QSysInfo::BigEndian) {
+    switch_endianness(bytes);
+  }
+
+  file->write(bytes);
+}

app/decoder/wave.h

@@ -0,0 +1,39 @@
+#ifndef WAVEAUDIO_H
+#define WAVEAUDIO_H
+
+#include <QByteArray>
+#include <QFile>
+
+#include "audio/sampleformat.h"
+#include "render/audio/audioparams.h"
+
+class WaveOutput
+{
+public:
+  WaveOutput(const QString& f,
+             const AudioRenderingParams& params);
+
+  ~WaveOutput();
+
+  bool open();
+
+  void write(const QByteArray& bytes);
+  void write(const char* bytes, int length);
+
+  void close();
+
+private:
+  template<typename T>
+  void write_int(QFile* file, T integer);
+
+  void switch_endianness(QByteArray &array);
+
+  QFile file_;
+
+  AudioRenderingParams params_;
+
+  int data_length_;
+
+};
+
+#endif // WAVEAUDIO_H

app/node/input/media/media.cpp

@@ -269,9 +269,6 @@ QVariant MediaInput::Value(NodeOutput *output, const rational &in, const rationa
     transform.scale(static_cast<float>(frame_->width()), static_cast<float>(frame_->height()));
     transform.scale(0.5f, 0.5f);
 
-    //float media_size = static_cast<float>(frame_->height()) / static_cast<float>(renderer->height() * renderer->divider());
-    //transform.scale(media_size, media_size);
-
     // Use pipeline to blit using transformation matrix from input
     if (renderer->params().mode() == olive::RenderMode::kOffline) {
       olive::gl::OCIOBlit(pipeline_, ocio_texture_, false, transform);

app/render/audio/audioparams.cpp

@@ -52,7 +52,7 @@ int AudioRenderingParams::time_to_bytes(const rational &time) const
 {
   Q_ASSERT(is_valid());
 
-  return qFloor(time.toDouble() * sample_rate()) * channel_count() * sample_size();
+  return qFloor(time.toDouble() * sample_rate()) * channel_count() * bytes_per_sample();
 }
 
 int AudioRenderingParams::channel_count() const
@@ -60,7 +60,7 @@ int AudioRenderingParams::channel_count() const
   return av_get_channel_layout_nb_channels(channel_layout());
 }
 
-int AudioRenderingParams::sample_size() const
+int AudioRenderingParams::bytes_per_sample() const
 {
   switch (format_) {
   case olive::SAMPLE_FMT_U8:
@@ -71,6 +71,7 @@ int AudioRenderingParams::sample_size() const
   case olive::SAMPLE_FMT_FLT:
     return 4;
   case olive::SAMPLE_FMT_DBL:
+  case olive::SAMPLE_FMT_S64:
     return 8;
   case olive::SAMPLE_FMT_INVALID:
   case olive::SAMPLE_FMT_COUNT:
@@ -80,6 +81,11 @@ int AudioRenderingParams::sample_size() const
   return 0;
 }
 
+int AudioRenderingParams::bits_per_sample() const
+{
+  return bytes_per_sample() * 8;
+}
+
 bool AudioRenderingParams::is_valid() const
 {
   return (sample_rate() > 0

app/render/audio/audioparams.h

@@ -30,7 +30,8 @@ public:
 
   int time_to_bytes(const rational& time) const;
   int channel_count() const;
-  int sample_size() const;
+  int bytes_per_sample() const;
+  int bits_per_sample() const;
   bool is_valid() const;
 
   const olive::SampleFormat& format() const;

app/render/video/videorenderer.cpp

@@ -344,14 +344,27 @@ void VideoRendererProcessor::ThreadCallback(RenderTexturePtr texture, const rati
 
   // If the connected output is using this time, signal it to update
   if (last_time_requested_ == time) {
+
     copy_buffer_.Bind();
-    texture->Bind();
 
-    QOpenGLContext::currentContext()->functions()->glViewport(0, 0, master_texture_->width(), master_texture_->height());
+    QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions();
 
-    olive::gl::Blit(copy_pipeline_);
+    if (texture == nullptr) {
+
+      // No texture, clear the master and push it
+      f->glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
+      f->glClear(GL_COLOR_BUFFER_BIT);
+
+    } else {
+      texture->Bind();
+
+      f->glViewport(0, 0, master_texture_->width(), master_texture_->height());
+
+      olive::gl::Blit(copy_pipeline_);
+
+      texture->Release();
+    }
 
-    texture->Release();
     copy_buffer_.Release();
 
     push_time_ = time;

app/widget/viewer/viewer.cpp

@@ -281,9 +281,11 @@ void ViewerWidget::NextFrame()
 
 void ViewerWidget::GoToEnd()
 {
-  Pause();
+  if (viewer_node_ != nullptr) {
+    Pause();
 
-  qWarning() << "No end frame support yet";
+    SetTime(olive::time_to_timestamp(viewer_node_->Length(), time_base_));
+  }
 }
 
 void ViewerWidget::ShuttleLeft()