began audio conform code in ffmpegdecoder

For accuracy, whenever we need to perform an audio resample, we need to do it
in advance. This is in a process called "conforming" and this commit introduces
the framework by which the decoder can automatically conform an audio stream
to arbitrary parameters for accurate rendering.

app/decoder/decoder.cpp

@@ -134,3 +134,10 @@ DecoderPtr Decoder::CreateFromID(const QString &id)
 
   return nullptr;
 }
+
+void Decoder::Conform(const AudioRenderingParams &params)
+{
+  Q_UNUSED(params)
+  qCritical() << "Conform called on an audio decoder that does not have a handler for it:" << id();
+  abort();
+}

app/decoder/decoder.h

@@ -180,6 +180,19 @@ public:
    */
   static DecoderPtr CreateFromID(const QString& id);
 
+  /**
+   * @brief Conform an audio stream to match certain parameters (audio only)
+   *
+   * Resamples and converts the currently open audio to match the params. If the audio doesn't need conforming (e.g.
+   * audio params already match or a conformed match already exists), this function will return immediately. Otherwise
+   * it will block the calling thread until the conform is complete. This function should therefore only be called
+   * from a background render thread.
+   *
+   * All audio decoders must override this. It's not pure since video decoders don't need to use this, but default
+   * behavior will abort since it should never be called.
+   */
+  virtual void Conform(const AudioRenderingParams& params);
+
 protected:
   bool open_;
 

app/decoder/ffmpeg/ffmpegdecoder.cpp

@@ -328,6 +328,68 @@ int64_t FFmpegDecoder::GetTimestampFromTime(const rational &time)
   return target_ts;
 }
 
+void FFmpegDecoder::Conform(const AudioRenderingParams &params)
+{
+  if (avstream_->codecpar->codec_type != AVMEDIA_TYPE_AUDIO) {
+    // Nothing to be done
+    return;
+  }
+
+  if (!LoadIndex()) {
+    Index();
+  }
+
+  // Get indexed WAV file
+  WaveInput input(GetIndexFilename());
+
+  if (input.open()) {
+    // If the parameters are equal, nothing to be done
+    if (input.params() == params) {
+      input.close();
+      return;
+    }
+
+    // Otherwise, let's start converting the format
+
+    // Generate destination filename for this conversion to see if it exists
+    QString conformed_fn = GetConformedFilename(params);
+
+    if (QFileInfo::exists(conformed_fn)) {
+      // We must have already conformed this format
+      input.close();
+      return;
+    }
+
+    // Set up resampler
+    SwrContext* resampler = swr_alloc_set_opts(nullptr,
+                                               static_cast<int64_t>(params.channel_layout()),
+                                               GetFFmpegSampleFormat(params.format()),
+                                               params.sample_rate(),
+                                               static_cast<int64_t>(input.params().channel_layout()),
+                                               GetFFmpegSampleFormat(input.params().format()),
+                                               input.params().sample_rate(),
+                                               0,
+                                               nullptr);
+
+    WaveOutput conformed_output(conformed_fn, params);
+    if (!conformed_output.open()) {
+      qWarning() << "Failed to open conformed output:" << conformed_fn;
+      input.close();
+      return;
+    }
+
+    //
+    //swr_convert(resampler, input.)
+
+    // Clean up
+    swr_free(&resampler);
+    conformed_output.close();
+    input.close();
+  } else {
+    qWarning() << "Failed to conform file:" << stream()->footage()->filename();
+  }
+}
+
 bool FFmpegDecoder::Probe(Footage *f)
 {
   if (open_) {
@@ -530,6 +592,20 @@ QString FFmpegDecoder::GetIndexFilename()
       .append(QString::number(avstream_->index));
 }
 
+QString FFmpegDecoder::GetConformedFilename(const AudioRenderingParams &params)
+{
+  QString index_fn = GetIndexFilename();
+
+  index_fn.append('.');
+  index_fn.append(QString::number(params.sample_rate()));
+  index_fn.append('.');
+  index_fn.append(QString::number(params.format()));
+  index_fn.append('.');
+  index_fn.append(QString::number(params.channel_layout()));
+
+  return index_fn;
+}
+
 bool FFmpegDecoder::LoadIndex()
 {
   switch (avstream_->codecpar->codec_type) {
@@ -620,7 +696,7 @@ void FFmpegDecoder::IndexAudio(AVPacket *pkt, AVFrame *frame)
   WaveOutput wave_out(GetIndexFilename(),
                       AudioRenderingParams(avstream_->codecpar->sample_rate,
                                            channel_layout,
-                                           GetNativeSampleRate(dst_sample_fmt)));
+                                           GetNativeSampleFormat(dst_sample_fmt)));
 
   int ret;
 
@@ -782,7 +858,7 @@ AVPixelFormat FFmpegDecoder::GetCompatiblePixelFormat(const AVPixelFormat &pix_f
                                            nullptr);
 }
 
-SampleFormat FFmpegDecoder::GetNativeSampleRate(const AVSampleFormat &smp_fmt)
+SampleFormat FFmpegDecoder::GetNativeSampleFormat(const AVSampleFormat &smp_fmt)
 {
   switch (smp_fmt) {
   case AV_SAMPLE_FMT_U8:
@@ -811,6 +887,29 @@ SampleFormat FFmpegDecoder::GetNativeSampleRate(const AVSampleFormat &smp_fmt)
   return SAMPLE_FMT_INVALID;
 }
 
+AVSampleFormat FFmpegDecoder::GetFFmpegSampleFormat(const SampleFormat &smp_fmt)
+{
+  switch (smp_fmt) {
+  case SAMPLE_FMT_U8:
+    return AV_SAMPLE_FMT_U8;
+  case SAMPLE_FMT_S16:
+    return AV_SAMPLE_FMT_S16;
+  case SAMPLE_FMT_S32:
+    return AV_SAMPLE_FMT_S32;
+  case SAMPLE_FMT_S64:
+    return AV_SAMPLE_FMT_S64;
+  case SAMPLE_FMT_FLT:
+    return AV_SAMPLE_FMT_FLT;
+  case SAMPLE_FMT_DBL:
+    return AV_SAMPLE_FMT_DBL;
+  case SAMPLE_FMT_INVALID:
+  case SAMPLE_FMT_COUNT:
+    break;
+  }
+
+  return AV_SAMPLE_FMT_NONE;
+}
+
 int FFmpegDecoder::CalculatePlaneHeight(int frame_height, const AVPixelFormat &format, int plane)
 {
   // FIXME: This seems dumb, but I can't find any FFmpeg function that returns this information

app/decoder/ffmpeg/ffmpegdecoder.h

@@ -54,6 +54,8 @@ public:
 
   virtual int64_t GetTimestampFromTime(const rational& time) override;
 
+  virtual void Conform(const AudioRenderingParams& params) override;
+
 private:
   /**
    * @brief Handle an error
@@ -100,11 +102,14 @@ private:
    * @brief Returns the filename for the index
    *
    * Retrieves the absolute filename of the index file for this stream. Decoder must be open for this to work correctly.
-   *
-   * @return
    */
   QString GetIndexFilename();
 
+  /**
+   * @brief Get the destination filename of an audio stream conformed to a set of parameters
+   */
+  QString GetConformedFilename(const AudioRenderingParams &params);
+
   /**
    * @brief Used internally to load a frame index into frame_index_
    *
@@ -132,7 +137,9 @@ private:
    */
   AVPixelFormat GetCompatiblePixelFormat(const AVPixelFormat& pix_fmt);
 
-  SampleFormat GetNativeSampleRate(const AVSampleFormat& smp_fmt);
+  SampleFormat GetNativeSampleFormat(const AVSampleFormat& smp_fmt);
+
+  AVSampleFormat GetFFmpegSampleFormat(const SampleFormat& smp_fmt);
 
   int CalculatePlaneHeight(int frame_height, const AVPixelFormat& format, int plane);
 

app/decoder/waveinput.cpp

@@ -124,7 +124,8 @@ bool WaveInput::open()
     return false;
   }
 
-  data_position_ = file_.pos() + 4;
+  data_stream >> data_size_;
+  data_position_ = file_.pos();
 
   return true;
 }
@@ -166,6 +167,11 @@ void WaveInput::close()
   }
 }
 
+int WaveInput::sample_count()
+{
+  return params_.bytes_to_samples(static_cast<int>(data_size_));
+}
+
 bool WaveInput::find_str(QFile *f, const char *str)
 {
   qint64 pos = f->pos();

app/decoder/waveinput.h

@@ -26,6 +26,8 @@ public:
 
   void close();
 
+  int sample_count();
+
 private:
   bool find_str(QFile* f, const char* str);
 
@@ -34,6 +36,8 @@ private:
   QFile file_;
 
   qint64 data_position_;
+
+  quint32 data_size_;
 };
 
 #endif // WAVEINPUT_H

app/render/audioparams.cpp

@@ -83,6 +83,13 @@ int AudioRenderingParams::samples_to_bytes(const int &samples) const
   return samples * channel_count() * bytes_per_sample_per_channel();
 }
 
+int AudioRenderingParams::bytes_to_samples(const int &bytes) const
+{
+  Q_ASSERT(is_valid());
+
+  return bytes / (channel_count() * bytes_per_sample_per_channel());
+}
+
 int AudioRenderingParams::channel_count() const
 {
   return av_get_channel_layout_nb_channels(channel_layout());

app/render/audioparams.h

@@ -31,6 +31,7 @@ public:
   int time_to_bytes(const rational& time) const;
   int time_to_samples(const rational& time) const;
   int samples_to_bytes(const int& samples) const;
+  int bytes_to_samples(const int &bytes) const;
   int channel_count() const;
   int bytes_per_sample_per_channel() const;
   int bits_per_sample() const;