miscellaneous other changes to make audio rendering work

Many changes were made throughout the codebase to support audio, these are
most of the small changes necessary.

The audio support still is not perfect. I still need to write in resampling
support. After that it should work correctly with all audio types.

app/audio/CMakeLists.txt

@@ -21,5 +21,6 @@ set(OLIVE_SOURCES
   audio/audiomanager.h
   audio/audiomanager.cpp
   audio/sampleformat.h
+  audio/sampleformat.cpp
   PARENT_SCOPE
 )

app/audio/audiohybriddevice.cpp

@@ -48,7 +48,11 @@ void AudioHybridDevice::Stop()
 {
   // Whatever is happening, stop it
   pushed_samples_.clear();
-  device_ = nullptr;
+
+  if (device_ != nullptr) {
+    device_->close();
+    device_ = nullptr;
+  }
 }
 
 void AudioHybridDevice::ConnectDevice(QIODevice *device)

app/audio/sampleformat.cpp

@@ -0,0 +1 @@
+#include "sampleformat.h"

app/audio/sampleformat.h

@@ -21,11 +21,6 @@
 #ifndef SAMPLEFORMAT_H
 #define SAMPLEFORMAT_H
 
-namespace olive {
-
-/**
- * @brief Olive's internal supported sample formats
- */
 enum SampleFormat {
   SAMPLE_FMT_INVALID = -1,
 
@@ -39,6 +34,4 @@ enum SampleFormat {
   SAMPLE_FMT_COUNT
 };
 
-}
-
 #endif // SAMPLEFORMAT_H

app/decoder/wave.cpp

@@ -1,143 +0,0 @@
-#include "wave.h"
-
-const int16_t kWAVIntegerFormat = 1;
-const int16_t kWAVFloatFormat = 3;
-
-WaveOutput::WaveOutput(const QString &f,
-                       const AudioRenderingParams& params) :
-  file_(f),
-  params_(params)
-{
-}
-
-WaveOutput::~WaveOutput()
-{
-  close();
-}
-
-bool WaveOutput::open()
-{
-  data_length_ = 0;
-
-  if (file_.open(QFile::WriteOnly)) {
-    // RIFF header
-    file_.write("RIFF");
-
-    // Total file size minus RIFF and this integer (minus 8 bytes, filled in later)
-    write_int<int32_t>(&file_, 0);
-
-    // File type header
-    file_.write("WAVE");
-
-    // Begin format descriptor chunk
-    file_.write("fmt ");
-
-    // Format chunk size
-    write_int<int32_t>(&file_, 16);
-
-    // Type of format
-    switch (params_.format()) {
-    case olive::SAMPLE_FMT_U8:
-    case olive::SAMPLE_FMT_S16:
-    case olive::SAMPLE_FMT_S32:
-    case olive::SAMPLE_FMT_S64:
-      write_int<int16_t>(&file_, kWAVIntegerFormat);
-      break;
-    case olive::SAMPLE_FMT_FLT:
-    case olive::SAMPLE_FMT_DBL:
-      write_int<int16_t>(&file_, kWAVFloatFormat);
-      break;
-    case olive::SAMPLE_FMT_INVALID:
-    case olive::SAMPLE_FMT_COUNT:
-      qWarning() << "Invalid sample format for WAVE audio";
-      file_.close();
-      return false;
-    }
-
-    // Number of channels
-    write_int<int16_t>(&file_, static_cast<int16_t>(params_.channel_count()));
-
-    // Sample rate
-    write_int<int32_t>(&file_, params_.sample_rate());
-
-    // Bytes per second
-    write_int<int32_t>(&file_, params_.samples_to_bytes(params_.sample_rate()));
-
-    // Bytes per sample
-    write_int<int16_t>(&file_, static_cast<int16_t>(params_.samples_to_bytes(1)));
-
-    // Bits per sample per channel
-    write_int<int16_t>(&file_, static_cast<int16_t>(params_.bits_per_sample()));
-
-    // Data chunk header
-    file_.write("data");
-
-    // Size of data chunk (filled in later)
-    write_int<int32_t>(&file_, 0);
-
-    return true;
-  }
-
-  return false;
-}
-
-void WaveOutput::write(const QByteArray &bytes)
-{
-  if (file_.isOpen()) {
-    file_.write(bytes);
-
-    data_length_ += bytes.size();
-  }
-}
-
-void WaveOutput::write(const char *bytes, int length)
-{
-  if (file_.isOpen()) {
-    file_.write(bytes, length);
-
-    data_length_ += length;
-  }
-}
-
-void WaveOutput::close()
-{
-  if (file_.isOpen()) {
-
-    // Write file sizes
-    file_.seek(4);
-    write_int<int32_t>(&file_, data_length_ + 36);
-
-    file_.seek(40);
-    write_int<int32_t>(&file_, data_length_);
-
-    file_.close();
-  }
-}
-
-void WaveOutput::switch_endianness(QByteArray& array)
-{
-  int half_sz = array.size()/2;
-
-  for (int i=0;i<half_sz;i++) {
-    int oppose_index = array.size() - i - 1;
-
-    char temp = array[i];
-    array[i] = array[oppose_index];
-    array[oppose_index] = temp;
-  }
-}
-
-template<typename T>
-void WaveOutput::write_int(QFile *file, T integer)
-{
-  QByteArray bytes;
-  bytes.resize(sizeof(T));
-  memcpy(bytes.data(), &integer, static_cast<size_t>(bytes.size()));
-
-  // WAV expects little-endian, so if the integer is big endian we need to switch
-  if (QSysInfo::ByteOrder == QSysInfo::BigEndian) {
-    switch_endianness(bytes);
-  }
-
-  file->write(bytes);
-}

app/decoder/wave.h

@@ -1,39 +0,0 @@
-#ifndef WAVEAUDIO_H
-#define WAVEAUDIO_H
-
-#include <QByteArray>
-#include <QFile>
-
-#include "audio/sampleformat.h"
-#include "render/audioparams.h"
-
-class WaveOutput
-{
-public:
-  WaveOutput(const QString& f,
-             const AudioRenderingParams& params);
-
-  ~WaveOutput();
-
-  bool open();
-
-  void write(const QByteArray& bytes);
-  void write(const char* bytes, int length);
-
-  void close();
-
-private:
-  template<typename T>
-  void write_int(QFile* file, T integer);
-
-  void switch_endianness(QByteArray &array);
-
-  QFile file_;
-
-  AudioRenderingParams params_;
-
-  int data_length_;
-
-};
-
-#endif // WAVEAUDIO_H

app/node/block/block.cpp

@@ -110,6 +110,7 @@ QVariant Block::Value(NodeOutput *output)
 void Block::EdgeAddedSlot(NodeEdgePtr edge)
 {
   if (edge->input() == previous_input()) {
+    // FIXME: No protection for if this connection is not a node
     static_cast<Block*>(edge->output()->parent())->next_ = this;
 
     // The blocks surrounding this one have changed, we need to Refresh()
@@ -123,6 +124,7 @@ void Block::EdgeAddedSlot(NodeEdgePtr edge)
 void Block::EdgeRemovedSlot(NodeEdgePtr edge)
 {
   if (edge->input() == previous_input()) {
+    // FIXME: No protection for if this connection is not a node
     static_cast<Block*>(edge->output()->parent())->next_ = nullptr;
 
     // The blocks surrounding this one have changed, we need to Refresh()

app/node/input/media/audio/audio.cpp

@@ -26,22 +26,17 @@ QString AudioInput::Description()
   return tr("Import an audio footage stream.");
 }
 
-QVariant AudioInput::Value(NodeOutput *)
+NodeOutput *AudioInput::samples_output()
 {
-  /*if (output == samples_output_) {
-    // Make sure decoder is set up
-    if (!SetupDecoder()) {
-      return 0;
-    }
+  return samples_output_;
+}
 
-    // Retrieve audio samples from decoder
-    frame_ = decoder_->Retrieve(in, out - in);
+QVariant AudioInput::Value(NodeOutput *output)
+{
+  if (output == samples_output_) {
+    // Simple passthrough from footage input
+    return footage_input_->value();
+  }
 
-    QByteArray samples;
-    samples.resize(frame_->audio_params().samples_to_bytes(frame_->sample_count()));
-    memcpy(samples.data(), frame_->data(), static_cast<size_t>(samples.size()));
-    return samples;
-  }*/
-
-  return 0;
+  return MediaInput::Value(output);
 }

app/node/input/media/audio/audio.h

@@ -13,6 +13,8 @@ public:
   virtual QString Category() override;
   virtual QString Description() override;
 
+  NodeOutput* samples_output();
+
 protected:
   virtual QVariant Value(NodeOutput* output) override;
 

app/node/input/media/video/video.h

@@ -27,7 +27,6 @@ public:
   //virtual void Hash(QCryptographicHash *hash, NodeOutput* from, const rational &time) override;
 
 protected:
-  //virtual QVariant Value(NodeOutput* output, const rational& in, const rational& out) override;
 
 private:
   NodeInput* matrix_input_;

app/node/output.h

@@ -48,7 +48,7 @@ public:
   void drop_cached_values();
 
 private:
-  QMap<TimeRange, QVariant> cached_values_;
+  QHash<TimeRange, QVariant> cached_values_;
 
 };
 

app/project/item/sequence/sequence.cpp

@@ -85,6 +85,7 @@ void Sequence::add_default_nodes()
 
   // Update the timebase on these nodes
   set_video_params(video_params_);
+  set_audio_params(audio_params_);
 }
 
 Item::Type Sequence::type() const

app/widget/playbackcontrols/playbackcontrols.cpp

@@ -132,13 +132,12 @@ void PlaybackControls::SetTimebase(const rational &r)
 
 void PlaybackControls::SetTime(const int64_t &r)
 {
-  if (time_base_.isNull()) {
-    return;
-  }
+  SetTimeLabelInternal(cur_tc_lbl_, r);
+}
 
-  cur_tc_lbl_->setText(olive::timestamp_to_timecode(r,
-                                                    time_base_,
-                                                    olive::CurrentTimecodeDisplay()));
+void PlaybackControls::SetEndTime(const int64_t &r)
+{
+  SetTimeLabelInternal(end_tc_lbl_, r);
 }
 
 void PlaybackControls::ShowPauseButton()
@@ -170,3 +169,14 @@ void PlaybackControls::UpdateIcons()
   next_frame_btn_->setIcon(olive::icon::NextFrame);
   go_to_end_btn_->setIcon(olive::icon::GoToEnd);
 }
+
+void PlaybackControls::SetTimeLabelInternal(QLabel* label, const int64_t& time)
+{
+  if (time_base_.isNull()) {
+    return;
+  }
+
+  label->setText(olive::timestamp_to_timecode(time,
+                                              time_base_,
+                                              olive::CurrentTimecodeDisplay()));
+}

app/widget/playbackcontrols/playbackcontrols.h

@@ -49,6 +49,8 @@ public:
 public slots:
   void SetTime(const int64_t &r);
 
+  void SetEndTime(const int64_t &r);
+
   void ShowPauseButton();
 
   void ShowPlayButton();
@@ -90,6 +92,8 @@ protected:
 private:
   void UpdateIcons();
 
+  void SetTimeLabelInternal(QLabel *label, const int64_t &time);
+
   QWidget* lower_left_container_;
   QWidget* lower_right_container_;
 

app/widget/timelinewidget/tool/import.cpp

@@ -28,6 +28,7 @@
 #include "core.h"
 #include "node/distort/transform/transform.h"
 #include "node/color/opacity/opacity.h"
+#include "node/input/media/audio/audio.h"
 #include "node/input/media/video/video.h"
 
 TrackType TrackTypeFromStreamType(Stream::Type stream_type)
@@ -205,10 +206,6 @@ void TimelineWidget::ImportTool::DragLeave(QDragLeaveEvent* event)
 void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event)
 {
   if (parent()->HasGhosts()) {
-    // We use QObject as the parent for the nodes we create. If there is no TimelineOutput node, this object going out
-    // of scope will delete the nodes. If there is, they'll become parents of the NodeGraph instead
-    QObject node_memory_manager;
-
     QUndoCommand* command = new QUndoCommand();
 
     QVector<Block*> block_items(parent()->ghost_items_.size());
@@ -216,27 +213,37 @@ void TimelineWidget::ImportTool::DragDrop(TimelineViewMouseEvent *event)
     for (int i=0;i<parent()->ghost_items_.size();i++) {
       TimelineViewGhostItem* ghost = parent()->ghost_items_.at(i);
 
-      ClipBlock* clip = new ClipBlock();
-      VideoInput* media = new VideoInput();
-      //TransformDistort* transform = new TransformDistort();
-      //OpacityNode* opacity = new OpacityNode();
-
-      // Set parents to node_memory_manager in case no TimelineOutput receives this signal
-      clip->setParent(&node_memory_manager);
-      media->setParent(&node_memory_manager);
-      //transform->setParent(&node_memory_manager);
-      //opacity->setParent(&node_memory_manager);
-
       StreamPtr footage_stream = ghost->data(TimelineViewGhostItem::kAttachedFootage).value<StreamPtr>();
-      media->SetFootage(footage_stream);
 
+      ClipBlock* clip = new ClipBlock();
       clip->set_length(ghost->Length());
       clip->set_block_name(footage_stream->footage()->name());
 
-      //NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input());
-      //NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input());
-      //NodeParam::ConnectEdge(transform->matrix_output(), media->matrix_input());
-      NodeParam::ConnectEdge(media->texture_output(), clip->texture_input());
+      switch (footage_stream->type()) {
+      case Stream::kVideo:
+      {
+        VideoInput* video_input = new VideoInput();
+        video_input->SetFootage(footage_stream);
+        NodeParam::ConnectEdge(video_input->texture_output(), clip->texture_input());
+
+        TransformDistort* transform = new TransformDistort();
+        NodeParam::ConnectEdge(transform->matrix_output(), video_input->matrix_input());
+
+        //OpacityNode* opacity = new OpacityNode();
+        //NodeParam::ConnectEdge(opacity->texture_output(), clip->texture_input());
+        //NodeParam::ConnectEdge(media->texture_output(), opacity->texture_input());
+        break;
+      }
+      case Stream::kAudio:
+      {
+        AudioInput* audio_input = new AudioInput();
+        audio_input->SetFootage(footage_stream);
+        NodeParam::ConnectEdge(audio_input->samples_output(), clip->texture_input());
+        break;
+      }
+      default:
+        break;
+      }
 
       if (event->GetModifiers() & Qt::ControlModifier) {
         //emit parent()->RequestInsertBlockAtTime(clip, ghost->GetAdjustedIn());