Merge branch 'olive-editor:master' into av1

app/codec/ffmpeg/ffmpegdecoder.cpp

@@ -43,7 +43,6 @@ extern "C" {
 #include "common/define.h"
 #include "common/ffmpegutils.h"
 #include "common/filefunctions.h"
-#include "common/functiontimer.h"
 #include "common/timecodefunctions.h"
 #include "render/framehashcache.h"
 #include "render/diskmanager.h"
@@ -164,125 +163,117 @@ TexturePtr FFmpegDecoder::RetrieveVideoInternal(const RetrieveVideoParams &p)
                      p.divider);
 
       TexturePtr tex = nullptr;
-      bool hwscale = true;
 
       // Attempt to use GLSL shader for faster YUV to RGB conversion
-      if (hwscale) {
-        if (src_fmt == AV_PIX_FMT_YUV420P
-            || src_fmt == AV_PIX_FMT_YUV422P
-            || src_fmt == AV_PIX_FMT_YUV444P
-            || src_fmt == AV_PIX_FMT_YUV420P10LE
-            || src_fmt == AV_PIX_FMT_YUV422P10LE
-            || src_fmt == AV_PIX_FMT_YUV444P10LE
-            || src_fmt == AV_PIX_FMT_YUV420P12LE
-            || src_fmt == AV_PIX_FMT_YUV422P12LE
-            || src_fmt == AV_PIX_FMT_YUV444P12LE) {
-          if (Yuv2RgbShader.isNull()) {
-            // Compile shader
-            Yuv2RgbShader = p.renderer->CreateNativeShader(ShaderCode(FileFunctions::ReadFileAsString(QStringLiteral(":/shaders/yuv2rgb.frag"))));
+      if (IsPixelFormatGLSLCompatible(static_cast<AVPixelFormat>(src_fmt))) {
+        if (Yuv2RgbShader.isNull()) {
+          // Compile shader
+          Yuv2RgbShader = p.renderer->CreateNativeShader(ShaderCode(FileFunctions::ReadFileAsString(QStringLiteral(":/shaders/yuv2rgb.frag"))));
+        }
+
+        if (!Yuv2RgbShader.isNull()) {
+          int px_size;
+          int bits_per_pixel;
+          switch (src_fmt) {
+          case AV_PIX_FMT_YUV420P:
+          case AV_PIX_FMT_YUV422P:
+          case AV_PIX_FMT_YUV444P:
+          default:
+            px_size = 1;
+            bits_per_pixel = 8;
+            break;
+          case AV_PIX_FMT_YUV420P10LE:
+          case AV_PIX_FMT_YUV422P10LE:
+          case AV_PIX_FMT_YUV444P10LE:
+            px_size = 2;
+            bits_per_pixel = 10;
+            break;
+          case AV_PIX_FMT_YUV420P12LE:
+          case AV_PIX_FMT_YUV422P12LE:
+          case AV_PIX_FMT_YUV444P12LE:
+            px_size = 2;
+            bits_per_pixel = 12;
+            break;
           }
 
-          if (!Yuv2RgbShader.isNull()) {
-            int px_size;
-            int bits_per_pixel;
-            switch (src_fmt) {
-            case AV_PIX_FMT_YUV420P:
-            case AV_PIX_FMT_YUV422P:
-            case AV_PIX_FMT_YUV444P:
-            default:
-              px_size = 1;
-              bits_per_pixel = 8;
-              break;
-            case AV_PIX_FMT_YUV420P10LE:
-            case AV_PIX_FMT_YUV422P10LE:
-            case AV_PIX_FMT_YUV444P10LE:
-              px_size = 2;
-              bits_per_pixel = 10;
-              break;
-            case AV_PIX_FMT_YUV420P12LE:
-            case AV_PIX_FMT_YUV422P12LE:
-            case AV_PIX_FMT_YUV444P12LE:
-              px_size = 2;
-              bits_per_pixel = 12;
-              break;
-            }
+          AVFrame *hw_in = f.get();
 
-            VideoParams plane_params = vp;
-            plane_params.set_channel_count(1);
+          VideoParams plane_params = vp;
+          plane_params.set_channel_count(1);
+          plane_params.set_format(native_internal_pix_fmt_);
+
+          if (p.divider != 1) {
+            ApplyScaler(f.get());
+            hw_in = working_frame_;
+          } else {
+            // Fallback: shouldn't ever really get here, but just in case
             plane_params.set_divider(1);
-            plane_params.set_format(native_internal_pix_fmt_);
-            TexturePtr y_plane = p.renderer->CreateTexture(plane_params, f->data[0], f->linesize[0] / px_size);
-
-            if (src_fmt == AV_PIX_FMT_YUV420P
-                || src_fmt == AV_PIX_FMT_YUV422P
-                || src_fmt == AV_PIX_FMT_YUV420P10LE
-                || src_fmt == AV_PIX_FMT_YUV422P10LE
-                || src_fmt == AV_PIX_FMT_YUV420P12LE
-                || src_fmt == AV_PIX_FMT_YUV422P12LE) {
-              plane_params.set_width(plane_params.width()/2);
-            }
-
-            if (src_fmt == AV_PIX_FMT_YUV420P
-                || src_fmt == AV_PIX_FMT_YUV420P10LE
-                || src_fmt == AV_PIX_FMT_YUV420P12LE) {
-              plane_params.set_height(plane_params.height()/2);
-            }
-
-            TexturePtr u_plane = p.renderer->CreateTexture(plane_params, f->data[1], f->linesize[1] / px_size);
-            TexturePtr v_plane = p.renderer->CreateTexture(plane_params, f->data[2], f->linesize[2] / px_size);
-
-            ShaderJob job;
-            job.Insert(QStringLiteral("y_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(y_plane)));
-            job.Insert(QStringLiteral("u_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(u_plane)));
-            job.Insert(QStringLiteral("v_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(v_plane)));
-            job.Insert(QStringLiteral("bits_per_pixel"), NodeValue(NodeValue::kInt, bits_per_pixel));
-            job.Insert(QStringLiteral("full_range"), NodeValue(NodeValue::kBoolean, f->color_range == AVCOL_RANGE_JPEG));
-
-            const int *yuv_coeffs = sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace(f.get()->colorspace));
-            job.Insert(QStringLiteral("yuv_crv"), NodeValue(NodeValue::kInt, yuv_coeffs[0]));
-            job.Insert(QStringLiteral("yuv_cgu"), NodeValue(NodeValue::kInt, yuv_coeffs[2]));
-            job.Insert(QStringLiteral("yuv_cgv"), NodeValue(NodeValue::kInt, yuv_coeffs[3]));
-            job.Insert(QStringLiteral("yuv_cbu"), NodeValue(NodeValue::kInt, yuv_coeffs[1]));
-
-            int interlacing = 0;
-            if (p.src_interlacing != VideoParams::kInterlaceNone) {
-              if (frame_rate_tb_.isNull()) {
-                frame_rate_tb_ = av_guess_frame_rate(instance_.fmt_ctx(), instance_.avstream(), f.get());
-
-                // Double frame rate for interlaced fields
-                frame_rate_tb_ *= 2;
-
-                // Flip frame rate so it can be used as a timebase
-                frame_rate_tb_.flip();
-              }
-
-              int64_t req = Timecode::time_to_timestamp(p.time, frame_rate_tb_);
-              int64_t frm = Timecode::rescale_timestamp(f->pts - instance_.avstream()->start_time, instance_.avstream()->time_base, frame_rate_tb_);
-
-              bool first = (req == frm);
-              bool top_first = (p.src_interlacing == VideoParams::kInterlacedTopFirst);
-
-              interlacing = (first == top_first) ? 1 : 2;
-            }
-            job.Insert(QStringLiteral("interlacing"), NodeValue(NodeValue::kInt, interlacing));
-            job.Insert(QStringLiteral("pixel_height"), NodeValue(NodeValue::kInt, f->height));
-
-            tex = p.renderer->CreateTexture(vp);
-            p.renderer->BlitToTexture(Yuv2RgbShader, job, tex.get(), false);
           }
+
+          TexturePtr y_plane = p.renderer->CreateTexture(plane_params, hw_in->data[0], hw_in->linesize[0] / px_size);
+
+          if (src_fmt == AV_PIX_FMT_YUV420P
+              || src_fmt == AV_PIX_FMT_YUV422P
+              || src_fmt == AV_PIX_FMT_YUV420P10LE
+              || src_fmt == AV_PIX_FMT_YUV422P10LE
+              || src_fmt == AV_PIX_FMT_YUV420P12LE
+              || src_fmt == AV_PIX_FMT_YUV422P12LE) {
+            plane_params.set_width(plane_params.width()/2);
+          }
+
+          if (src_fmt == AV_PIX_FMT_YUV420P
+              || src_fmt == AV_PIX_FMT_YUV420P10LE
+              || src_fmt == AV_PIX_FMT_YUV420P12LE) {
+            plane_params.set_height(plane_params.height()/2);
+          }
+
+          TexturePtr u_plane = p.renderer->CreateTexture(plane_params, hw_in->data[1], hw_in->linesize[1] / px_size);
+          TexturePtr v_plane = p.renderer->CreateTexture(plane_params, hw_in->data[2], hw_in->linesize[2] / px_size);
+
+          ShaderJob job;
+          job.Insert(QStringLiteral("y_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(y_plane)));
+          job.Insert(QStringLiteral("u_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(u_plane)));
+          job.Insert(QStringLiteral("v_channel"), NodeValue(NodeValue::kTexture, QVariant::fromValue(v_plane)));
+          job.Insert(QStringLiteral("bits_per_pixel"), NodeValue(NodeValue::kInt, bits_per_pixel));
+          job.Insert(QStringLiteral("full_range"), NodeValue(NodeValue::kBoolean, f->color_range == AVCOL_RANGE_JPEG));
+
+          const int *yuv_coeffs = sws_getCoefficients(FFmpegUtils::GetSwsColorspaceFromAVColorSpace(f.get()->colorspace));
+          job.Insert(QStringLiteral("yuv_crv"), NodeValue(NodeValue::kInt, yuv_coeffs[0]));
+          job.Insert(QStringLiteral("yuv_cgu"), NodeValue(NodeValue::kInt, yuv_coeffs[2]));
+          job.Insert(QStringLiteral("yuv_cgv"), NodeValue(NodeValue::kInt, yuv_coeffs[3]));
+          job.Insert(QStringLiteral("yuv_cbu"), NodeValue(NodeValue::kInt, yuv_coeffs[1]));
+
+          int interlacing = 0;
+          if (p.src_interlacing != VideoParams::kInterlaceNone) {
+            if (frame_rate_tb_.isNull()) {
+              frame_rate_tb_ = av_guess_frame_rate(instance_.fmt_ctx(), instance_.avstream(), f.get());
+
+              // Double frame rate for interlaced fields
+              frame_rate_tb_ *= 2;
+
+              // Flip frame rate so it can be used as a timebase
+              frame_rate_tb_.flip();
+            }
+
+            int64_t req = Timecode::time_to_timestamp(p.time, frame_rate_tb_);
+            int64_t frm = Timecode::rescale_timestamp(f->pts - instance_.avstream()->start_time, instance_.avstream()->time_base, frame_rate_tb_);
+
+            bool first = (req == frm);
+            bool top_first = (p.src_interlacing == VideoParams::kInterlacedTopFirst);
+
+            interlacing = (first == top_first) ? 1 : 2;
+          }
+          job.Insert(QStringLiteral("interlacing"), NodeValue(NodeValue::kInt, interlacing));
+          job.Insert(QStringLiteral("pixel_height"), NodeValue(NodeValue::kInt, f->height));
+
+          tex = p.renderer->CreateTexture(vp);
+          p.renderer->BlitToTexture(Yuv2RgbShader, job, tex.get(), false);
         }
       }
 
       if (!tex) {
         // Fallback to software pixel format conversion
-        int r;
-
-        r = av_buffersrc_add_frame_flags(buffersrc_ctx_, f.get(), AV_BUFFERSRC_FLAG_KEEP_REF);
-        if (r < 0) {
-          return nullptr;
-        }
-        r = av_buffersink_get_frame(buffersink_ctx_, working_frame_);
-        if (r < 0) {
+        if (!ApplyScaler(f.get())) {
           return nullptr;
         }
 
@@ -717,6 +708,19 @@ const char *FFmpegDecoder::GetInterlacingModeInFFmpeg(VideoParams::Interlacing i
   }
 }
 
+bool FFmpegDecoder::IsPixelFormatGLSLCompatible(AVPixelFormat f)
+{
+  return f == AV_PIX_FMT_YUV420P
+      || f == AV_PIX_FMT_YUV422P
+      || f == AV_PIX_FMT_YUV444P
+      || f == AV_PIX_FMT_YUV420P10LE
+      || f == AV_PIX_FMT_YUV422P10LE
+      || f == AV_PIX_FMT_YUV444P10LE
+      || f == AV_PIX_FMT_YUV420P12LE
+      || f == AV_PIX_FMT_YUV422P12LE
+      || f == AV_PIX_FMT_YUV444P12LE;
+}
+
 /* OLD UNUSED CODE: Keeping this around in case the code proves useful
 
 void FFmpegDecoder::CacheFrameToDisk(AVFrame *f)
@@ -1023,7 +1027,7 @@ bool FFmpegDecoder::InitScaler(AVFrame *input, const RetrieveVideoParams& params
   }
 
   // Add format filter if necessary
-  if (ideal_pix_fmt != input->format) {
+  if (ideal_pix_fmt != input->format && !IsPixelFormatGLSLCompatible(static_cast<AVPixelFormat>(input->format))) {
     AVFilterContext* format_filter;
 
     snprintf(filter_args, kFilterArgSz, "pix_fmts=%u", ideal_pix_fmt);
@@ -1097,6 +1101,22 @@ void FFmpegDecoder::RemoveFirstFrame()
   cache_at_zero_ = false;
 }
 
+bool FFmpegDecoder::ApplyScaler(AVFrame *in)
+{
+  int r;
+
+  r = av_buffersrc_add_frame_flags(buffersrc_ctx_, in, AV_BUFFERSRC_FLAG_KEEP_REF);
+  if (r < 0) {
+    return false;
+  }
+  r = av_buffersink_get_frame(buffersink_ctx_, working_frame_);
+  if (r < 0) {
+    return false;
+  }
+
+  return true;
+}
+
 int FFmpegDecoder::MaximumQueueSize()
 {
   // Fairly arbitrary size. This used to need to be the number of current threads to ensure any

app/codec/ffmpeg/ffmpegdecoder.h

@@ -142,6 +142,8 @@ private:
 
   static const char* GetInterlacingModeInFFmpeg(VideoParams::Interlacing interlacing);
 
+  static bool IsPixelFormatGLSLCompatible(AVPixelFormat f);
+
   AVFramePtr GetFrameFromCache(const int64_t &t) const;
 
   void ClearFrameCache();
@@ -150,6 +152,8 @@ private:
 
   void RemoveFirstFrame();
 
+  bool ApplyScaler(AVFrame *in);
+
   static int MaximumQueueSize();
 
   RetrieveVideoParams filter_params_;

app/codec/samplebuffer.cpp

@@ -36,7 +36,7 @@ SampleBuffer::SampleBuffer(const AudioParams &audio_params, const rational &leng
   allocate();
 }
 
-SampleBuffer::SampleBuffer(const AudioParams &audio_params, int samples_per_channel) :
+SampleBuffer::SampleBuffer(const AudioParams &audio_params, size_t samples_per_channel) :
   audio_params_(audio_params),
   sample_count_per_channel_(samples_per_channel)
 {
@@ -58,12 +58,7 @@ void SampleBuffer::set_audio_params(const AudioParams &params)
   audio_params_ = params;
 }
 
-const int &SampleBuffer::sample_count() const
-{
-  return sample_count_per_channel_;
-}
-
-void SampleBuffer::set_sample_count(const int &sample_count)
+void SampleBuffer::set_sample_count(const size_t &sample_count)
 {
   if (is_allocated()) {
     qWarning() << "Tried to set sample count on allocated sample buffer";
@@ -73,11 +68,6 @@ void SampleBuffer::set_sample_count(const int &sample_count)
   sample_count_per_channel_ = sample_count;
 }
 
-bool SampleBuffer::is_allocated() const
-{
-  return !data_.isEmpty();
-}
-
 void SampleBuffer::allocate()
 {
   if (!audio_params_.is_valid()) {
@@ -113,10 +103,10 @@ void SampleBuffer::reverse()
     return;
   }
 
-  int half_nb_sample = sample_count_per_channel_ / 2;
+  size_t half_nb_sample = sample_count_per_channel_ / 2;
 
-  for (int i=0;i<half_nb_sample;i++) {
-    int opposite_ind = sample_count_per_channel_ - i - 1;
+  for (size_t i=0;i<half_nb_sample;i++) {
+    size_t opposite_ind = sample_count_per_channel_ - i - 1;
 
     for (int j=0;j<audio_params_.channel_count();j++) {
       std::swap(data_[j][i], data_[j][opposite_ind]);
@@ -133,15 +123,15 @@ void SampleBuffer::speed(double speed)
 
   sample_count_per_channel_ = qRound(static_cast<double>(sample_count_per_channel_) / speed);
 
-  QVector< QVector<float> > output_data;
+  std::vector< std::vector<float> > output_data;
 
   output_data.resize(audio_params_.channel_count());
   for (int i=0; i<audio_params_.channel_count(); i++) {
     output_data[i].resize(sample_count_per_channel_);
   }
 
-  for (int i=0;i<sample_count_per_channel_;i++) {
-    int input_index = qFloor(static_cast<double>(i) * speed);
+  for (size_t i=0;i<sample_count_per_channel_;i++) {
+    size_t input_index = qFloor(static_cast<double>(i) * speed);
 
     for (int j=0;j<audio_params_.channel_count();j++) {
       output_data[j][i] = data_[j][input_index];
@@ -161,12 +151,12 @@ void SampleBuffer::transform_volume(float f)
 void SampleBuffer::transform_volume_for_channel(int channel, float volume)
 {
   float *cdat = data_[channel].data();
-  int unopt_start = 0;
+  size_t unopt_start = 0;
 
 #if defined(Q_PROCESSOR_X86) || defined(Q_PROCESSOR_ARM)
   __m128 mult = _mm_load1_ps(&volume);
   unopt_start = (sample_count_per_channel_ / 4) * 4;
-  for (int j=0; j<unopt_start; j+=4) {
+  for (size_t j=0; j<unopt_start; j+=4) {
     float *here = cdat + j;
     __m128 samples = _mm_loadu_ps(here);
     __m128 multiplied = _mm_mul_ps(samples, mult);
@@ -174,19 +164,19 @@ void SampleBuffer::transform_volume_for_channel(int channel, float volume)
   }
 #endif
 
-  for (int j=unopt_start; j<sample_count_per_channel_; j++) {
+  for (size_t j=unopt_start; j<sample_count_per_channel_; j++) {
     cdat[j] *= volume;
   }
 }
 
-void SampleBuffer::transform_volume_for_sample(int sample_index, float volume)
+void SampleBuffer::transform_volume_for_sample(size_t sample_index, float volume)
 {
   for (int i=0;i<audio_params().channel_count();i++) {
     transform_volume_for_sample_on_channel(sample_index, i, volume);
   }
 }
 
-void SampleBuffer::transform_volume_for_sample_on_channel(int sample_index, int channel, float volume)
+void SampleBuffer::transform_volume_for_sample_on_channel(size_t sample_index, int channel, float volume)
 {
   data_[channel][sample_index] *= volume;
 }
@@ -203,12 +193,12 @@ void SampleBuffer::silence()
   silence(0, sample_count_per_channel_);
 }
 
-void SampleBuffer::silence(int start_sample, int end_sample)
+void SampleBuffer::silence(size_t start_sample, size_t end_sample)
 {
   silence_bytes(start_sample * sizeof(float), end_sample * sizeof(float));
 }
 
-void SampleBuffer::silence_bytes(int start_byte, int end_byte)
+void SampleBuffer::silence_bytes(size_t start_byte, size_t end_byte)
 {
   if (!is_allocated()) {
     qWarning() << "Tried to fill an unallocated sample buffer";
@@ -220,7 +210,7 @@ void SampleBuffer::silence_bytes(int start_byte, int end_byte)
   }
 }
 
-void SampleBuffer::set(int channel, const float *data, int sample_offset, int sample_length)
+void SampleBuffer::set(int channel, const float *data, size_t sample_offset, size_t sample_length)
 {
   if (!is_allocated()) {
     qWarning() << "Tried to fill an unallocated sample buffer";
@@ -236,14 +226,14 @@ void SampleBuffer::clamp_channel(int channel)
   const float max = 1.0f;
 
   float *cdat = data_[channel].data();
-  int unopt_start = 0;
+  size_t unopt_start = 0;
 
 #if defined(Q_PROCESSOR_X86) || defined(Q_PROCESSOR_ARM)
   __m128 min_sse = _mm_load1_ps(&min);
   __m128 max_sse = _mm_load1_ps(&max);
 
   unopt_start = (sample_count_per_channel_ / 4) * 4;
-  for (int j=0; j<unopt_start; j+=4) {
+  for (size_t j=0; j<unopt_start; j+=4) {
     float *here = cdat + j;
     __m128 samples = _mm_loadu_ps(here);
 
@@ -254,7 +244,7 @@ void SampleBuffer::clamp_channel(int channel)
   }
 #endif
 
-  for (int sample=unopt_start; sample<sample_count(); sample++) {
+  for (size_t sample=unopt_start; sample<sample_count(); sample++) {
     float &s = data(channel)[sample];
     s = std::clamp(s, min, max);
   }

app/codec/samplebuffer.h

@@ -40,13 +40,13 @@ class SampleBuffer
 public:
   SampleBuffer();
   SampleBuffer(const AudioParams& audio_params, const rational& length);
-  SampleBuffer(const AudioParams& audio_params, int samples_per_channel);
+  SampleBuffer(const AudioParams& audio_params, size_t samples_per_channel);
 
   const AudioParams& audio_params() const;
   void set_audio_params(const AudioParams& params);
 
-  const int &sample_count() const;
-  void set_sample_count(const int &sample_count);
+  const size_t &sample_count() const { return sample_count_per_channel_; }
+  void set_sample_count(const size_t &sample_count);
   void set_sample_count(const rational &length)
   {
     set_sample_count(audio_params_.time_to_samples(length));
@@ -59,13 +59,13 @@ public:
 
   const float* data(int channel) const
   {
-    return data_.at(channel).constData();
+    return data_.at(channel).data();
   }
 
-  QVector<float *> to_raw_ptrs()
+  std::vector<float *> to_raw_ptrs()
   {
-    QVector<float *> r(data_.size());
-    for (int i=0; i<r.size(); i++) {
+    std::vector<float *> r(data_.size());
+    for (size_t i=0; i<r.size(); i++) {
       r[i] = data_[i].data();
     }
     return r;
@@ -73,7 +73,7 @@ public:
 
   int channel_count() const { return data_.size(); }
 
-  bool is_allocated() const;
+  bool is_allocated() const { return !data_.empty(); }
   void allocate();
   void destroy();
 
@@ -81,17 +81,17 @@ public:
   void speed(double speed);
   void transform_volume(float f);
   void transform_volume_for_channel(int channel, float volume);
-  void transform_volume_for_sample(int sample_index, float volume);
-  void transform_volume_for_sample_on_channel(int sample_index, int channel, float volume);
+  void transform_volume_for_sample(size_t sample_index, float volume);
+  void transform_volume_for_sample_on_channel(size_t sample_index, int channel, float volume);
 
   void clamp();
 
   void silence();
-  void silence(int start_sample, int end_sample);
-  void silence_bytes(int start_byte, int end_byte);
+  void silence(size_t start_sample, size_t end_sample);
+  void silence_bytes(size_t start_byte, size_t end_byte);
 
-  void set(int channel, const float* data, int sample_offset, int sample_length);
-  void set(int channel, const float* data, int sample_length)
+  void set(int channel, const float* data, size_t sample_offset, size_t sample_length);
+  void set(int channel, const float* data, size_t sample_length)
   {
     set(channel, data, 0, sample_length);
   }
@@ -101,9 +101,9 @@ private:
 
   AudioParams audio_params_;
 
-  int sample_count_per_channel_;
+  size_t sample_count_per_channel_;
 
-  QVector< QVector<float> > data_;
+  std::vector< std::vector<float> > data_;
 
 };