multicam: refactor to heavily optimize multicam monitor retrieval

app/widget/multicam/multicamdisplay.cpp

@@ -26,7 +26,9 @@ namespace olive {
 
 MulticamDisplay::MulticamDisplay(QWidget *parent) :
   super(parent),
-  node_(nullptr)
+  node_(nullptr),
+  rows_(0),
+  cols_(0)
 {
 }
 
@@ -55,6 +57,113 @@ void MulticamDisplay::OnPaint()
   }
 }
 
+void MulticamDisplay::OnDestroy()
+{
+  shader_ = QVariant();
+}
+
+TexturePtr MulticamDisplay::LoadCustomTextureFromFrame(const QVariant &v)
+{
+  if (v.canConvert<QVector<TexturePtr> >()) {
+    QVector<TexturePtr> tex = v.value<QVector<TexturePtr> >();
+
+    TexturePtr main = renderer()->CreateTexture(this->GetViewportParams());
+
+    int rows, cols;
+    MultiCamNode::GetRowsAndColumns(tex.size(), &rows, &cols);
+
+    if (shader_.isNull() || rows_ != rows || cols_ != cols) {
+      if (!shader_.isNull()) {
+        renderer()->DestroyNativeShader(shader_);
+      }
+
+      shader_ = renderer()->CreateNativeShader(ShaderCode(GenerateShaderCode(rows, cols)));
+
+      rows_ = rows;
+      cols_ = cols;
+    }
+
+    ShaderJob job;
+
+    for (int i=0; i<tex.size(); i++) {
+      int c, r;
+      MultiCamNode::IndexToRowCols(i, rows, cols, &r, &c);
+      job.Insert(QStringLiteral("tex_%1_%2").arg(QString::number(r), QString::number(c)), NodeValue(NodeValue::kTexture, tex.at(i)));
+    }
+
+    renderer()->BlitToTexture(shader_, job, main.get());
+
+    return main;
+  } else {
+    return super::LoadCustomTextureFromFrame(v);
+  }
+}
+
+QString dblToGlsl(double d)
+{
+  return QString::number(d, 'f');
+}
+
+QString MulticamDisplay::GenerateShaderCode(int rows, int cols)
+{
+  int multiplier = std::max(cols, rows);
+
+  QStringList shader;
+
+  shader.append(QStringLiteral("in vec2 ove_texcoord;"));
+  shader.append(QStringLiteral("out vec4 frag_color;"));
+
+  for (int x=0;x<cols;x++) {
+    for (int y=0;y<rows;y++) {
+      shader.append(QStringLiteral("uniform sampler2D tex_%1_%2;").arg(QString::number(y), QString::number(x)));
+      shader.append(QStringLiteral("uniform bool tex_%1_%2_enabled;").arg(QString::number(y), QString::number(x)));
+    }
+  }
+
+  shader.append(QStringLiteral("void main() {"));
+
+  for (int x=0;x<cols;x++) {
+    if (x > 0) {
+      shader.append(QStringLiteral("  else"));
+    }
+    if (x == cols-1) {
+      shader.append(QStringLiteral("  {"));
+    } else {
+      shader.append(QStringLiteral("  if (ove_texcoord.x < %1) {").arg(dblToGlsl(double(x+1)/double(multiplier))));
+    }
+
+    for (int y=0;y<rows;y++) {
+      if (y > 0) {
+        shader.append(QStringLiteral("    else"));
+      }
+      if (y == rows-1) {
+        shader.append(QStringLiteral("    {"));
+      } else {
+        shader.append(QStringLiteral("    if (ove_texcoord.y < %1) {").arg(dblToGlsl(double(y+1)/double(multiplier))));
+      }
+      QString input = QStringLiteral("tex_%1_%2").arg(QString::number(y), QString::number(x));
+      shader.append(QStringLiteral("      vec2 coord = vec2((ove_texcoord.x+%1)*%2, (ove_texcoord.y+%3)*%4);").arg(
+                      dblToGlsl( - double(x)/double(multiplier)),
+                      dblToGlsl(multiplier),
+                      dblToGlsl( - double(y)/double(multiplier)),
+                      dblToGlsl(multiplier)
+                      ));
+      shader.append(QStringLiteral("      if (%1_enabled && coord.x >= 0.0 && coord.x < 1.0 && coord.y >= 0.0 && coord.y < 1.0) {").arg(input));
+      shader.append(QStringLiteral("        frag_color = texture(%1, coord);").arg(input));
+      shader.append(QStringLiteral("      } else {"));
+      shader.append(QStringLiteral("        discard;"));
+      shader.append(QStringLiteral("      }"));
+      shader.append(QStringLiteral("    }"));
+    }
+
+    shader.append(QStringLiteral("  }"));
+  }
+
+  shader.append(QStringLiteral("}"));
+
+  return shader.join('\n');
+}
+
 void MulticamDisplay::SetMulticamNode(MultiCamNode *n)
 {
   node_ = n;

app/widget/multicam/multicamdisplay.h

@@ -37,9 +37,19 @@ public:
 protected:
   virtual void OnPaint() override;
 
+  virtual void OnDestroy() override;
+
+  virtual TexturePtr LoadCustomTextureFromFrame(const QVariant &v) override;
+
 private:
+  static QString GenerateShaderCode(int rows, int cols);
+
   MultiCamNode *node_;
 
+  QVariant shader_;
+  int rows_;
+  int cols_;
+
 };
 
 }

app/widget/multicam/multicamwidget.cpp

@@ -20,27 +20,41 @@
 
 #include "multicamwidget.h"
 #include "widget/nodeparamview/nodeparamviewundo.h"
+#include "widget/timeruler/timeruler.h"
 #include "widget/timelinewidget/undo/timelineundosplit.h"
 
 namespace olive {
 
-#define super ViewerWidget
+#define super TimeBasedWidget
 
 MulticamWidget::MulticamWidget(QWidget *parent) :
-  super{new MulticamDisplay(), parent},
+  super{false, false, parent},
   node_(nullptr),
   clip_(nullptr)
 {
-  auto_cacher()->SetMulticamMode(true);
-  auto_cacher()->SetIgnoreCacheRequests(true);
+  auto layout = new QVBoxLayout(this);
 
-  connect(display_widget(), &ViewerDisplayWidget::DragStarted, this, &MulticamWidget::DisplayClicked);
+  sizer_ = new ViewerSizer(this);
+  layout->addWidget(sizer_);
+
+  display_ = new MulticamDisplay(this);
+  display_->SetShowWidgetBackground(true);
+  connect(display_, &ViewerDisplayWidget::DragStarted, this, &MulticamWidget::DisplayClicked);
+
+  connect(sizer_, &ViewerSizer::RequestScale, display_, &ViewerDisplayWidget::SetMatrixZoom);
+  connect(sizer_, &ViewerSizer::RequestTranslate, display_, &ViewerDisplayWidget::SetMatrixTranslate);
+  connect(display_, &ViewerDisplayWidget::HandDragMoved, sizer_, &ViewerSizer::HandDragMove);
+  sizer_->SetWidget(display_);
+
+  layout->addWidget(this->ruler());
+  layout->addWidget(this->scrollbar());
 }
 
 void MulticamWidget::SetMulticamNode(MultiCamNode *n)
 {
   node_ = n;
-  static_cast<MulticamDisplay*>(display_widget())->SetMulticamNode(n);
+  display_->SetMulticamNode(n);
+  display_->update();
 }
 
 void MulticamWidget::SetClip(ClipBlock *clip)
@@ -48,13 +62,20 @@ void MulticamWidget::SetClip(ClipBlock *clip)
   clip_ = clip;
 }
 
-RenderTicketPtr MulticamWidget::GetSingleFrame(const rational &t, bool dry)
+void MulticamWidget::ConnectNodeEvent(ViewerOutput *n)
 {
-  if (node_) {
-    return auto_cacher()->GetSingleFrame(node_, t, dry);
-  } else {
-    return super::GetSingleFrame(t, dry);
-  }
+  connect(n, &ViewerOutput::SizeChanged, sizer_, &ViewerSizer::SetChildSize);
+  connect(n, &ViewerOutput::PixelAspectChanged, sizer_, &ViewerSizer::SetPixelAspectRatio);
+
+  VideoParams vp = n->GetVideoParams();
+  sizer_->SetChildSize(vp.width(), vp.height());
+  sizer_->SetPixelAspectRatio(vp.pixel_aspect_ratio());
+}
+
+void MulticamWidget::DisconnectNodeEvent(ViewerOutput *n)
+{
+  disconnect(n, &ViewerOutput::SizeChanged, sizer_, &ViewerSizer::SetChildSize);
+  disconnect(n, &ViewerOutput::PixelAspectChanged, sizer_, &ViewerSizer::SetPixelAspectRatio);
 }
 
 void MulticamWidget::DisplayClicked(const QPoint &p)
@@ -63,9 +84,9 @@ void MulticamWidget::DisplayClicked(const QPoint &p)
     return;
   }
 
-  QPointF click = display_widget()->ScreenToScenePoint(p);
-  int width = display_widget()->GetVideoParams().width();
-  int height = display_widget()->GetVideoParams().height();
+  QPointF click = display_->ScreenToScenePoint(p);
+  int width = display_->GetVideoParams().width();
+  int height = display_->GetVideoParams().height();
 
   if (click.x() < 0 || click.y() < 0 || click.x() >= width || click.y() >= height) {
     return;

app/widget/multicam/multicamwidget.h

@@ -27,20 +27,27 @@
 
 namespace olive {
 
-class MulticamWidget : public ViewerWidget
+class MulticamWidget : public TimeBasedWidget
 {
   Q_OBJECT
 public:
   explicit MulticamWidget(QWidget *parent = nullptr);
 
+  MulticamDisplay *GetDisplayWidget() const { return display_; }
+
   void SetMulticamNode(MultiCamNode *n);
 
   void SetClip(ClipBlock *clip);
 
 protected:
-  virtual RenderTicketPtr GetSingleFrame(const rational &t, bool dry = false) override;
+  virtual void ConnectNodeEvent(ViewerOutput *n) override;
+  virtual void DisconnectNodeEvent(ViewerOutput *n) override;
 
 private:
+  ViewerSizer *sizer_;
+
+  MulticamDisplay *display_;
+
   MultiCamNode *node_;
 
   ClipBlock *clip_;

app/widget/viewer/viewer.cpp

@@ -44,6 +44,7 @@
 #include "viewerpreventsleep.h"
 #include "widget/audiomonitor/audiomonitor.h"
 #include "widget/menu/menu.h"
+#include "widget/multicam/multicamdisplay.h"
 #include "widget/nodeparamview/nodeparamviewundo.h"
 #include "widget/timelinewidget/tool/add.h"
 #include "widget/timeruler/timeruler.h"
@@ -82,7 +83,6 @@ ViewerWidget::ViewerWidget(ViewerDisplayWidget *display, QWidget *parent) :
 
   // Create main OpenGL-based view and sizer
   sizer_ = new ViewerSizer();
-  sizer_->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding);
   layout->addWidget(sizer_);
 
   display_widget_ = display;
@@ -260,7 +260,7 @@ void ViewerWidget::DisconnectNodeEvent(ViewerOutput *n)
   CloseAudioProcessor();
   audio_scrub_watchers_.clear();
 
-  SetDisplayImage(QVariant());
+  SetDisplayImage(nullptr);
 
   ruler()->SetPlaybackCache(nullptr);
 
@@ -998,10 +998,18 @@ bool ViewerWidget::ViewerMightBeAStill()
   return GetConnectedNode() && GetConnectedNode()->GetConnectedTextureOutput() && GetConnectedNode()->GetVideoLength().isNull();
 }
 
-void ViewerWidget::SetDisplayImage(QVariant frame)
+void ViewerWidget::SetDisplayImage(RenderTicketPtr ticket)
 {
   foreach (ViewerDisplayWidget *dw, playback_devices_) {
-    dw->SetImage(frame);
+    QVariant push;
+    if (ticket) {
+      if (dynamic_cast<MulticamDisplay*>(dw)) {
+        push = ticket->property("multicam_output");
+      } else {
+        push = ticket->Get();
+      }
+    }
+    dw->SetImage(push);
   }
 }
 
@@ -1160,7 +1168,7 @@ void ViewerWidget::RendererGeneratedFrame()
         }
       }
 
-      SetDisplayImage(ticket->Get());
+      SetDisplayImage(ticket->GetTicket());
     }
   }
 
@@ -1182,7 +1190,14 @@ void ViewerWidget::RendererGeneratedFrameForQueue()
         rational ts = watcher->property("time").value<rational>();
 
         foreach (ViewerDisplayWidget *dw, playback_devices_) {
-          dw->queue()->AppendTimewise({ts, frame}, playback_speed_);
+          QVariant push;
+          if (dynamic_cast<MulticamDisplay*>(dw)) {
+            push = watcher->GetTicket()->property("multicam_output");
+          } else {
+            push = frame;
+          }
+
+          dw->queue()->AppendTimewise({ts, push}, playback_speed_);
         }
 
         if (prequeuing_video_) {

app/widget/viewer/viewer.h

@@ -104,6 +104,16 @@ public:
 
   PreviewAutoCacher *GetCacher() const { return auto_cacher_; }
 
+  void AddPlaybackDevice(ViewerDisplayWidget *vw)
+  {
+    playback_devices_.push_back(vw);
+  }
+
+  void SetMulticamNode(MultiCamNode *n)
+  {
+    auto_cacher()->SetMulticamNode(n);
+  }
+
 public slots:
   void Play(bool in_to_out_only);
 
@@ -219,7 +229,7 @@ private:
 
   bool ViewerMightBeAStill();
 
-  void SetDisplayImage(QVariant frame);
+  void SetDisplayImage(RenderTicketPtr ticket);
 
   RenderTicketWatcher *RequestNextFrameForQueue(bool increment = true);
 

app/widget/viewer/viewerdisplay.cpp

@@ -398,6 +398,8 @@ void ViewerDisplayWidget::OnPaint()
       } else if (TexturePtr texture = load_frame_.value<TexturePtr>()) {
         // This is a GPU texture, switch to it directly
         texture_ = texture;
+      } else {
+        texture_ = LoadCustomTextureFromFrame(load_frame_);
       }
 
       emit TextureChanged(texture_);

app/widget/viewer/viewerdisplay.h

@@ -245,6 +245,11 @@ protected:
     return TimeRange(node_time, node_time + gizmo_params_.frame_rate_as_time_base());
   }
 
+  virtual TexturePtr LoadCustomTextureFromFrame(const QVariant &v)
+  {
+    return nullptr;
+  }
+
 protected slots:
   /**
    * @brief Paint function to display the texture (received in SetTexture()) on screen.

app/widget/viewer/viewerqueue.h

@@ -27,12 +27,14 @@
 
 namespace olive {
 
-struct ViewerPlaybackFrame {
+struct ViewerPlaybackFrame
+{
   rational timestamp;
   QVariant frame;
 };
 
-class ViewerQueue : public std::list<ViewerPlaybackFrame> {
+class ViewerQueue : public std::list<ViewerPlaybackFrame>
+{
 public:
   ViewerQueue() = default;
 

app/widget/viewer/viewersizer.cpp

@@ -39,6 +39,8 @@ ViewerSizer::ViewerSizer(QWidget *parent) :
   vert_scrollbar_ = new QScrollBar(Qt::Vertical, this);
   vert_scrollbar_->setVisible(false);
   connect(vert_scrollbar_, &QScrollBar::valueChanged, this, &ViewerSizer::ScrollBarMoved);
+
+  setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding);
 }
 
 void ViewerSizer::SetWidget(QWidget *widget)

app/widget/viewer/viewersizer.h

@@ -51,6 +51,7 @@ public:
    */
   void SetWidget(QWidget* widget);
 
+public slots:
   /**
    * @brief Set resolution to use
    *
@@ -70,7 +71,6 @@ public:
    */
   void SetZoom(int percent);
 
-public slots:
   void HandDragMove(int x, int y);
 
 signals: