/*** Olive - Non-Linear Video Editor Copyright (C) 2022 Olive Team Modifications Copyright (C) 2025 mikesolar This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ***/ #include "viewerdisplay.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/define.h" #include "common/html.h" #include "common/qtutils.h" #include "config/config.h" #include "core.h" #include "node/block/subtitle/subtitle.h" #include "codec/frame.h" #include "node/gizmo/path.h" #include "node/gizmo/point.h" #include "node/gizmo/polygon.h" #include "node/gizmo/screen.h" #include "window/mainwindow/mainwindow.h" namespace olive { #define super ManagedDisplayWidget ViewerDisplayWidget::ViewerDisplayWidget(QWidget *parent) : super(parent) , deinterlace_texture_(nullptr) , signal_cursor_color_(false) , gizmos_(nullptr) , current_gizmo_(nullptr) , gizmo_drag_started_(false) , show_subtitles_(true) , subtitle_tracks_(nullptr) , hand_dragging_(false) , deinterlace_(false) , show_fps_(false) , frames_skipped_(0) , show_widget_background_(false) , playback_speed_(0) , push_mode_(k_push_null) , add_band_(false) , queue_starved_(false) , text_edit_(nullptr) { connect(Core::instance(), &Core::tool_changed, this, &ViewerDisplayWidget::tool_changed); // Initializes cursor based on tool update_cursor(); const int k_frame_rate_average_count = 8; frame_rate_averages_.resize(k_frame_rate_average_count); inner_widget()->setAcceptDrops(true); } ViewerDisplayWidget::~ViewerDisplayWidget() { delete text_edit_; MANAGEDDISPLAYWIDGET_DEFAULT_DESTRUCTOR_INNER; } void ViewerDisplayWidget::set_matrix_translate(const QMatrix4x4 &mat) { translate_matrix_ = mat; update_matrix(); } void ViewerDisplayWidget::set_matrix_zoom(const QMatrix4x4 &mat) { scale_matrix_ = mat; update_matrix(); } void ViewerDisplayWidget::set_matrix_crop(const QMatrix4x4 &mat) { crop_matrix_ = mat; update(); } void ViewerDisplayWidget::update_cursor() { if (Core::instance()->tool() == Tool::k_hand) { this->inner_widget()->setCursor(Qt::OpenHandCursor); } else if (Core::instance()->tool() == Tool::k_add) { this->inner_widget()->setCursor(Qt::CrossCursor); } else { this->inner_widget()->unsetCursor(); } } void ViewerDisplayWidget::set_signal_cursor_color_enabled(bool e) { signal_cursor_color_ = e; set_inner_mouse_tracking(e); } void ViewerDisplayWidget::set_image(const QVariant &buffer) { load_frame_ = buffer; if (load_frame_.isNull()) { push_mode_ = k_push_null; } else { push_mode_ = k_push_frame; } update(); } void ViewerDisplayWidget::set_blank() { push_mode_ = k_push_blank; update(); } void ViewerDisplayWidget::tool_changed() { update_cursor(); } void ViewerDisplayWidget::set_deinterlacing(bool e) { deinterlace_ = e; if (!deinterlace_) { if (!deinterlace_shader_.isNull()) { renderer()->destroy_native_shader(deinterlace_shader_); deinterlace_shader_.clear(); } deinterlace_texture_ = nullptr; } update(); } const ViewerSafeMarginInfo &ViewerDisplayWidget::get_safe_margin() const { return safe_margin_; } void ViewerDisplayWidget::set_safe_margins(const ViewerSafeMarginInfo &safe_margin) { if (safe_margin_ != safe_margin) { safe_margin_ = safe_margin; update(); } } void ViewerDisplayWidget::set_gizmos(Node *node) { if (gizmos_ != node) { gizmos_ = node; update(); } } void ViewerDisplayWidget::set_video_params(const VideoParams ¶ms) { gizmo_params_ = params; if (gizmos_) { update(); } } void ViewerDisplayWidget::set_audio_params(const AudioParams ¶ms) { gizmo_audio_params_ = params; if (gizmos_) { update(); } } void ViewerDisplayWidget::set_time(const Rational &time) { time_ = time; if (gizmos_) { update(); } } void ViewerDisplayWidget::set_subtitle_tracks(Sequence *list) { if (subtitle_tracks_) { disconnect(subtitle_tracks_, &Sequence::subtitles_changed, this, &ViewerDisplayWidget::subtitles_changed); } subtitle_tracks_ = list; if (subtitle_tracks_) { connect(subtitle_tracks_, &Sequence::subtitles_changed, this, &ViewerDisplayWidget::subtitles_changed); } update(); } QPointF ViewerDisplayWidget::transform_viewer_space_to_buffer_space(const QPointF &pos) { /* * Inversion will only fail if the viewer has been scaled by 0 in any direction * which I think should never happen. */ return pos * generate_display_transform().inverted(); } void ViewerDisplayWidget::reset_fps_timer() { fps_timer_start_ = QDateTime::currentMSecsSinceEpoch(); fps_timer_update_count_ = 0; frames_skipped_ = 0; frame_rate_average_count_ = 0; Core::instance()->clear_status_bar_message(); } void ViewerDisplayWidget::increment_skipped_frames() { frames_skipped_++; Core::instance()->show_status_bar_message( tr("%n skipped frame(s) detected during playback", nullptr, frames_skipped_), 10000); } bool ViewerDisplayWidget::eventFilter(QObject *o, QEvent *e) { if (o == this->inner_widget()) { switch (e->type()) { case QEvent::MouseButtonPress: { QMouseEvent *mouse = static_cast(e); if (!(mouse->flags() & Qt::MouseEventCreatedDoubleClick)) { if (on_mouse_press(mouse)) { return true; } } break; } case QEvent::MouseMove: emit_color_at_cursor(static_cast(e)); if (on_mouse_move(static_cast(e))) { return true; } break; case QEvent::MouseButtonRelease: if (on_mouse_release(static_cast(e))) { return true; } break; case QEvent::MouseButtonDblClick: if (on_mouse_double_click(static_cast(e))) { return true; } break; case QEvent::ShortcutOverride: case QEvent::KeyPress: if (on_key_press(static_cast(e))) { return true; } break; case QEvent::KeyRelease: if (on_key_release(static_cast(e))) { return true; } break; case QEvent::DragEnter: { auto drag_enter = static_cast(e); if (text_edit_) { forward_drag_event_to_text_edit(drag_enter); } else { emit drag_entered(drag_enter); } if (drag_enter->isAccepted()) { return true; } break; } case QEvent::DragMove: { auto drag_move = static_cast(e); if (text_edit_) { forward_drag_event_to_text_edit(drag_move); } if (drag_move->isAccepted()) { return true; } break; } case QEvent::DragLeave: { auto drag_leave = static_cast(e); if (text_edit_) { forward_drag_event_to_text_edit(drag_leave); } else { emit drag_left(drag_leave); } if (drag_leave->isAccepted()) { return true; } break; } case QEvent::Drop: { auto drop = static_cast(e); if (text_edit_) { forward_drag_event_to_text_edit(drop); } else { emit dropped(drop); } if (drop->isAccepted()) { return true; } break; } default: break; } } else if (o == text_edit_) { switch (e->type()) { case QEvent::Paint: update(); return true; default: break; } } return super::eventFilter(o, e); } void ViewerDisplayWidget::on_paint() { const bool backend_neutral = is_backend_neutral(); QPainter bg_painter; bool bg_painter_active = false; if (backend_neutral) { // Backend-neutral path: draw background directly with QPainter. The // image itself will be rendered offscreen, downloaded, and painted below. bg_painter.begin(paint_device()); bg_painter_active = true; bg_painter.fillRect(get_inner_rect(), show_widget_background_ ? palette().window().color() : Qt::black); } else { // Clear background to empty QColor bg_color = show_widget_background_ ? palette().window().color() : Qt::black; renderer()->clear_destination(nullptr, bg_color.redF(), bg_color.greenF(), bg_color.blueF()); } VideoParams device_params; ColorTransformJob ctj; bool have_ctj = false; // We only draw if we have a pipeline if (push_mode_ != k_push_null) { // Draw texture through color transform device_params = get_viewport_params(); if (push_mode_ == k_push_blank) { if (!backend_neutral) { draw_blank(device_params); } } else if (color_service()) { bool drew_backend_neutral_frame = false; if (FramePtr frame = load_frame_.value()) { if (!drew_backend_neutral_frame && (!texture_ || texture_->renderer() != renderer() // Some implementations don't like it if we upload to a texture created in another (albeit shared) context || texture_->width() != frame->width() || texture_->height() != frame->height() || texture_->format() != frame->format() || texture_->channel_count() != frame->channel_count())) { texture_ = renderer()->create_texture( frame->video_params(), frame->data(), frame->linesize_pixels()); } else if (!drew_backend_neutral_frame) { texture_->upload(frame->data(), frame->linesize_pixels()); } } else if (TexturePtr texture = load_frame_.value()) { // This is a GPU texture, switch to it directly when possible. if (!drew_backend_neutral_frame && texture && texture->renderer() && texture->renderer() != renderer()) { if (texture->renderer()->is_open_gl() && renderer()->is_open_gl()) { // Shared OpenGL contexts can display the producer texture // directly. Avoid readback here because the producer // renderer may belong to a render thread whose context // cannot be made current from the GUI paint callback. texture_ = texture; } else { // Cross-backend texture: download and re-upload FramePtr frame = Frame::create(); frame->set_video_params(texture->params()); if (frame->allocate()) { texture->renderer()->download_from_texture( texture->id(), texture->params(), frame->data(), frame->linesize_pixels()); texture_ = renderer()->create_texture( frame->video_params(), frame->data(), frame->linesize_pixels()); } else { texture_ = texture; } } } else if (!drew_backend_neutral_frame) { texture_ = texture; } } else { texture_ = load_custom_texture_from_frame(load_frame_); } if (drew_backend_neutral_frame) { texture_ = nullptr; } emit texture_changed(texture_); push_mode_ = k_push_unnecessary; if (!drew_backend_neutral_frame) { TexturePtr texture_to_draw = texture_; if (!texture_to_draw || texture_to_draw->is_dummy()) { if (!backend_neutral) { draw_blank(device_params); } } else { if (deinterlace_) { if (deinterlace_shader_.isNull()) { deinterlace_shader_ = renderer()->create_native_shader( ShaderCode(FileFunctions::read_file_as_string( QStringLiteral( ":/shaders/deinterlace.frag")))); } if (!deinterlace_texture_ || deinterlace_texture_->params() != texture_to_draw->params()) { // (Re)create texture deinterlace_texture_ = renderer()->create_texture( texture_to_draw->params()); } ShaderJob job; job.insert( QStringLiteral("resolution_in"), NodeValue(NodeValue::k_vec2, QVector2D(texture_to_draw->width(), texture_to_draw->height()))); job.insert( QStringLiteral("ove_maintex"), NodeValue(NodeValue::k_texture, QVariant::fromValue(texture_to_draw))); renderer()->blit_to_texture(deinterlace_shader_, job, deinterlace_texture_.get()); texture_to_draw = deinterlace_texture_; } ctj.set_color_processor(color_service()); ctj.set_input_texture(texture_to_draw); ctj.set_input_alpha_association( OAK_CONFIG("ReassocLinToNonLin").toBool() ? k_alpha_associated : k_alpha_none); ctj.set_clear_destination_enabled(false); ctj.set_transform_matrix(combined_matrix_flipped_); ctj.set_crop_matrix(crop_matrix_); ctj.set_force_opaque(true); have_ctj = true; } } } else { } } if (have_ctj) { if (backend_neutral) { draw_backend_neutral(ctj, &bg_painter); } else { renderer()->blit_color_managed(ctj, device_params); } } if (bg_painter_active) { bg_painter.end(); } // Draw gizmos if we have any if (gizmos_) { QPainter p(paint_device()); generate_gizmo_transforms(); p.setWorldTransform(gizmo_last_draw_transform_); gizmos_->update_gizmo_positions( gizmo_db_, NodeGlobals(gizmo_params_, gizmo_audio_params_, gizmo_draw_time_, LoopMode::k_loop_mode_off)); foreach (NodeGizmo *gizmo, gizmos_->get_gizmos()) { if (gizmo->is_visible()) { gizmo->draw(&p); } } if (text_edit_) { QPixmap pm(text_edit_->width(), text_edit_->height()); pm.fill(Qt::transparent); QPainter pixp(&pm); text_edit_->paint(&pixp, active_text_gizmo_->get_vertical_alignment()); p.drawPixmap(text_edit_pos_, pm); } } // Draw action/title safe areas if (safe_margin_.is_enabled()) { QPainter p(paint_device()); p.setWorldTransform(generate_world_transform()); p.setPen(QPen(Qt::lightGray, 0)); p.setBrush(Qt::NoBrush); int x = 0, y = 0, w = width(), h = height(); if (safe_margin_.custom_ratio()) { double widget_ar = static_cast(width()) / static_cast(height()); if (widget_ar > safe_margin_.ratio()) { // Widget is wider than margins w = h * safe_margin_.ratio(); x = width() / 2 - w / 2; } else { h = w / safe_margin_.ratio(); y = height() / 2 - h / 2; } } p.drawRect(w / 20 + x, h / 20 + y, w / 10 * 9, h / 10 * 9); p.drawRect(w / 10 + x, h / 10 + y, w / 10 * 8, h / 10 * 8); int cross = qMin(w, h) / 32; QLine lines[] = { QLine(rect().center().x() - cross, rect().center().y(), rect().center().x() + cross, rect().center().y()), QLine(rect().center().x(), rect().center().y() - cross, rect().center().x(), rect().center().y() + cross) }; p.drawLines(lines, 2); } if (show_fps_) { { qint64 now = QDateTime::currentMSecsSinceEpoch(); double frame_rate; if (now == fps_timer_start_) { // This will cause a divide by zero, so we do nothing here frame_rate = 0; } else { frame_rate = double(fps_timer_update_count_) / double((now - fps_timer_start_) / 1000.0); } if (frame_rate > 0) { frame_rate_averages_[frame_rate_average_count_ % frame_rate_averages_.size()] = frame_rate; frame_rate_average_count_++; } } if (frame_rate_average_count_ >= frame_rate_averages_.size()) { QPainter p(paint_device()); double average = 0.0; for (int i = 0; i < frame_rate_averages_.size(); i++) { average += frame_rate_averages_[i]; } average /= double(frame_rate_averages_.size()); draw_text_with_crude_shadow( &p, get_inner_rect(), tr("%1 FPS").arg(QString::number(average, 'f', 1))); if (frames_skipped_ > 0) { draw_text_with_crude_shadow( &p, get_inner_rect().adjusted(0, p.fontMetrics().height(), 0, 0), tr("%1 frames skipped").arg(frames_skipped_)); } } } // Extraordinarily basic subtitle renderer. Hoping to swap this out with libass at some point. draw_subtitle_tracks(); if (add_band_) { QPainter p(paint_device()); QColor highlight = palette().highlight().color(); p.setPen(highlight); highlight.setAlpha(128); p.setBrush(highlight); p.drawRect(QRect(add_band_start_, add_band_end_).normalized()); } // In backend-neutral mode there is no native buffer swap, so Qt will not // emit frameSwapped automatically. Emit it ourselves so the playback queue // keeps advancing (UpdateFromQueue is connected to it during Play()). if (backend_neutral) { emit frame_swapped(); } } void ViewerDisplayWidget::on_destroy() { if (!deinterlace_shader_.isNull()) { renderer()->destroy_native_shader(deinterlace_shader_); deinterlace_shader_.clear(); } if (!blank_shader_.isNull()) { renderer()->destroy_native_shader(blank_shader_); blank_shader_.clear(); } super::on_destroy(); texture_ = nullptr; deinterlace_texture_ = nullptr; backend_neutral_texture_ = nullptr; backend_neutral_buffer_.clear(); backend_neutral_cpu_image_ = QImage(); backend_neutral_cpu_display_frame_.reset(); backend_neutral_cpu_source_frame_.reset(); backend_neutral_cpu_source_texture_.reset(); backend_neutral_cpu_color_id_.clear(); if (load_frame_.isNull()) { push_mode_ = k_push_null; } else { push_mode_ = k_push_frame; } } QPointF ViewerDisplayWidget::get_texture_position(const QPoint &screen_pos) { return get_texture_position(screen_pos.x(), screen_pos.y()); } QPointF ViewerDisplayWidget::get_texture_position(const QSize &size) { return get_texture_position(size.width(), size.height()); } QPointF ViewerDisplayWidget::get_texture_position(const double &x, const double &y) { return QPointF(x / gizmo_params_.width(), y / gizmo_params_.height()); } void ViewerDisplayWidget::draw_text_with_crude_shadow(QPainter *painter, const QRect &rect, const QString &text, const QTextOption &opt) { painter->setPen(Qt::black); painter->drawText(rect.adjusted(1, 1, 0, 0), text, opt); painter->setPen(Qt::white); painter->drawText(rect, text, opt); } Rational ViewerDisplayWidget::get_gizmo_time() { return get_adjusted_time(get_time_target(), gizmos_, time_, Node::k_transform_towards_input); } bool ViewerDisplayWidget::is_hand_drag(QMouseEvent *event) const { return event->button() == Qt::MiddleButton || Core::instance()->tool() == Tool::k_hand; } void ViewerDisplayWidget::update_matrix() { combined_matrix_ = scale_matrix_ * translate_matrix_; combined_matrix_flipped_ = combined_matrix_; // OpenGL's framebuffer origin is bottom-left and texture data is uploaded // top-down, so the viewer matrix must flip Y to display images right-side // up. Vulkan's framebuffer and texture coordinate origins are both top-left, // so the same flip would invert the image. Default to the OpenGL flip when // no renderer is available yet. if (!renderer() || !renderer()->is_vulkan()) { QMatrix4x4 flip; flip.scale(1.0f, -1.0f, 1.0f); combined_matrix_flipped_ = flip * combined_matrix_flipped_; } update(); } QTransform ViewerDisplayWidget::generate_world_transform() { /* * Get matrix elements (roughly) as below in column major order * * | Sx 0 0 Tx | * | 0 Sy 0 Ty | * | 0 0 Sz Tz | * | 0 0 0 1 | */ float *d = combined_matrix_.data(); QTransform world; // Move corner of canvas to correct point world.translate(width() * 0.5 - width() * *(d) * 0.5, height() * 0.5 - height() * *(d + 5) * 0.5); // Scale world.scale(*(d), *(d + 5)); // Translate for mouse movement world.translate(*(d + 12) * width() * 0.5 / *(d), *(d + 13) * height() * 0.5 / *(d + 5)); return world; } QTransform ViewerDisplayWidget::generate_display_transform() { QVector2D viewer_scale(get_texture_position(size())); QTransform gizmo_transform = generate_world_transform(); gizmo_transform.scale(viewer_scale.x(), viewer_scale.y()); gizmo_transform.scale( gizmo_params_.pixel_aspect_ratio().flipped().to_double(), 1); return gizmo_transform; } QTransform ViewerDisplayWidget::generate_gizmo_transform(NodeTraverser >, const TimeRange &range) { QTransform t = generate_display_transform(); if (get_time_target()) { Node *target = get_time_target(); if (ViewerOutput *v = dynamic_cast(target)) { if (Node *n = v->get_connected_texture_output()) { target = n; } } QTransform nt; gt.transform(&nt, gizmos_, target, range); t.translate(gizmo_params_.width() * 0.5, gizmo_params_.height() * 0.5); t.scale(gizmo_params_.width(), gizmo_params_.height()); t = nt * t; t.scale(1.0 / gizmo_params_.width(), 1.0 / gizmo_params_.height()); t.translate(-gizmo_params_.width() * 0.5, -gizmo_params_.height() * 0.5); } return t; } NodeGizmo *ViewerDisplayWidget::try_gizmo_press(const NodeValueRow &row, const QPointF &p) { if (!gizmos_) { return nullptr; } for (auto it = gizmos_->get_gizmos().crbegin(); it != gizmos_->get_gizmos().crend(); it++) { NodeGizmo *gizmo = *it; if (gizmo->is_visible()) { if (PointGizmo *point = dynamic_cast(gizmo)) { if (point->get_clicking_rect(gizmo_last_draw_transform_) .contains(p)) { return point; } } else if (PolygonGizmo *poly = dynamic_cast(gizmo)) { if (poly->get_polygon().containsPoint(p, Qt::OddEvenFill)) { return poly; } } else if (PathGizmo *path = dynamic_cast(gizmo)) { if (path->get_path().contains(p)) { return path; } } else if (ScreenGizmo *screen = dynamic_cast(gizmo)) { // NOTE: Perhaps this should limit to the actual visible screen space? We'll see. return screen; } } } return nullptr; } void ViewerDisplayWidget::open_text_gizmo(TextGizmo *text, QMouseEvent *event) { generate_gizmo_transforms(); gizmos_->update_gizmo_positions( gizmo_db_, NodeGlobals(gizmo_params_, gizmo_audio_params_, gizmo_draw_time_, LoopMode::k_loop_mode_off)); active_text_gizmo_ = text; connect(active_text_gizmo_, &TextGizmo::rect_changed, this, &ViewerDisplayWidget::update_active_text_gizmo_size); text_transform_ = generate_gizmo_transform(); text_transform_inverted_ = text_transform_.inverted(); // Create text editor text_edit_ = new ViewerTextEditor(text_transform_.m11(), this); // Set text editor's gizmo property for later use text_edit_->setProperty("gizmo", reinterpret_cast(text)); // Install ourselves as event filter so we can receive the text editor's paint events text_edit_->installEventFilter(this); // Disable focus on text editor text_edit_->setFocusPolicy(Qt::NoFocus); // Disable mouse events on text editor text_edit_->setAttribute(Qt::WA_TransparentForMouseEvents); // "Show" text editor so that it throws paint events, even though its paint event is disabled text_edit_->show(); // Convert HTML to Qt document Html::html_to_doc(text_edit_->document(), text->get_html()); // Connect text change event to propagate back to node connect(text_edit_, &ViewerTextEditor::textChanged, this, &ViewerDisplayWidget::text_edit_changed); // Connect destroyed signal to cleanup after destruction connect(text_edit_, &ViewerTextEditor::destroyed, this, &ViewerDisplayWidget::text_edit_destroyed); // Set text editor's size to logical size QRectF text_rect = update_active_text_gizmo_size(); // Emit text gizmo activation signal emit text->activated(); // Create toolbar text_toolbar_ = new ViewerTextEditorToolBar(text_edit_); text_toolbar_->setWindowFlags(Qt::Tool | Qt::FramelessWindowHint); connect(text_toolbar_, &ViewerTextEditorToolBar::vertical_alignment_changed, text, &TextGizmo::set_vertical_alignment); connect(text, &TextGizmo::vertical_alignment_changed, text_toolbar_, &ViewerTextEditorToolBar::set_vertical_alignment); text_toolbar_->set_vertical_alignment(text->get_vertical_alignment()); text_edit_->connect_tool_bar(text_toolbar_); QPoint toolbar_pos = mapToGlobal(text_transform_.map(text_edit_pos_).toPoint()); if (QScreen *screen = qApp->screenAt(toolbar_pos)) { // Determine whether to anchor to the top of the rect of the bottom if (toolbar_pos.y() - text_toolbar_->height() >= screen->geometry().top()) { toolbar_pos.setY(toolbar_pos.y() - text_toolbar_->height()); } else { toolbar_pos.setY( toolbar_pos.y() + text_transform_.map(text_rect).boundingRect().height()); } // Clamp X if (toolbar_pos.x() + text_toolbar_->width() > screen->geometry().right()) { toolbar_pos.setX(screen->geometry().right() - text_toolbar_->width()); } // Clamp Y if (toolbar_pos.y() + text_toolbar_->height() > screen->geometry().bottom()) { toolbar_pos.setY(screen->geometry().bottom() - text_toolbar_->height()); } } else { // Fallback toolbar_pos.setY(toolbar_pos.y() - text_toolbar_->height()); } text_toolbar_->move(toolbar_pos); text_toolbar_->show(); // Allow widget to take keyboard focus inner_widget()->setFocusPolicy(Qt::StrongFocus); inner_widget()->setMouseTracking(true); connect(qApp, &QApplication::focusChanged, this, &ViewerDisplayWidget::focus_changed); // Start text cursor where the user clicked if (event) { QPoint click_pos = text_transform_inverted_.map(event->pos()) - text_edit_pos_.toPoint(); text_edit_->setTextCursor(text_edit_->cursorForPosition(click_pos)); } // Grab focus back from the toolbar connect(text_toolbar_, &ViewerTextEditorToolBar::first_paint, this, [this] { Core::instance()->main_window()->activateWindow(); inner_widget()->setFocus(); }); } bool ViewerDisplayWidget::on_mouse_press(QMouseEvent *event) { if (is_hand_drag(event)) { // Handle hand drag hand_last_drag_pos_ = event->pos(); hand_dragging_ = true; emit hand_drag_started(); inner_widget()->setCursor(Qt::ClosedHandCursor); return true; } else if (text_edit_ && forward_mouse_event_to_text_edit(event, true)) { return true; } else if (event->button() == Qt::LeftButton) { if (Core::instance()->tool() == Tool::k_add && (Core::instance()->get_selected_addable_object() == Tool::k_addable_shape || Core::instance()->get_selected_addable_object() == Tool::k_addable_title)) { add_band_start_ = event->pos(); add_band_end_ = add_band_start_; add_band_ = true; } else if ((current_gizmo_ = try_gizmo_press( gizmo_db_, gizmo_last_draw_transform_inverted_.map( event->pos())))) { // Handle gizmo click gizmo_start_drag_ = event->pos(); gizmo_last_drag_ = gizmo_start_drag_; current_gizmo_->set_globals( NodeGlobals(gizmo_params_, gizmo_audio_params_, generate_gizmo_time(), LoopMode::k_loop_mode_off)); } else { // Handle standard drag emit drag_started(event->pos()); } return true; } return false; } bool ViewerDisplayWidget::on_mouse_move(QMouseEvent *event) { // Handle hand dragging if (hand_dragging_) { // Emit movement emit hand_drag_moved(event->x() - hand_last_drag_pos_.x(), event->y() - hand_last_drag_pos_.y()); hand_last_drag_pos_ = event->pos(); return true; } else if (text_edit_ && forward_mouse_event_to_text_edit(event)) { return true; } else if (add_band_) { add_band_end_ = event->pos(); update(); return true; } else if (current_gizmo_) { // Signal movement if (DraggableGizmo *draggable = dynamic_cast(current_gizmo_)) { if (!gizmo_drag_started_) { QPointF start = screen_to_scene_point(gizmo_start_drag_); Rational gizmo_time = get_gizmo_time(); NodeTraverser t; t.set_cache_video_params(gizmo_params_); t.set_cache_audio_params(gizmo_audio_params_); NodeValueRow row = t.generate_row( gizmos_, TimeRange(gizmo_time, gizmo_time + gizmo_params_.frame_rate_as_time_base())); draggable->drag_start(row, start.x(), start.y(), gizmo_time); gizmo_drag_started_ = true; } QPointF v = screen_to_scene_point(event->pos()); switch (draggable->get_drag_value_behavior()) { case DraggableGizmo::k_absolute: // Above value is correct break; case DraggableGizmo::k_delta_from_previous: v -= screen_to_scene_point(gizmo_last_drag_); gizmo_last_drag_ = event->pos(); break; case DraggableGizmo::k_delta_from_start: v -= screen_to_scene_point(gizmo_start_drag_); break; } draggable->drag_move(v.x(), v.y(), event->modifiers()); return true; } } return false; } bool ViewerDisplayWidget::on_mouse_release(QMouseEvent *e) { if (hand_dragging_) { // Handle hand drag emit hand_drag_ended(); hand_dragging_ = false; update_cursor(); return true; } else if (text_edit_ && forward_mouse_event_to_text_edit(e)) { return true; } else if (add_band_) { QRect band_rect = QRect(add_band_start_, add_band_end_).normalized(); if (band_rect.width() > 1 && band_rect.height() > 1) { QRectF r = generate_display_transform().inverted().mapRect(band_rect); emit create_addable_at(r); } add_band_ = false; return true; } else if (current_gizmo_) { // Handle gizmo if (gizmo_drag_started_) { MultiUndoCommand *command = new MultiUndoCommand(); if (DraggableGizmo *draggable = dynamic_cast(current_gizmo_)) { draggable->drag_end(command); } Core::instance()->undo_stack()->push(command, tr("Dragged Gizmo")); gizmo_drag_started_ = false; } current_gizmo_ = nullptr; return true; } return false; } bool ViewerDisplayWidget::on_mouse_double_click(QMouseEvent *event) { if (text_edit_ && forward_mouse_event_to_text_edit(event)) { return true; } else if (event->button() == Qt::LeftButton && gizmos_) { QPointF ptr = transform_viewer_space_to_buffer_space(event->pos()); foreach (NodeGizmo *g, gizmos_->get_gizmos()) { if (TextGizmo *text = dynamic_cast(g)) { if (text->get_rect().contains(ptr)) { open_text_gizmo(text, event); return true; } } } } return false; } bool ViewerDisplayWidget::on_key_press(QKeyEvent *e) { if (text_edit_) { if (e->key() == Qt::Key_Escape) { close_text_editor(); return true; } else { return forward_event_to_text_edit(e); } } return false; } bool ViewerDisplayWidget::on_key_release(QKeyEvent *e) { if (text_edit_) { return forward_event_to_text_edit(e); } return false; } void ViewerDisplayWidget::emit_color_at_cursor(QMouseEvent *e) { // Do this no matter what, emits signal to any pixel samplers if (signal_cursor_color_) { Color reference, display; if (texture_) { QPointF pixel_pos = generate_display_transform().inverted().map(e->pos()); pixel_pos /= texture_->params().divider(); make_current(); reference = renderer()->get_pixel_from_texture(texture_.get(), pixel_pos); if (color_service()) { display = color_service()->convert_color(reference); } else { display = reference; } } emit cursor_color(reference, display); } } void ViewerDisplayWidget::draw_subtitle_tracks() { if (!show_subtitles_ || !subtitle_tracks_) { return; } const QVector &subtitle_tracklist = subtitle_tracks_->track_list(Track::k_subtitle)->get_tracks(); if (subtitle_tracklist.empty()) { return; } // Scale font size by transform QTransform display_transform = generate_display_transform(); qreal font_sz = OAK_CONFIG("DefaultSubtitleSize").toInt(); font_sz *= display_transform.m11(); if (qIsNaN(font_sz)) { return; } QPainterPath path; QTransform transform = generate_world_transform(); QRect bounding_box = transform.mapRect(rect()); QFont f; f.setPointSizeF(font_sz); QString family = OAK_CONFIG("DefaultSubtitleFamily").toString(); if (!family.isEmpty()) { f.setFamily(family); } f.setWeight(static_cast( OAK_CONFIG("DefaultSubtitleWeight").toInt())); bounding_box.adjust(bounding_box.width() / 10, bounding_box.height() / 10, -bounding_box.width() / 10, -bounding_box.height() / 10); QFontMetrics fm(f); for (int j = subtitle_tracklist.size() - 1; j >= 0; j--) { Track *sub_track = subtitle_tracklist.at(j); if (!sub_track->is_muted()) { if (SubtitleBlock *sub = dynamic_cast( sub_track->visible_block_at_time(time_))) { // Split into lines QStringList list = QtUtils::word_wrap_string( sub->get_text(), fm, bounding_box.width()); for (int i = list.size() - 1; i >= 0; i--) { int w = QtUtils::q_font_metrics_width(fm, list.at(i)); path.addText(bounding_box.width() / 2 - w / 2, bounding_box.height() - fm.height() * (list.size() - i) + fm.ascent(), f, list.at(i)); } } } } bool antialias = OAK_CONFIG("AntialiasSubtitles").toBool(); QPixmap *aa_pixmap; QPainter *text_painter; if (antialias) { // QPainter only supports anti-aliasing in software, so to achieve it, we draw to a // software buffer first and then draw that onto the hardware aa_pixmap = new QPixmap(bounding_box.width(), bounding_box.height()); aa_pixmap->fill(Qt::transparent); text_painter = new QPainter(aa_pixmap); } else { // Just draw straight to the hardware text_painter = new QPainter(paint_device()); // Offset path by however much is necessary path.translate(bounding_box.x(), bounding_box.y()); } text_painter->setPen(QPen(Qt::black, f.pointSizeF() / 16)); text_painter->setBrush(Qt::white); text_painter->setRenderHint(QPainter::Antialiasing); text_painter->drawPath(path); delete text_painter; if (antialias) { // We just drew to a software buffer, now draw this image onto the hardware device QPainter p(paint_device()); p.drawPixmap(bounding_box.x(), bounding_box.y(), *aa_pixmap); delete aa_pixmap; } } template void ViewerDisplayWidget::forward_drag_event_to_text_edit(T *e) { // HACK: Absolutely filthy hack. We need to be able to transform the mouse coordinates for our // proxied QTextEdit, however unlike QMouseEvents, Qt's drag events don't allow modifying // the position after construction. Unhelpfully, Qt also explicitly forbids users creating // their own drag events because they "rely on Qt's internal state". So in order to forward // drag events, we defy this by creating our own events, but DON'T process them through Qt's // event queue and instead just send them directly to the widget (requiring its protected // drag events to be made public). That way Qt stays happy, because as far as it's // concerned it's only interfacing with this widget, and the QTextEdit gets to receive // transformed events. It's a terrible hack, but seems to work. if constexpr (std::is_same_v) { text_edit_->dragLeaveEvent(e); } else { T relay(adjust_pos_by_v_align(get_virtual_pos_for_text_edit(e->pos())).toPoint(), e->possibleActions(), e->mimeData(), e->mouseButtons(), e->keyboardModifiers()); if (e->type() == QEvent::DragEnter) { text_edit_->dragEnterEvent(static_cast(&relay)); } else if (e->type() == QEvent::DragMove) { text_edit_->dragMoveEvent(static_cast(&relay)); } else if (e->type() == QEvent::Drop) { text_edit_->dropEvent(&relay); } if (relay.isAccepted()) { e->accept(); } } } bool ViewerDisplayWidget::forward_mouse_event_to_text_edit(QMouseEvent *event, bool check_if_outside) { if (current_gizmo_) { return false; } // Transform screen mouse coords to world mouse coords QPointF local_pos = get_virtual_pos_for_text_edit(event->pos()); if (event->type() == QEvent::MouseMove && event->buttons() == Qt::NoButton) { QPointF mapped = text_transform_inverted_.map(event->pos()) - text_edit_pos_; if (mapped.x() >= 0 && mapped.y() >= 0 && mapped.x() < text_edit_->width() && mapped.y() < text_edit_->height()) { inner_widget()->setCursor(Qt::IBeamCursor); } else { inner_widget()->unsetCursor(); } } if (check_if_outside) { if (local_pos.x() < 0 || local_pos.x() >= text_edit_->width() || local_pos.y() < 0 || local_pos.y() >= text_edit_->height()) { // Allow clicking other gizmos so the user can resize while the text editor is active if ((current_gizmo_ = try_gizmo_press( gizmo_db_, gizmo_last_draw_transform_inverted_.map(event->pos())))) { return false; } else { close_text_editor(); return true; } } } local_pos = adjust_pos_by_v_align(local_pos); QMouseEvent derived(event->type(), local_pos, event->windowPos(), event->screenPos(), event->button(), event->buttons(), event->modifiers(), event->source()); return forward_event_to_text_edit(&derived); } bool ViewerDisplayWidget::forward_event_to_text_edit(QEvent *event) { qApp->sendEvent(text_edit_->viewport(), event); bool e = event->isAccepted(); if (e) { update(); } return e; } QPointF ViewerDisplayWidget::adjust_pos_by_v_align(QPointF p) { switch (active_text_gizmo_->get_vertical_alignment()) { case Qt::AlignTop: // Do nothing break; case Qt::AlignVCenter: p.setY(p.y() - text_edit_->height() / 2 + text_edit_->document()->size().height() / 2); break; case Qt::AlignBottom: p.setY(p.y() - text_edit_->height() + text_edit_->document()->size().height()); break; } return p; } void ViewerDisplayWidget::close_text_editor() { text_edit_->deleteLater(); text_edit_ = nullptr; disconnect(active_text_gizmo_, &TextGizmo::rect_changed, this, &ViewerDisplayWidget::update_active_text_gizmo_size); active_text_gizmo_ = nullptr; } void ViewerDisplayWidget::generate_gizmo_transforms() { NodeTraverser gt; gt.set_cache_video_params(gizmo_params_); gt.set_cache_audio_params(gizmo_audio_params_); gizmo_draw_time_ = generate_gizmo_time(); if (gizmos_) { gizmo_db_ = gt.generate_row(gizmos_, gizmo_draw_time_); } gizmo_last_draw_transform_ = generate_gizmo_transform(gt, gizmo_draw_time_); gizmo_last_draw_transform_inverted_ = gizmo_last_draw_transform_.inverted(); } void ViewerDisplayWidget::draw_blank(const VideoParams &device_params) { if (blank_shader_.isNull()) { blank_shader_ = renderer()->create_native_shader(ShaderCode()); } ShaderJob job; job.insert(QStringLiteral("ove_mvpmat"), NodeValue(NodeValue::k_matrix, combined_matrix_flipped_)); job.insert(QStringLiteral("ove_cropmatrix"), NodeValue(NodeValue::k_matrix, crop_matrix_)); renderer()->blit(blank_shader_, job, device_params, false); } bool ViewerDisplayWidget::draw_backend_neutral_frame(const FramePtr &frame, QPainter *painter) { if (!frame || !frame->is_allocated() || !painter || !painter->isActive() || !color_service()) { return false; } const QString color_id = QString::fromUtf8(color_service()->id()); if (backend_neutral_cpu_source_frame_.get() == frame.get() && backend_neutral_cpu_color_id_ == color_id && !backend_neutral_cpu_image_.isNull()) { painter->save(); painter->setRenderHint(QPainter::SmoothPixmapTransform, true); painter->setWorldTransform(generate_world_transform(), false); painter->drawImage(rect(), backend_neutral_cpu_image_); painter->restore(); return true; } // Do not run OCIO CPU conversion from paintEvent. Some OCIO processors are // not safe to apply on this GUI path and a crash here kills preview. Worker // frames tagged with display: have already been color managed; // untagged frames are drawn directly as a safe fallback. FramePtr display_frame = frame; QImage source_image; if (display_frame->format() == PixelFormat::u8 && display_frame->channel_count() == VideoParams::k_rgba_channel_count) { backend_neutral_cpu_display_frame_ = display_frame; backend_neutral_cpu_image_ = QImage(reinterpret_cast(display_frame->const_data()), display_frame->width(), display_frame->height(), display_frame->linesize_bytes(), QImage::Format_RGBA8888); source_image = backend_neutral_cpu_image_; } else if (display_frame->format() == PixelFormat::u8 && display_frame->channel_count() == VideoParams::k_rgb_channel_count) { backend_neutral_cpu_display_frame_ = display_frame; backend_neutral_cpu_image_ = QImage(reinterpret_cast(display_frame->const_data()), display_frame->width(), display_frame->height(), display_frame->linesize_bytes(), QImage::Format_RGB888); source_image = backend_neutral_cpu_image_; } else { backend_neutral_cpu_display_frame_.reset(); const int bytes_per_pixel = display_frame->video_params().get_bytes_per_pixel(); if (backend_neutral_cpu_image_.size() != QSize(display_frame->width(), display_frame->height()) || backend_neutral_cpu_image_.format() != QImage::Format_RGBA8888) { backend_neutral_cpu_image_ = QImage(display_frame->width(), display_frame->height(), QImage::Format_RGBA8888); } for (int y = 0; y < display_frame->height(); ++y) { uchar *dst = backend_neutral_cpu_image_.scanLine(y); const char *src = display_frame->const_data() + y * display_frame->linesize_bytes(); for (int x = 0; x < display_frame->width(); ++x) { Color c(src + x * bytes_per_pixel, display_frame->format(), display_frame->channel_count()); dst[x * 4 + 0] = static_cast(qBound(0, int(c.red() * 255.0), 255)); dst[x * 4 + 1] = static_cast(qBound(0, int(c.green() * 255.0), 255)); dst[x * 4 + 2] = static_cast(qBound(0, int(c.blue() * 255.0), 255)); dst[x * 4 + 3] = 255; } } source_image = backend_neutral_cpu_image_; } backend_neutral_cpu_source_frame_ = frame; backend_neutral_cpu_color_id_ = color_id; painter->save(); painter->setRenderHint(QPainter::SmoothPixmapTransform, true); painter->setWorldTransform(generate_world_transform(), false); painter->drawImage(rect(), source_image); painter->restore(); return true; } bool ViewerDisplayWidget::draw_backend_neutral_texture(const TexturePtr &texture, QPainter *painter) { if (!texture || texture->is_dummy() || !texture->renderer() || !painter || !painter->isActive() || !color_service()) { return false; } const QString color_id = QString::fromUtf8(color_service()->id()); if (backend_neutral_cpu_source_texture_.get() == texture.get() && backend_neutral_cpu_color_id_ == color_id && !backend_neutral_cpu_image_.isNull()) { painter->save(); painter->setRenderHint(QPainter::SmoothPixmapTransform, true); painter->setWorldTransform(generate_world_transform(), false); painter->drawImage(rect(), backend_neutral_cpu_image_); painter->restore(); return true; } FramePtr frame = Frame::create(); frame->set_video_params(texture->params()); if (!frame->allocate()) { return false; } texture->download(frame->data(), frame->linesize_pixels()); if (!draw_backend_neutral_frame(frame, painter)) { return false; } backend_neutral_cpu_source_texture_ = texture; backend_neutral_cpu_color_id_ = color_id; return true; } // Renders a backend-neutral frame by drawing into an offscreen backend texture, // downloading it to CPU memory, then painting that image with QPainter. void ViewerDisplayWidget::draw_backend_neutral(const ColorTransformJob &ctj, QPainter *painter) { if (!painter || !painter->isActive()) { return; } const int texture_width = static_cast(width() * devicePixelRatioF()); const int texture_height = static_cast(height() * devicePixelRatioF()); const VideoParams offscreen_params(texture_width, texture_height, PixelFormat::u8, VideoParams::k_rgba_channel_count); if (!backend_neutral_texture_ || backend_neutral_texture_->params() != offscreen_params) { // The offscreen texture is sized in device pixels so high-DPI widgets // draw one downloaded pixel per device pixel after setDevicePixelRatio(). backend_neutral_texture_ = renderer()->create_texture(offscreen_params); backend_neutral_buffer_.resize( texture_width * texture_height * VideoParams::get_bytes_per_pixel(PixelFormat::u8, VideoParams::k_rgba_channel_count)); } if (!backend_neutral_texture_ || backend_neutral_texture_->is_dummy()) { return; } ColorTransformJob local_ctj = ctj; local_ctj.set_clear_destination_enabled(true); // Reuse the normal color-management shader path, but render into a texture // instead of an OpenGL widget framebuffer. renderer()->blit_color_managed(local_ctj, backend_neutral_texture_.get()); backend_neutral_texture_->download(backend_neutral_buffer_.data(), 0); const int bytes_per_pixel = VideoParams::get_bytes_per_pixel( PixelFormat::u8, VideoParams::k_rgba_channel_count); QImage img( reinterpret_cast(backend_neutral_buffer_.constData()), texture_width, texture_height, texture_width * bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied); img.setDevicePixelRatio(devicePixelRatioF()); // QImage references backend_neutral_buffer_ directly; draw it before the // buffer can be resized or reused by a later paint. painter->drawImage(QPoint(0, 0), img); } void ViewerDisplayWidget::set_show_fps(bool e) { show_fps_ = e; update(); } void ViewerDisplayWidget::request_start_editing_text() { if (gizmos_) { foreach (NodeGizmo *gizmo, gizmos_->get_gizmos()) { if (TextGizmo *text = dynamic_cast(gizmo)) { open_text_gizmo(text); break; } } } } void ViewerDisplayWidget::play(const int64_t &start_timestamp, const int &playback_speed, const Rational &timebase, bool start_updating) { playback_timebase_ = timebase; playback_speed_ = playback_speed; // The facade playback engine owns the master clock; seed the display // clock with the start timestamp (the ViewerWidget keeps feeding it // from oakengine_playback_get_position afterwards). external_ts_.store(start_timestamp); if (start_updating) { connect(this, &ViewerDisplayWidget::frame_swapped, this, &ViewerDisplayWidget::update_from_queue); update(); } } void ViewerDisplayWidget::pause() { disconnect(this, &ViewerDisplayWidget::frame_swapped, this, &ViewerDisplayWidget::update_from_queue); external_ts_.store(-1); queue_.clear(); queue_starved_ = false; } QPointF ViewerDisplayWidget::screen_to_scene_point(const QPoint &p) { if (gizmo_last_draw_transform_.isIdentity()) { generate_gizmo_transforms(); } return p * gizmo_last_draw_transform_inverted_; } void ViewerDisplayWidget::update_from_queue() { const int64_t t = external_ts_.load(); if (t < 0) { // No facade playback position fed yet. return; } Rational time = Timecode::timestamp_to_time(t, playback_timebase_); bool popped = false; if (queue_.empty()) { queue_starved_ = true; emit queue_starved(); } else { while (!queue_.empty()) { const ViewerPlaybackFrame &pf = queue_.front(); if (pf.timestamp == time) { // Frame was in queue, no need to decode anything set_image(pf.frame); if (queue_starved_) { queue_starved_ = false; emit queue_no_longer_starved(); } return; } else if ((pf.timestamp > time) == (playback_speed_ > 0)) { // The next frame in the queue is too new, so just do a regular update. Either the // frame we want will arrive in time, or we'll just have to skip it. break; } else { queue_.pop_front(); if (popped) { // We've already popped a frame in this loop, meaning a frame has been skipped increment_skipped_frames(); } else { // Shown a frame and progressed to the next one increment_frame_count(); popped = true; } if (queue_.empty()) { queue_starved_ = true; emit queue_starved(); break; } } } } update(); } void ViewerDisplayWidget::text_edit_changed() { ViewerTextEditor *editor = static_cast(sender()); TextGizmo *gizmo = reinterpret_cast( editor->property("gizmo").value()); QString html = Html::doc_to_html(editor->document()); gizmo->update_input_html(html, get_gizmo_time()); } void ViewerDisplayWidget::text_edit_destroyed() { TextGizmo *gizmo = reinterpret_cast( sender()->property("gizmo").value()); emit gizmo->deactivated(); text_edit_ = nullptr; text_toolbar_ = nullptr; inner_widget()->setMouseTracking(false); inner_widget()->setFocusPolicy(Qt::NoFocus); update_cursor(); disconnect(qApp, &QApplication::focusChanged, this, &ViewerDisplayWidget::focus_changed); } void ViewerDisplayWidget::subtitles_changed(const TimeRange &r) { if (time_ >= r.in() && time_ < r.out()) { update(); } } void ViewerDisplayWidget::focus_changed(QWidget *old, QWidget *now) { if (!now) { // Ignore this return; } bool unfocused = true; while (now) { if (now == text_toolbar_ || now == this) { unfocused = false; break; } else { now = now->parentWidget(); } } if (unfocused) { close_text_editor(); } } QRectF ViewerDisplayWidget::update_active_text_gizmo_size() { QRectF text_rect = active_text_gizmo_->get_rect(); text_edit_pos_ = text_rect.topLeft(); text_edit_->setGeometry(text_rect.toRect()); return text_rect; } }