From 55bd1132cd93880ff1249b5ba6986eaf52bf9c63 Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Thu, 20 Aug 2026 00:28:14 +0800 Subject: [PATCH] feat(app): OFX Interact viewer integration - overlay drawing and event forwarding - Main-process interact instances for the selected OFX effect card (create on selection change, describe, destroy on deselect/close), coexisting with the render-worker plugin instances per the OFX multi-instance model. - Program viewer composites the interact's overlay: draw into a GL FBO via gl_bridge, read back, straight-alpha 'over' composite onto the displayed frame; cached and only re-rendered on frame/time/ viewport/instance change or plugin redraw requests. - Event forwarding: picture-area pointer maps through the contain-fit letterbox inverse to OFX pen coordinates (pen_motion/down/up); Keystroke to OFX key symbols (ASCII, navigation, F1-F35) for key_down/up; a 50ms idle pump; global shortcut consumption keeps precedence. - e2e with the real test plugin: lifecycle marker assertions, pen/key event records, and macOS GL overlay compositing verified (265 tests green incl. gpui_widgets viewer suite). --- Cargo.toml | 1 + build.rs | 41 ++ crates/oakplugin/tests/interact_test.rs | 4 +- src/i18n.rs | 10 + src/oakui/engine.rs | 11 + src/oakui/ofx.rs | 775 +++++++++++++++++++++++- src/oakui/real.rs | 20 + src/panels/ofx_params.rs | 672 +++++++++++++++++++- src/panels/program_viewer.rs | 258 +++++++- src/panels/source_viewer.rs | 4 + 10 files changed, 1761 insertions(+), 35 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 786f9b397..42749c046 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ path = "src/lib.rs" # Doctests are disabled: the app links the oak* module crates (which carry # media/codec dependencies); the doc examples' assertions are covered by # unit tests instead (see `oakui/timecode`). +doctest = false [[bin]] name = "oak-editor" diff --git a/build.rs b/build.rs index d42e1e376..6aeabd549 100644 --- a/build.rs +++ b/build.rs @@ -37,4 +37,45 @@ fn main() { // at the real system library. println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib"); } + // --- OFX interact end-to-end test plugin --------------------------------- + // The app-side interact tests (src/oakui/ofx.rs) drive the *real* + // minimal test plugin (crates/oakplugin/cbits/oak_test_plugin.c) through + // the app's interact wiring. oakplugin compiles the same C file into its + // own OUT_DIR, but build-script env vars do not cross crates, so compile + // it here too — the test assembles a plugin bundle from the app's + // OUT_DIR. + println!("cargo:rerun-if-changed=crates/oakplugin/cbits/oak_test_plugin.c"); + build_test_plugin(&os); +} + +/// Compiles the minimal OFX test plugin as a shared library into +/// `$OUT_DIR/oak_test_plugin.{dylib,so}` (same recipe as oakplugin's +/// build.rs). Only the interact branch of the plugin is exercised by the +/// app-side tests; the GL parts are macOS-gated inside the C source. +fn build_test_plugin(os: &str) { + use std::process::Command; + let out = std::env::var("OUT_DIR").expect("OUT_DIR"); + let cc = std::env::var("CC").unwrap_or_else(|_| "cc".into()); + let (link_flag, ext) = if os == "macos" { + ("-dynamiclib", "dylib") + } else { + ("-shared", "so") + }; + let mut args = vec![ + "-fPIC".to_string(), + "-Icrates/oakplugin/ofx".to_string(), + "crates/oakplugin/cbits/oak_test_plugin.c".to_string(), + link_flag.to_string(), + "-o".to_string(), + format!("{out}/oak_test_plugin.{ext}"), + ]; + if os == "macos" { + args.push("-framework".into()); + args.push("OpenGL".into()); + } + let status = Command::new(&cc) + .args(&args) + .status() + .expect("compile OFX test plugin failed"); + assert!(status.success(), "OFX test plugin compile failed"); } diff --git a/crates/oakplugin/tests/interact_test.rs b/crates/oakplugin/tests/interact_test.rs index d8fbe373c..4dca2e291 100644 --- a/crates/oakplugin/tests/interact_test.rs +++ b/crates/oakplugin/tests/interact_test.rs @@ -318,8 +318,10 @@ fn interact_draw_renders_plugin_colours() { ); // 像素断言：矩形区域 (10..30)² 的 GL 像素 → 翻转后 frame y 33..53。 + // F32 RGBA 每像素 16 字节（紧凑行）。 + let stride = (w as usize) * 16; let px = |x: usize, y: usize| -> [f32; 4] { - let p = &img.pixels()[y * (w as usize) * 4 + x * 4..y * (w as usize) * 4 + x * 4 + 4]; + let p = &img.pixels()[y * stride + x * 16..y * stride + x * 16 + 16]; [ f32::from_ne_bytes(p[0..4].try_into().unwrap()), f32::from_ne_bytes(p[4..8].try_into().unwrap()), diff --git a/src/i18n.rs b/src/i18n.rs index 4dd503269..d8d4293fb 100644 --- a/src/i18n.rs +++ b/src/i18n.rs @@ -423,6 +423,11 @@ const EN: &[(&str, &str)] = &[ ("inspector.badge.openfx", "OpenFX"), // --- OpenFX progress --- ("ofx.progress.title", "OpenFX Plugin Progress"), + // --- OpenFX color picker --- + ("ofx.color.hex", "Hex"), + ("ofx.color.ok", "OK"), + ("ofx.color.cancel", "Cancel"), + ("ofx.color.invalid", "Invalid hex color"), // --- dialogs --- ("dialog.cancel", "Cancel"), ("dialog.close", "Close"), @@ -893,6 +898,11 @@ const ZH: &[(&str, &str)] = &[ ("inspector.badge.openfx", "OpenFX"), // --- OpenFX progress --- ("ofx.progress.title", "OpenFX 插件进度"), + // --- OpenFX 取色器 --- + ("ofx.color.hex", "十六进制"), + ("ofx.color.ok", "确定"), + ("ofx.color.cancel", "取消"), + ("ofx.color.invalid", "无效的十六进制颜色"), // --- dialogs --- ("dialog.cancel", "取消"), ("dialog.close", "关闭"), diff --git a/src/oakui/engine.rs b/src/oakui/engine.rs index 199b99bd3..c4665847d 100644 --- a/src/oakui/engine.rs +++ b/src/oakui/engine.rs @@ -383,6 +383,17 @@ pub trait AppEngine: Err("effect push button not supported".into()) } + /// The plugin instance handle (the oakplugin registry key) of the OFX + /// effect the program viewer's interact should target — the inspector's + /// current selection, i.e. the first expanded OFX plugin card in the + /// selected clip's chain. `None` when there is no candidate (no + /// project, no selected clip, no expanded plugin card). The program + /// viewer feeds this to `oakui::ofx::sync_active_interact`; the default + /// keeps engines without a plugin selection inert. + fn ofx_interact_target(&self, _cx: &App) -> Option { + None + } + /// Applies a node-editor edit request to the engine's model. fn apply_node_graph_event(&mut self, event: &NodeGraphEvent, cx: &mut Context); diff --git a/src/oakui/ofx.rs b/src/oakui/ofx.rs index 27e91be8f..dbecc016b 100644 --- a/src/oakui/ofx.rs +++ b/src/oakui/ofx.rs @@ -39,13 +39,21 @@ //! Preview/export rendering runs through the process-isolated oak-worker //! pool (M15 S2), so plugin rendering happens in the worker process where //! this main-process reporter factory is not in effect. The wiring still -//! serves the in-process render paths (e.g. the test-only inline backend) -//! and future work; worker-side progress forwarding over IPC is a TODO. +//! serves the in-process render paths (e.g. the test-only inline backend). +//! Worker-side progress is forwarded over the control plane: the worker +//! installs its own reporter factory whose reporters stream +//! `plugin_progress` NDJSON events to the dispatcher ([`init`] registers +//! the dispatcher callback, which feeds the same channel as the inline +//! reporter); the dialog's Cancel button broadcasts `plugin_cancel` to the +//! workers ([`cancel_plugin_render`]). use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; +use gpui::{Keystroke, Point, RenderImage, Size}; use oakplugin::progress::{ReporterFactory, UiProgressReporter}; +use oakplugin::suites::interact::Interact; +use oakplugin::suites::status; use oakplugin::suites::timeline::{ActiveViewerProvider, ViewerTimeInfo}; /// One progress event a plugin reporter pushed to the app channel (drained @@ -132,9 +140,15 @@ pub fn update_project_extent(width: f64, height: f64) { } /// Requests cancellation of the running plugin render (the progress -/// dialog's Cancel button). The next progressStart resets the flag. +/// dialog's Cancel button). The next progressStart resets the flag. Also +/// forwards the cancel to the render workers (their progress reporters +/// answer false from the next batch boundary on). pub fn cancel_plugin_render() { CANCEL.store(true, Ordering::Relaxed); + // The worker-process plugin renders are cancelled through the control + // plane (`plugin_cancel`); the in-flight frame completes (batch + // granularity — the worker processes messages between batches). + oakrender::procpool::request_plugin_cancel_all(); } // --------------------------------------------------------------------------- @@ -206,13 +220,410 @@ pub fn init() -> usize { oakplugin::progress::set_reporter_factory(Some(reporter_factory())); // 4. Active-viewer time provider (timeline suite fallback). oakplugin::suites::timeline::set_active_viewer_provider(Some(viewer_provider())); - // 5. Project extent (the engine refreshes it whenever the sequence + // 5. Worker-forwarded plugin progress (the render workers run plugin + // renders in their own process and stream `plugin_progress` NDJSON + // events over the control plane; the dispatcher hands them to this + // callback, which feeds the same channel the inline reporter uses). + oakrender::procpool::set_plugin_progress_cb(Some(Arc::new(|label, message, fraction| { + if let Some(tx) = progress_tx() { + let _ = tx.send(PluginProgressEvent { + label, + message, + fraction, + }); + } + }))); + // 6. Project extent (the engine refreshes it whenever the sequence // changes; keep the oakplugin side in sync with the default). let (w, h) = *extent_slot().lock().unwrap_or_else(|e| e.into_inner()); oakplugin::node_factory::set_project_extent(w, h); registered.len() } +// --------------------------------------------------------------------------- +// Main-process Interact instance management (the program viewer's overlay + +// event target) +// --------------------------------------------------------------------------- + +/// The viewport parameters of an interact: the plugin's draw/pen coordinate +/// space. `width`/`height` are the viewport size in pixels (the displayed +/// frame's pixel grid), `pixel_scale` is the canonical→pixel ratio (1.0 at +/// 1:1, the offscreen buffer's scale). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct InteractViewport { + /// Viewport width in pixels. + pub width: f64, + /// Viewport height in pixels. + pub height: f64, + /// Canonical→pixel scale (`(1.0, 1.0)` at 1:1). + pub pixel_scale: (f64, f64), +} + +impl InteractViewport { + /// The 1:1 viewport matching a frame of `width`×`height` pixels. + pub fn at_frame_size(width: u32, height: u32) -> Self { + Self { + width: width as f64, + height: height as f64, + pixel_scale: (1.0, 1.0), + } + } +} + +/// The app-side holder of the selected effect's live interact. +/// +/// The interact is created on the *main process* `oakplugin::Instance` of +/// the selected plugin effect node (the same registry the inspector's +/// push-button path uses), distinct from the worker-process render +/// instances — OFX allows a plugin to have several instances, and the +/// interact is a UI-event host, not a renderer. +struct ActiveInteract { + /// The oakplugin instance registry key (the plugin node's + /// `plugin_instance_handle`). + instance: u64, + /// The live interact (created via `Instance::new_interact`). + interact: Arc, + /// The viewport of the last overlay draw (updated on every successful + /// composite; the source of [`active_interact`]'s viewport). + viewport: InteractViewport, +} + +/// The single active interact: at most one plugin's custom UI is on the +/// program viewer at a time (the inspector's current selection target). +static ACTIVE_INTERACT: OnceLock>> = OnceLock::new(); + +fn active_interact_slot() -> &'static Mutex> { + ACTIVE_INTERACT.get_or_init(|| Mutex::new(None)) +} + +/// Recomputes the active interact for the current selection target. The +/// program viewer calls this from its frame sync with the selected OFX +/// effect's plugin instance handle (or `None` when the selection has no +/// plugin candidate — no selected clip, the chain has no OFX plugin card, +/// or the project closed). +/// +/// Creates the interact on the target instance (`new_interact` → +/// describe → create_instance; a plugin without an interact yields `None` +/// and stays inert — a normal no-op), and destroys the previous one when +/// the target changed. No-op while the target is unchanged. +pub fn sync_active_interact(instance: Option) { + // Unchanged target: nothing to do. + { + let mut slot = active_interact_slot().lock().unwrap_or_else(|e| e.into_inner()); + if slot + .as_ref() + .is_some_and(|active| Some(active.instance) == instance) + { + return; + } + // Target changed or cleared: destroy the old interact first (the + // kOfxActionDestroyInstanceInteract notification; idempotent). The + // app lock is released before the plugin call — a plugin callback + // must never re-enter this slot. + if let Some(active) = slot.take() { + drop(slot); + active.interact.destroy(); + } + } + let Some(id) = instance else { + return; + }; + let Some(inst) = oakplugin::node_factory::instance_from_id(id) else { + // The node/instance is gone (effect deleted): nothing to attach to. + return; + }; + let Some(interact) = inst.value.new_interact() else { + // The plugin has no interact: normal no-op. + return; + }; + // Complete the instance sequence (ofxInteract.h: NewInteract → + // Describe → CreateInstance before any draw/pen/key action). + interact.describe(); + interact.create_instance(); + let mut slot = active_interact_slot().lock().unwrap_or_else(|e| e.into_inner()); + *slot = Some(ActiveInteract { + instance: id, + interact, + viewport: InteractViewport { + width: 0.0, + height: 0.0, + pixel_scale: (1.0, 1.0), + }, + }); +} + +/// The active interact: `(instance, interact, viewport)`, or `None` when +/// no interact is live (no selection target, or the plugin has no +/// interact). The viewport reports the last drawn size and updates as the +/// viewer's frame size changes. +pub fn active_interact() -> Option<(u64, Arc, InteractViewport)> { + let slot = active_interact_slot().lock().unwrap_or_else(|e| e.into_inner()); + slot.as_ref().map(|a| (a.instance, a.interact.clone(), a.viewport)) +} + +/// Records the viewport the interact was last drawn at (keeps +/// [`active_interact`]'s viewport current). +fn note_interact_viewport(instance: u64, viewport: InteractViewport) { + let mut slot = active_interact_slot().lock().unwrap_or_else(|e| e.into_inner()); + if let Some(active) = slot.as_mut() { + if active.instance == instance { + active.viewport = viewport; + } + } +} + +// --------------------------------------------------------------------------- +// Overlay rendering + compositing +// --------------------------------------------------------------------------- + +/// Converts the viewer's BGRA8 [`RenderImage`] (the engine's display +/// format) into tightly packed F32 RGBA `(width, height, samples)` for the +/// compositing path. Returns `None` when the image has no CPU bytes. +fn bgra_image_to_f32_rgba(img: &RenderImage) -> Option<(u32, u32, Vec)> { + let bytes = img.as_bytes(0)?; + let size = img.size(0); + let w = size.width.0 as usize; + let h = size.height.0 as usize; + let expected = w * h * 4; + if bytes.len() < expected { + return None; + } + let mut out = Vec::with_capacity(expected); + for px in bytes[..expected].chunks_exact(4) { + out.push(px[2] as f32 / 255.0); // R + out.push(px[1] as f32 / 255.0); // G + out.push(px[0] as f32 / 255.0); // B + out.push(px[3] as f32 / 255.0); // A + } + Some((w as u32, h as u32, out)) +} + +/// Reads an `oakplugin` F32 RGBA image into a tightly packed `Vec`. +fn read_image_f32(img: &oakplugin::image::Image) -> Vec { + img.pixels() + .chunks_exact(4) + .map(|c| f32::from_ne_bytes(c[0..4].try_into().unwrap())) + .collect() +} + +/// Composites the straight-alpha overlay over `base` (both tightly packed +/// F32 RGBA, same length) with non-premultiplied "source-over": +/// +/// ```text +/// out.rgb = src.rgb * src.a + dst.rgb * (1 - src.a) +/// out.a = src.a + dst.a * (1 - src.a) +/// ``` +/// +/// # Why straight alpha +/// +/// The plugin draws into a fresh RGBA32F FBO with GL blend factors +/// `SRC_ALPHA` / `ONE_MINUS_SRC_ALPHA` (ofxDrawSuite.h's "over" +/// compositing) after a transparent-black clear, so the readback holds +/// *straight* (non-premultiplied) alpha. The formula above is the matching +/// straight-alpha "over". `base` is the displayed frame (alpha 1.0), so +/// the result alpha stays 1.0. Returns `None` on a length mismatch. +pub fn composite_overlay(overlay: &[f32], base: &[f32]) -> Option> { + if overlay.len() != base.len() || overlay.len() % 4 != 0 { + return None; + } + let mut out = vec![0.0f32; overlay.len()]; + for i in (0..overlay.len()).step_by(4) { + let a = overlay[i + 3].clamp(0.0, 1.0); + let dst_a = base[i + 3].clamp(0.0, 1.0); + out[i] = overlay[i] * a + base[i] * (1.0 - a); + out[i + 1] = overlay[i + 1] * a + base[i + 1] * (1.0 - a); + out[i + 2] = overlay[i + 2] * a + base[i + 2] * (1.0 - a); + out[i + 3] = a + dst_a * (1.0 - a); + } + Some(out) +} + +/// Renders the interact's overlay into a fresh offscreen FBO (RGBA32F) and +/// returns the frame composited over `base` as a BGRA8 [`RenderImage`] for +/// the viewer. +/// +/// # The GL sequence +/// +/// ```text +/// app acquire → CGL current (whole sequence) +/// create_output_texture → RGBA32F output texture (real GL name) +/// create_fbo / bind → the plugin draws into this FBO +/// clear to transparent 0 → straight-alpha "over" of the strokes +/// Interact::draw → the plugin's draw action (re-acquires GL +/// re-entrantly on the same thread, draws via +/// native GL + the Draw suite) +/// read_pixels_to_image → straight-alpha F32 RGBA overlay +/// composite_overlay → alpha "over" the current frame +/// ``` +/// +/// The app holds one guard across the whole sequence so the readback stays +/// on the plugin's context; `Interact::draw` re-enters the guard per its +/// own contract (gl_bridge nesting avoids the deadlock). Returns `None` +/// when the interact is inert — no GL, a viewport/frame mismatch, the +/// plugin failed, or draw returned a non-OK status — in which case the +/// caller shows the base frame unchanged. +pub fn draw_interact_composite( + instance: u64, + interact: &Interact, + viewport: &InteractViewport, + time: f64, + base: &RenderImage, +) -> Option> { + let (w, h) = (viewport.width as i32, viewport.height as i32); + if w <= 0 || h <= 0 { + return None; + } + let (bw, bh, base_f32) = bgra_image_to_f32_rgba(base)?; + if bw != w as u32 || bh != h as u32 { + return None; + } + let mut params = oakplugin::render::VideoParams::default(); + params.width = w; + params.height = h; + params.format = oakplugin::render::PIXEL_FORMAT_F32; + + let _guard = oakplugin::gl_bridge::acquire().ok()?; + let tex = oakplugin::gl_bridge::create_output_texture(w, h, ¶ms).ok()?; + let fbo = match oakplugin::gl_bridge::create_fbo(tex, w, h) { + Ok(fbo) => fbo, + Err(_) => { + oakplugin::gl_bridge::delete_gl_texture(tex); + return None; + } + }; + oakplugin::gl_bridge::bind_fbo(fbo); + oakplugin::gl_bridge::set_viewport(w, h); + // Clear to transparent black: the plugin's strokes composite "over" + // nothing, so the readback holds straight alpha (see + // [`composite_overlay`]). + oakplugin::gl_bridge::gl_clear_color(0.0, 0.0, 0.0, 0.0); + oakplugin::gl_bridge::gl_clear(); + + let st = interact.draw( + (viewport.width, viewport.height), + viewport.pixel_scale, + time, + // Background image: Phase 1 passes no composited background (the + // interact draws over a flat transparent clear; the frame is + // composited on the app side afterwards). + None, + ); + let overlay = oakplugin::gl_bridge::read_pixels_to_image(w, h, ¶ms); + oakplugin::gl_bridge::delete_fbo(fbo); + oakplugin::gl_bridge::delete_gl_texture(tex); + let (Ok(overlay), st) = (overlay, st) else { + return None; + }; + if st != status::OK { + return None; + } + let merged = composite_overlay(&read_image_f32(&overlay), &base_f32)?; + note_interact_viewport(instance, *viewport); + Some(Arc::new(super::frames::f32_rgba_to_bgra_image(bw, bh, &merged))) +} + +// --------------------------------------------------------------------------- +// Event forwarding (pen + key) and the idle pump +// --------------------------------------------------------------------------- + +/// Maps a pointer position in the picture area (local pixels, top-left +/// origin) to the OFX pen coordinates — the plugin's viewport pixels (the +/// displayed frame's pixel grid). The picture shows the frame with a +/// "contain" fit (letterboxed), so the mapping is the inverse of the +/// object-fit scale plus centering. Returns `None` when the pointer is in +/// the letterbox (outside the frame's pixel rect). +pub fn viewport_pixel_to_pen( + local: Point, + area: Size, + frame: Size, +) -> Option<(f64, f64)> { + let (aw, ah) = (area.width as f64, area.height as f64); + let (fw, fh) = (frame.width as f64, frame.height as f64); + if aw <= 0.0 || ah <= 0.0 || fw <= 0.0 || fh <= 0.0 { + return None; + } + let scale = (aw / fw).min(ah / fh); + let offset_x = (aw - fw * scale) / 2.0; + let offset_y = (ah - fh * scale) / 2.0; + let fx = (local.x as f64 - offset_x) / scale; + let fy = (local.y as f64 - offset_y) / scale; + if fx < 0.0 || fy < 0.0 || fx >= fw || fy >= fh { + return None; + } + Some((fx, fy)) +} + +/// Maps a gpui keystroke to the OFX key symbol (`ofxKeySyms.h` values from +/// `oakplugin::host::KEY_*`) and the key-string character +/// (`kOfxPropKeyString`: the UTF-8 character, empty for keys without one). +/// +/// Covers the common keys — alphanumerics, the arrows, return, escape, +/// backspace/delete, tab, home/end, page up/down and the function keys; +/// anything else maps to [`oakplugin::host::KEY_UNKNOWN`] with an empty +/// string. +pub fn key_symbol(keystroke: &Keystroke) -> (i32, String) { + use oakplugin::host as ofx_key; + let key = keystroke.key.as_str(); + let named = match key { + "space" => Some(ofx_key::KEY_SPACE), + "enter" | "return" => Some(ofx_key::KEY_RETURN), + "escape" => Some(ofx_key::KEY_ESCAPE), + "backspace" => Some(ofx_key::KEY_BACKSPACE), + "delete" => Some(ofx_key::KEY_DELETE), + "tab" => Some(ofx_key::KEY_TAB), + "home" => Some(ofx_key::KEY_HOME), + "end" => Some(ofx_key::KEY_END), + "left" => Some(ofx_key::KEY_LEFT), + "right" => Some(ofx_key::KEY_RIGHT), + "up" => Some(ofx_key::KEY_UP), + "down" => Some(ofx_key::KEY_DOWN), + "pageup" => Some(ofx_key::KEY_PAGE_UP), + "pagedown" => Some(ofx_key::KEY_PAGE_DOWN), + _ => None, + }; + if let Some(sym) = named { + return (sym, String::new()); + } + // Function keys: f1..f35 → KEY_F1 + (n-1). + if let Some(rest) = key.strip_prefix('f') { + if let Ok(n) = rest.parse::() { + if (1..=35).contains(&n) { + return (ofx_key::KEY_F1 + (n as i32 - 1), String::new()); + } + } + } + // Printable single-character keys (letters/digits/punctuation): the + // symbol is the ASCII code (ofxKeySyms maps printable ASCII 1:1; + // KEY_A=0x61 … KEY_Z=0x7a, KEY_SPACE=0x20), the string is the char. + if key.len() == 1 && key.is_ascii() { + return (key.as_bytes()[0] as i32, key.to_string()); + } + (ofx_key::KEY_UNKNOWN, String::new()) +} + +/// The idle pump for the active interact. The program viewer calls this +/// from its throttled timer (~10 Hz, not every render): the app forwards +/// `kOfxInteractActionIdle` so the plugin can run lightweight UI work +/// (marquee feedback, cursor animation). No-op without an active interact. +pub fn pump_interact_idle() { + use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; + // Throttle to ~10 Hz: the panel timer may fire faster than the OFX + // idle cadence, and idle is meant to run when the app is otherwise + // quiet, not every animation frame. + static LAST_IDLE_MS: AtomicU64 = AtomicU64::new(0); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + if now.saturating_sub(LAST_IDLE_MS.load(AtomicOrdering::Relaxed)) < 100 { + return; + } + LAST_IDLE_MS.store(now, AtomicOrdering::Relaxed); + if let Some((_, interact, _)) = active_interact() { + let _ = interact.idle(); + } +} + #[cfg(test)] mod tests { use super::*; @@ -245,4 +656,360 @@ mod tests { let slot = extent_slot(); assert_eq!(*slot.lock().unwrap(), (1280.0, 720.0)); } + + // --------------------------------------------------------------------------- + // Interact viewport mapping / key symbols / compositing (pure) + // --------------------------------------------------------------------------- + + use gpui::{point, size}; + + /// 视口像素 → pen 坐标：the picture shows the frame with a contain fit, + /// so a 2:1 area/frame scale maps local pixels 1:2 to frame pixels, and + /// the letterbox returns None. + #[test] + fn viewport_pixel_to_pen_maps_contain_fit() { + let area = size(640.0, 360.0); + let frame = size(320.0, 180.0); + // Frame fills the area at scale 2 (no letterbox): local 40,40 → 20,20. + let p = viewport_pixel_to_pen(point(40.0, 40.0), area, frame); + assert_eq!(p, Some((20.0, 20.0))); + // Corner maps to the frame corner. + let p = viewport_pixel_to_pen(point(0.0, 0.0), area, frame); + assert_eq!(p, Some((0.0, 0.0))); + let p = viewport_pixel_to_pen(point(640.0, 360.0), area, frame); + // Exclusive upper bound: exactly the far edge is outside the frame. + assert_eq!(p, None); + } + + /// Letterboxing: a 4:3 area showing a 16:9 frame leaves vertical bars; + /// pointers in the bars map to None, the scaled rect maps 1:2. + #[test] + fn viewport_pixel_to_pen_respects_letterbox() { + let area = size(640.0, 480.0); + let frame = size(320.0, 180.0); + // Scale = min(640/320, 480/180) = 2; the frame occupies 640×360, + // leaving 60px top/bottom bars. + let p = viewport_pixel_to_pen(point(60.0, 240.0), area, frame); + assert_eq!(p, Some((30.0, 90.0))); + // Inside the top letterbox bar. + let p = viewport_pixel_to_pen(point(320.0, 10.0), area, frame); + assert_eq!(p, None); + // Zero-size inputs. + assert_eq!(viewport_pixel_to_pen(point(0.0, 0.0), size(0.0, 0.0), frame), None); + assert_eq!(viewport_pixel_to_pen(point(0.0, 0.0), area, size(0.0, 0.0)), None); + } + + /// Key mapping: the common keys land on the ofxKeySyms values; printable + /// single characters carry their ASCII symbol + the character string. + #[test] + fn key_symbol_maps_common_keys() { + use oakplugin::host as ofx_key; + let ks = |key: &str| gpui::Keystroke::parse(key).unwrap(); + // Alphanumerics: symbol = ASCII code, string = the char. + assert_eq!(key_symbol(&ks("a")), (ofx_key::KEY_A, "a".to_string())); + assert_eq!(key_symbol(&ks("z")), (ofx_key::KEY_Z, "z".to_string())); + assert_eq!(key_symbol(&ks("1")), (b'1' as i32, "1".to_string())); + // Named keys: symbol only, empty string. + assert_eq!(key_symbol(&ks("space")), (ofx_key::KEY_SPACE, String::new())); + assert_eq!(key_symbol(&ks("enter")), (ofx_key::KEY_RETURN, String::new())); + assert_eq!(key_symbol(&ks("escape")), (ofx_key::KEY_ESCAPE, String::new())); + assert_eq!(key_symbol(&ks("backspace")), (ofx_key::KEY_BACKSPACE, String::new())); + assert_eq!(key_symbol(&ks("delete")), (ofx_key::KEY_DELETE, String::new())); + assert_eq!(key_symbol(&ks("left")), (ofx_key::KEY_LEFT, String::new())); + assert_eq!(key_symbol(&ks("right")), (ofx_key::KEY_RIGHT, String::new())); + assert_eq!(key_symbol(&ks("up")), (ofx_key::KEY_UP, String::new())); + assert_eq!(key_symbol(&ks("down")), (ofx_key::KEY_DOWN, String::new())); + assert_eq!(key_symbol(&ks("home")), (ofx_key::KEY_HOME, String::new())); + assert_eq!(key_symbol(&ks("end")), (ofx_key::KEY_END, String::new())); + assert_eq!(key_symbol(&ks("pageup")), (ofx_key::KEY_PAGE_UP, String::new())); + assert_eq!(key_symbol(&ks("pagedown")), (ofx_key::KEY_PAGE_DOWN, String::new())); + assert_eq!(key_symbol(&ks("tab")), (ofx_key::KEY_TAB, String::new())); + assert_eq!(key_symbol(&ks("f1")), (ofx_key::KEY_F1, String::new())); + assert_eq!(key_symbol(&ks("f12")), (ofx_key::KEY_F1 + 11, String::new())); + // Unknown multi-character keys. + let (sym, s) = key_symbol(&ks("insert")); + assert_eq!(sym, ofx_key::KEY_UNKNOWN); + assert!(s.is_empty()); + } + + /// Straight-alpha "over" compositing: the overlay replaces the base where + /// it is opaque, blends where it is translucent, and stays 1.0 alpha. + #[test] + fn composite_overlay_blends_alpha() { + // Opaque red over blue → red. + let overlay = [1.0, 0.0, 0.0, 1.0]; + let base = [0.0, 0.0, 1.0, 1.0]; + let out = composite_overlay(&overlay, &base).unwrap(); + assert_eq!(out, [1.0, 0.0, 0.0, 1.0]); + // Half-alpha red over blue → 0.5 red + 0.5 blue. + let overlay = [1.0, 0.0, 0.0, 0.5]; + let out = composite_overlay(&overlay, &base).unwrap(); + for (i, expected) in [(0, 0.5), (1, 0.0), (2, 0.5), (3, 1.0)] { + assert!((out[i] - expected).abs() < 1e-6, "channel {i}: {} != {expected}", out[i]); + } + // Transparent overlay → base unchanged. + let overlay = [0.0, 0.0, 0.0, 0.0]; + assert_eq!(composite_overlay(&overlay, &base).unwrap(), base); + // Length mismatch → None. + assert!(composite_overlay(&[0.0; 4], &[0.0; 8]).is_none()); + } + + /// The engine frame's BGRA8 → F32 RGBA conversion round-trips through + /// the viewer format (the compositing path's input). + #[test] + fn bgra_frame_roundtrips_to_f32_rgba() { + let (w, h) = (2u32, 1u32); + let samples = [1.0, 0.0, 0.5, 1.0, 0.25, 0.5, 0.75, 1.0]; + let img = super::super::frames::f32_rgba_to_bgra_image(w, h, &samples); + let (got_w, got_h, rgba) = bgra_image_to_f32_rgba(&img).unwrap(); + assert_eq!((got_w, got_h), (w, h)); + for (i, expected) in [(0, 1.0), (1, 0.0), (2, 0.5), (3, 1.0), (4, 0.25), (5, 0.5), (6, 0.75), (7, 1.0)] { + assert!((rgba[i] - expected).abs() < 0.01, "channel {i}: {} != {expected}", rgba[i]); + } + } + + // --------------------------------------------------------------------------- + // End-to-end with the minimal test plugin (cbits/oak_test_plugin.c) + // --------------------------------------------------------------------------- + + const INTERACT_PLUGIN_ID: &str = "org.oak.test-plugin.interact"; + /// The plugin records every interact action into the file this env var + /// names. + const MARKER_ENV: &str = "OAK_TEST_PLUGIN_INTERACT_MARKER"; + /// Serializes the process-global Host singleton across host-touching + /// tests (the plugin cache is a process singleton with no lock). + static HOST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// The minimal test plugin's scan directory (the bundle is assembled + /// from the dylib the app build script compiles into OUT_DIR), or `None` + /// when the plugin is unavailable (the test skips). + fn test_plugin_scan_dir() -> Option { + let out = std::path::PathBuf::from(env!("OUT_DIR")); + let lib = if cfg!(target_os = "macos") { + out.join("oak_test_plugin.dylib") + } else { + out.join("oak_test_plugin.so") + }; + if !lib.is_file() { + return None; + } + let bundle = std::env::temp_dir() + .join(format!("oak-app-test-plugin-{}", std::process::id())) + .join("oak-test-plugin.ofx.bundle"); + let platform = if cfg!(target_os = "macos") { + "MacOS" + } else { + "Linux-x86-64" + }; + let bin_dir = bundle.join("Contents").join(platform); + std::fs::create_dir_all(&bin_dir).ok()?; + let target = bin_dir.join("plugin"); + if !target.exists() { + std::fs::copy(&lib, &target).ok()?; + } + Some(bundle.parent().unwrap().to_path_buf()) + } + + fn scan_interact_plugin() -> bool { + let Some(dir) = test_plugin_scan_dir() else { + println!("SKIP: minimal test plugin not built"); + return false; + }; + if oakplugin::host::Host::global().cache.scan_path(&dir).is_err() { + println!("SKIP: test plugin scan failed"); + return false; + } + oakplugin::node_factory::register_plugin_nodes(); + true + } + + fn marker_path(tag: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!( + "oak-app-interact-{}-{}.log", + std::process::id(), + tag + )) + } + + fn read_marker(path: &std::path::Path) -> Vec { + std::fs::read_to_string(path) + .unwrap_or_default() + .lines() + .map(|l| l.to_string()) + .collect() + } + + /// App-layer interact creation + synthetic event forwarding, asserting + /// the plugin really received each action and its arguments (the marker + /// file the plugin appends to). + #[test] + fn interact_e2e_lifecycle_and_event_forwarding() { + let _lock = HOST_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + if !scan_interact_plugin() { + return; + } + let marker = marker_path("e2e"); + let _ = std::fs::remove_file(&marker); + unsafe { std::env::set_var(MARKER_ENV, &marker) }; + + // (1) app-layer creation: `sync_active_interact` on the plugin + // instance handle creates the interact (new_interact → describe → + // create_instance). + let inst = oakplugin::host::Host::global() + .create_instance(INTERACT_PLUGIN_ID, None) + .expect("interact variant instance"); + let handle = oakplugin::node_factory::register_instance(inst); + sync_active_interact(Some(handle)); + + let (active_handle, interact, _viewport) = + active_interact().expect("active interact created"); + assert_eq!(active_handle, handle); + + // (2) synthetic mouse forwarding: the picture-local → pen mapping + // (the panel's forward path) then the pen actions, with the pen-down + // state carried by the move's button state. + let area = size(640.0, 360.0); + let frame = size(320.0, 180.0); + let (px, py) = + viewport_pixel_to_pen(point(40.0, 40.0), area, frame).expect("contain fit maps 2:1"); + assert_eq!((px, py), (20.0, 20.0)); + assert_eq!(interact.pen_motion((px, py), true, 5.0), status::OK); + assert_eq!(interact.pen_down((px, py), 5.0), status::OK); + assert_eq!(interact.pen_up((px, py), 5.0), status::OK); + + // (3) keys and idle. + assert_eq!( + interact.key_down(oakplugin::host::KEY_A, "a", 5.0), + status::OK + ); + assert_eq!( + interact.key_up(oakplugin::host::KEY_A, "a", 5.0), + status::OK + ); + assert_eq!(interact.idle(), status::OK); + + // (4) clearing the selection destroys the interact. + sync_active_interact(None); + assert!( + active_interact().is_none(), + "clearing the selection should destroy the active interact" + ); + + unsafe { std::env::remove_var(MARKER_ENV) }; + let lines = read_marker(&marker); + let _ = std::fs::remove_file(&marker); + + // Lifecycle reached the plugin, in order. + let seq = ["new_interact", "describe", "create", "destroy"]; + let pos: Vec = seq + .iter() + .map(|s| lines.iter().position(|l| l == s)) + .collect::>>() + .unwrap_or_else(|| panic!("lifecycle actions missing: {lines:?}")); + assert!( + pos.windows(2).all(|w| w[0] < w[1]), + "lifecycle order should be new_interact→describe→create→destroy" + ); + + // The forwarded events with real arguments (C %g drops trailing + // zeros). + assert!( + lines.iter().any(|l| l == "pen_motion vp=20,20 canon=20,20 pressure=1"), + "pen_motion not recorded with pen-down state: {lines:?}" + ); + assert!( + lines.iter().any(|l| l == "pen_down vp=20,20 canon=20,20 pressure=1"), + "pen_down not recorded: {lines:?}" + ); + assert!( + lines.iter().any(|l| l == "pen_up vp=20,20 canon=20,20 pressure=0"), + "pen_up not recorded: {lines:?}" + ); + assert!( + lines.iter().any(|l| l == "key_down sym=97 str=a"), + "key_down not recorded: {lines:?}" + ); + assert!( + lines.iter().any(|l| l == "idle"), + "idle not recorded: {lines:?}" + ); + + oakplugin::host::Host::global().shutdown(); + } + + /// End-to-end overlay: the plugin's draw action really renders into the + /// offscreen FBO, and the composite over a synthetic base frame carries + /// the plugin-drawn colours (macOS real GL; gated by `OAK_GPU_TESTS`). + #[test] + fn interact_e2e_draw_overlay_composites_plugin_colours() { + let _lock = HOST_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + if !scan_interact_plugin() { + return; + } + if !cfg!(target_os = "macos") || std::env::var_os("OAK_GPU_TESTS").is_none() { + println!("SKIP: draw overlay needs macOS GL (set OAK_GPU_TESTS)"); + return; + } + let marker = marker_path("draw"); + let _ = std::fs::remove_file(&marker); + unsafe { std::env::set_var(MARKER_ENV, &marker) }; + + let inst = oakplugin::host::Host::global() + .create_instance(INTERACT_PLUGIN_ID, None) + .expect("interact variant instance"); + let handle = oakplugin::node_factory::register_instance(inst); + sync_active_interact(Some(handle)); + let (_, interact, _) = active_interact().expect("active interact created"); + + // A 64×64 opaque blue base frame. + let (w, h) = (64u32, 64u32); + let mut base_samples = vec![0.0f32; (w * h * 4) as usize]; + for px in base_samples.chunks_exact_mut(4) { + px.copy_from_slice(&[0.0, 0.0, 1.0, 1.0]); + } + let base = Arc::new(super::super::frames::f32_rgba_to_bgra_image(w, h, &base_samples)); + let viewport = InteractViewport::at_frame_size(w, h); + let composite = draw_interact_composite(handle, &interact, &viewport, 0.0, &base) + .expect("GL overlay composite"); + + // The test plugin clears to opaque dark grey (0.05) and draws a + // solid rectangle (0.9,0.1,0.2) at canonical 10..30 — pixel scale 1 + // maps it to pixels 10..30. The opaque overlay fully replaces the + // base frame. + let bytes = composite.as_bytes(0).expect("composite bytes"); + let px_at = |x: u32, y: u32| { + let i = ((y * w + x) * 4) as usize; + ( + bytes[i + 2] as f32 / 255.0, + bytes[i + 1] as f32 / 255.0, + bytes[i] as f32 / 255.0, + ) + }; + let inside = px_at(20, 20); + assert!( + (inside.0 - 0.9).abs() < 0.03 + && (inside.1 - 0.1).abs() < 0.03 + && (inside.2 - 0.2).abs() < 0.03, + "plugin-drawn rectangle should composite at (20,20): {inside:?}" + ); + let outside = px_at(5, 5); + assert!( + (outside.0 - 0.05).abs() < 0.03 + && (outside.1 - 0.05).abs() < 0.03 + && (outside.2 - 0.05).abs() < 0.03, + "plugin clear colour should fill the rest: {outside:?}" + ); + + // The draw action reached the plugin with the viewport args. + unsafe { std::env::remove_var(MARKER_ENV) }; + let lines = read_marker(&marker); + let _ = std::fs::remove_file(&marker); + assert!( + lines.iter().any(|l| l.starts_with("draw vp=64x64")), + "draw not recorded with viewport: {lines:?}" + ); + + sync_active_interact(None); + oakplugin::host::Host::global().shutdown(); + } } diff --git a/src/oakui/real.rs b/src/oakui/real.rs index 7f0418f60..be9378b8d 100644 --- a/src/oakui/real.rs +++ b/src/oakui/real.rs @@ -3295,6 +3295,26 @@ impl AppEngine for RealEngine { Ok(()) } + fn ofx_interact_target(&self, _cx: &App) -> Option { + let Some(project) = self.project_ref() else { + return None; + }; + let Some(host) = self.selected_clip_node() else { + return None; + }; + let guard = graphops::lock(project); + for node in super::effectchain::chain(&guard.graph, host) { + // The inspector's current selection: the first *expanded* + // OFX plugin card (its parameter UI is on screen, so its + // custom interact drives the viewer overlay). + if !self.expanded_effects.contains(&node.identity()) { + continue; + } + return super::effectchain::plugin_instance_handle(&guard.graph, node); + } + None + } + fn apply_effect_event(&mut self, event: &EffectStackEvent, cx: &mut Context) { match event { EffectStackEvent::EnableToggled { effect, enabled } => { diff --git a/src/panels/ofx_params.rs b/src/panels/ofx_params.rs index 8a8c6ea5a..6ffc1433a 100644 --- a/src/panels/ofx_params.rs +++ b/src/panels/ofx_params.rs @@ -27,7 +27,9 @@ //! - combo → [`ComboBox`] fed from the repeated `("combo_option", _)` //! properties; string-combo values come from `("combo_value", _)` //! - text → [`EditableTextState`] -//! - vec2 / vec3 / color → one [`SpinBox`] per component +//! - vec2 / vec3 → one [`SpinBox`] per component +//! - color → a swatch + deferred popup picker ([`OfxColorPicker`]: +//! R/G/B/A sliders, live preview, hex input, Cancel/OK) //! - push button → a clickable button (`AppEngine::effect_push_button`) //! //! Secret (HIDDEN) inputs never reach the snapshot, so they render @@ -38,15 +40,22 @@ //! created fresh per expanded-card render), so it carries no state of its //! own; values are re-synced from the engine each frame. +use std::sync::Arc; + use gpui::effect_stack::EffectId; use gpui::colors::DefaultColors; use gpui::{ - div, prelude::*, px, ClickEvent, Context, Entity, Render, SharedString, Window, + div, prelude::*, px, rgb, size, point, ClickEvent, Context, Entity, EventEmitter, Render, + SharedString, Window, +}; +use gpui::{ + Anchor, App, Bounds, ElementId, Hsla, KeyDownEvent, MouseButton, MouseDownEvent, MouseUpEvent, + Point, Pixels, Rgba, anchored, canvas, deferred, fill, }; use gpui_elements::editable_text::{text_input, EditableTextState, StringStorage}; use gpui_widgets::checkbox::{CheckBox, CheckBoxEvent, CheckState}; use gpui_widgets::combo_box::{ComboBox, ComboBoxEvent, ComboBoxOption}; -use gpui_widgets::slider::{Slider, SliderModel}; +use gpui_widgets::slider::{Slider, SliderEvent, SliderModel}; use gpui_widgets::spinbox::{SpinBox, SpinBoxEvent}; use gpui_widgets::value::{SliderValue, ValueKind}; @@ -74,9 +83,11 @@ enum ControlKind { CheckBox(Entity), /// A combo box (combo / string combo). Combo(Entity), - /// One spinbox per component (vec2 / vec3 / color); the usize is the + /// One spinbox per component (vec2 / vec3); the usize is the /// component index within the value. Spin(Vec<(Entity, usize)>), + /// A colour swatch + popup picker (color). + Color(Entity), /// A text field (string). Text(Entity), /// A push button (rendered inline, no entity). @@ -155,6 +166,18 @@ impl OfxParamsView { } } } + ControlKind::Color(picker) => { + if let NodeValue::Color(v) = param.value { + let color = Rgba { + r: v[0] as f32, + g: v[1] as f32, + b: v[2] as f32, + a: v[3] as f32, + }; + let picker = picker.clone(); + picker.update(cx, |picker, cx| picker.set_committed(color, cx)); + } + } ControlKind::Text(editor) => { let text = match ¶m.value { NodeValue::Text(s) => s.clone(), @@ -330,21 +353,14 @@ fn build_control( *next_id += 1; ControlKind::Text(editor) } - ValueType::Vec2 | ValueType::Vec3 | ValueType::Color => { + ValueType::Vec2 | ValueType::Vec3 => { let components = value_components(¶m.value); let count = if param.value_type == ValueType::Vec2 { 2 - } else if param.value_type == ValueType::Vec3 { + } else { 3 - } else { - 4 - }; - let (min, max) = if param.value_type == ValueType::Color { - // Colour channels live in 0..1 (or the attached min/max). - (0.0f64, 1.0f64) - } else { - default_range(param.value_type) }; + let (min, max) = default_range(param.value_type); let mut spins = Vec::new(); for channel in 0..count { let value = components.get(channel).copied().unwrap_or(0.0).clamp(min, max); @@ -355,6 +371,20 @@ fn build_control( } ControlKind::Spin(spins) } + ValueType::Color => { + // Colour params get the swatch + popup picker (channels are + // always 0..1 regardless of any attached min/max). + let components = value_components(¶m.value); + let color = Rgba { + r: components.get(0).copied().unwrap_or(0.0) as f32, + g: components.get(1).copied().unwrap_or(0.0) as f32, + b: components.get(2).copied().unwrap_or(0.0) as f32, + a: components.get(3).copied().unwrap_or(1.0) as f32, + }; + let picker = cx.new(|cx| OfxColorPicker::new(*next_id, color, window, cx)); + *next_id += 1; + ControlKind::Color(picker) + } ValueType::PushButton => ControlKind::PushButton, // Custom / binary and anything without an editable control: a // read-only line (or nothing). @@ -466,6 +496,25 @@ fn wire_controls(view: &OfxParamsView, cx: &mut Context { + let picker = picker.clone(); + cx.subscribe(&picker, move |_, _, event: &OfxColorEvent, cx| { + // Only OK commits; slider drags update the draft inside + // the picker, so a drag session is one undo row. + if let OfxColorEvent::Committed(color) = event { + let nv = NodeValue::Color([ + color.r as f64, + color.g as f64, + color.b as f64, + color.a as f64, + ]); + engine.update(cx, |engine, cx| { + let _ = engine.set_effect_param(effect, &input_id, nv, cx); + }); + } + }) + .detach(); + } ControlKind::Text(_editor) => { // The text field commits explicitly (the commit button in the // row). No event subscription here: the params view is rebuilt @@ -565,6 +614,9 @@ impl Render for OfxParamsView { } row.into_any_element() } + ControlKind::Color(picker) => { + div().flex_1().child(picker.clone()).into_any_element() + } ControlKind::Text(editor) => { let weak = editor.downgrade(); let engine = self.engine.clone(); @@ -642,3 +694,595 @@ impl Render for OfxParamsView { body } } + +// --------------------------------------------------------------------------- +// OfxColorPicker — colour swatch + deferred popup picker +// --------------------------------------------------------------------------- + +/// A request emitted by an [`OfxColorPicker`]. +#[derive(Debug, Clone, Copy, PartialEq)] +enum OfxColorEvent { + /// The popup was opened (draft reset to the committed colour). + Opened, + /// The popup was dismissed without committing. + Cancelled, + /// OK pressed: the caller should commit this colour (undoable). + Committed(Rgba), +} + +/// A colour swatch with a deferred popup picker, used for OFX colour +/// parameters (the replacement for the old per-channel spinboxes). +/// +/// - The swatch shows the **committed** colour (the engine value) over a +/// two-tone checkerboard so alpha is visible. +/// - Clicking opens a deferred popup with four 0..1 channel sliders, a +/// live preview swatch, a hex field (`#RRGGBB` / `#RRGGBBAA`, validated) +/// and Cancel / OK buttons. +/// - Slider drags only mutate the **draft**; only OK emits +/// [`OfxColorEvent::Committed`], which the params view routes through +/// [`AppEngine::set_effect_param`] (undoable). A drag session is +/// therefore a single undo row. Cancel / Escape / outside click discard +/// the draft. +/// +/// The picker is a child entity of the params view and carries its own +/// state across frames; [`OfxColorPicker::set_committed`] re-syncs it from +/// the engine each frame (undo / redo / external edits land on the swatch). +struct OfxColorPicker { + /// Stable control id (element ids / slider ids). + control: usize, + /// Whether the popup is open. + open: bool, + /// Popup anchor position (window space, set when opening). + position: Point, + /// The committed colour (what the swatch shows). + committed: Rgba, + /// The draft colour while the popup is open (what OK commits). + draft: Rgba, + /// Whether the last hex parse failed (error hint in the popup). + hex_error: bool, + /// Whether the popup was open when the swatch was pressed (a second + /// click closes it). + was_open_at_down: bool, + /// Channel sliders, R/G/B/A in 0..1. + r: Entity, + g: Entity, + b: Entity, + a: Entity, + /// The hex editor (`#RRGGBB` / `#RRGGBBAA`). + hex: Entity, +} + +impl OfxColorPicker { + /// Create a picker for `control` showing `color`. + pub(crate) fn new( + control: usize, + color: Rgba, + window: &mut Window, + cx: &mut Context, + ) -> Self { + let mut picker = Self { + control, + open: false, + position: Point::default(), + committed: color, + draft: color, + hex_error: false, + was_open_at_down: false, + r: Self::channel_slider(cx, window, 0, color.r as f64), + g: Self::channel_slider(cx, window, 1, color.g as f64), + b: Self::channel_slider(cx, window, 2, color.b as f64), + a: Self::channel_slider(cx, window, 3, color.a as f64), + hex: cx.new(|cx| EditableTextState::new(StringStorage::default(), cx)), + }; + let hex_text = format_hex(color); + picker.hex.update(cx, |hex, cx| hex.emplace(&hex_text, cx)); + picker + } + + /// Build a 0..1 float slider for one channel and subscribe its edits to + /// [`Self::on_slider`]. + fn channel_slider( + cx: &mut Context, + window: &mut Window, + channel: usize, + value: f64, + ) -> Entity { + let model = SliderModel::new(ValueKind::Float, 0.0, 1.0, 0.01, value.clamp(0.0, 1.0)); + let slider = cx.new(|cx| Slider::new(channel * 1000 + 100, model, window, cx)); + cx.subscribe( + &slider, + move |this: &mut Self, _: Entity, event: &SliderEvent, cx| { + this.on_slider(channel, event, cx); + }, + ) + .detach(); + slider + } + + /// A slider changed: update the draft channel and re-format the hex + /// field (no commit — the value only lands on OK). + fn on_slider(&mut self, channel: usize, event: &SliderEvent, cx: &mut Context) { + if let SliderEvent::ValueChanged { value, .. } = event { + let v = value.to_f64().clamp(0.0, 1.0) as f32; + match channel { + 0 => self.draft.r = v, + 1 => self.draft.g = v, + 2 => self.draft.b = v, + 3 => self.draft.a = v, + _ => return, + } + self.hex_error = false; + let hex_entity = self.hex.clone(); + let hex_text = format_hex(self.draft); + hex_entity.update(cx, |hex, cx| { + if hex.as_str() != hex_text { + hex.emplace(&hex_text, cx); + } + }); + cx.notify(); + } + } + + /// Re-sync the sliders and the hex field from the current draft (no + /// events emitted; used when the draft is reset or committed). + fn sync_from_draft(&self, cx: &mut Context) { + let values = [ + self.draft.r as f64, + self.draft.g as f64, + self.draft.b as f64, + self.draft.a as f64, + ]; + let sliders = [&self.r, &self.g, &self.b, &self.a]; + for (slider, value) in sliders.iter().zip(values.iter()) { + let slider = slider.clone(); + slider.update(cx, |slider, _| { + slider.set_value(SliderValue::Float(*value)); + }); + } + let hex_entity = self.hex.clone(); + let hex_text = format_hex(self.draft); + hex_entity.update(cx, |hex, cx| { + if hex.as_str() != hex_text { + hex.emplace(&hex_text, cx); + } + }); + } + + /// Apply a committed colour from the engine (called every frame from + /// the params view's value sync). While the popup is open the draft is + /// left alone so an in-progress edit is not clobbered by re-syncs. + pub(crate) fn set_committed(&mut self, color: Rgba, cx: &mut Context) { + if self.committed == color { + return; + } + self.committed = color; + if !self.open { + self.draft = color; + self.sync_from_draft(cx); + } + cx.notify(); + } + + fn open_menu(&mut self, position: Point, cx: &mut Context) { + if !self.open { + self.open = true; + self.position = position; + // Start from the committed colour. + self.draft = self.committed; + self.hex_error = false; + self.sync_from_draft(cx); + cx.emit(OfxColorEvent::Opened); + cx.notify(); + } + } + + /// Cancel: discard the draft, keep the committed colour. + fn close_menu(&mut self, cx: &mut Context) { + if self.open { + self.open = false; + self.hex_error = false; + self.sync_from_draft(cx); + cx.emit(OfxColorEvent::Cancelled); + cx.notify(); + } + } + + /// OK: validate a hand-typed hex edit, then commit the draft. + fn commit(&mut self, cx: &mut Context) { + let text = self.hex.read(cx).as_str().trim().to_string(); + if !text.is_empty() { + match parse_hex(&text) { + Some(color) => self.draft = color, + None => { + self.hex_error = true; + cx.notify(); + return; + } + } + } + self.hex_error = false; + let color = self.draft; + self.open = false; + self.committed = color; + self.sync_from_draft(cx); + cx.emit(OfxColorEvent::Committed(color)); + cx.notify(); + } + + /// The popup's anchored subtree (deferred, above the card). + fn popup_anchored( + &self, + cx: &mut Context, + colors: &Arc, + ) -> gpui::Deferred { + let control = self.control; + let draft = self.draft; + let hex_weak = self.hex.downgrade(); + let hex_error = self.hex_error; + + // Four labelled 0..1 channel sliders. + let mut slider_rows = div().flex().flex_col().gap_1(); + for (label, slider) in [ + ("R", &self.r), + ("G", &self.g), + ("B", &self.b), + ("A", &self.a), + ] { + slider_rows = slider_rows.child( + div() + .flex() + .items_center() + .gap_1() + .child( + div() + .w(px(12.0)) + .text_sm() + .text_color(colors.text) + .child(label), + ) + .child(div().flex_1().child(slider.clone())), + ); + } + + // Live preview swatch (checkerboard + draft) next to the hex field. + let preview_canvas = canvas( + |bounds, _window, _cx| bounds, + move |bounds, _content, window, cx| { + paint_checker_swatch(bounds, draft, window, cx); + }, + ); + let preview = div() + .w(px(36.0)) + .h(px(24.0)) + .rounded_sm() + .border_1() + .border_color(colors.border) + .overflow_hidden() + .child(preview_canvas); + let hex_row = div() + .flex() + .items_center() + .gap_1() + .child(preview) + .child( + div() + .flex_1() + .rounded_md() + .border_1() + .border_color(colors.border) + .bg(colors.background) + .px_2() + .py_1() + .child( + text_input(format!("ofx-color-hex-{control}")) + .state(hex_weak) + .accepts_input(true), + ), + ); + + // Hex parse error hint. + let error_hint = if hex_error { + div() + .text_xs() + .text_color(gpui::rgba(0xff5555)) + .child(crate::i18n::tr("ofx.color.invalid")) + .into_any_element() + } else { + div().into_any_element() + }; + + // Cancel / OK. + let cancel = div() + .id(SharedString::from(format!("ofx-color-cancel-{control}"))) + .cursor_pointer() + .rounded_sm() + .border_1() + .border_color(colors.border) + .bg(colors.background) + .text_sm() + .text_color(colors.text) + .px_2() + .py_1() + .child(crate::i18n::tr("ofx.color.cancel")) + .on_click(cx.listener(|this, _event: &ClickEvent, _window, cx| { + this.close_menu(cx); + })); + let ok = div() + .id(SharedString::from(format!("ofx-color-ok-{control}"))) + .cursor_pointer() + .rounded_sm() + .border_1() + .border_color(colors.border) + .bg(colors.selected) + .text_sm() + .text_color(colors.text) + .px_2() + .py_1() + .child(crate::i18n::tr("ofx.color.ok")) + .on_click(cx.listener(|this, _event: &ClickEvent, _window, cx| { + this.commit(cx); + })); + let buttons = div().flex().justify_between().gap_1().child(cancel).child(ok); + + deferred( + anchored() + .position(self.position) + .anchor(Anchor::TopLeft) + .offset(point(px(0.0), px(36.0))) + .snap_to_window_with_margin(px(8.0)) + .child( + div() + .w(px(240.0)) + .p_2() + .rounded_lg() + .border_1() + .border_color(colors.border) + .bg(colors.container) + .flex() + .flex_col() + .gap_1() + .debug_selector(|| "ofx-color-popup".into()) + .on_mouse_up_out( + MouseButton::Left, + cx.listener(|this, _event: &MouseUpEvent, _window, cx| { + this.close_menu(cx); + }), + ) + .on_key_down(cx.listener(|this, event: &KeyDownEvent, _window, cx| { + if event.keystroke.key == "escape" { + this.close_menu(cx); + } + })) + .child(slider_rows) + .child(hex_row) + .child(error_hint) + .child(buttons), + ), + ) + .with_priority(1) + } +} + +impl EventEmitter for OfxColorPicker {} + +impl Render for OfxColorPicker { + fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { + let colors = cx.default_colors().clone(); + let control = self.control; + let committed = self.committed; + + let swatch = div() + .id(ElementId::named_usize("ofx-color-swatch", control)) + .w(px(28.0)) + .h(px(28.0)) + .rounded_md() + .border_1() + .border_color(if self.open { + colors.selected + } else { + colors.border + }) + .cursor_pointer() + .overflow_hidden() + .debug_selector(|| "ofx-color-swatch".into()) + .on_mouse_down( + MouseButton::Left, + cx.listener(|this, _event: &MouseDownEvent, _window, _cx| { + this.was_open_at_down = this.open; + }), + ) + .on_click(cx.listener(|this, event: &ClickEvent, _window, cx| { + if this.was_open_at_down { + this.close_menu(cx); + } else { + this.open_menu(event.position(), cx); + } + cx.stop_propagation(); + })) + .child(canvas( + |bounds, _window, _cx| bounds, + move |bounds, _content, window, cx| { + paint_checker_swatch(bounds, committed, window, cx); + }, + )); + + let popup = if self.open { + self.popup_anchored(cx, &colors) + } else { + deferred(div()) + }; + + div().relative().child(swatch).child(popup) + } +} + +/// Parse `#RRGGBB` or `#RRGGBBAA` into an [`Rgba`] (0..1 components). +/// Rejects a missing `#`, wrong lengths and non-hex digits. +fn parse_hex(input: &str) -> Option { + let s = input.trim(); + let s = s.strip_prefix('#')?; + if s.len() != 6 && s.len() != 8 { + return None; + } + let mut bytes = [0u8; 4]; + for i in 0..s.len() / 2 { + bytes[i] = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).ok()?; + } + let (r, g, b, a) = if s.len() == 8 { + (bytes[0], bytes[1], bytes[2], bytes[3]) + } else { + (bytes[0], bytes[1], bytes[2], 255) + }; + Some(Rgba { + r: r as f32 / 255.0, + g: g as f32 / 255.0, + b: b as f32 / 255.0, + a: a as f32 / 255.0, + }) +} + +/// Format an [`Rgba`] as `#RRGGBB` (opaque alpha) or `#RRGGBBAA`. +fn format_hex(color: Rgba) -> String { + let to = |v: f32| (v.clamp(0.0, 1.0) * 255.0).round() as u8; + let (r, g, b, a) = (to(color.r), to(color.g), to(color.b), to(color.a)); + if a == 255 { + format!("#{r:02X}{g:02X}{b:02X}") + } else { + format!("#{r:02X}{g:02X}{b:02X}{a:02X}") + } +} + +/// Paint a two-tone checkerboard (8 px cells) with `color` over it — the +/// alpha channel reads through the checkerboard. +fn paint_checker_swatch( + bounds: Bounds, + color: Rgba, + window: &mut Window, + _cx: &mut App, +) { + const CELL: f32 = 8.0; + let width = f32::from(bounds.size.width); + let height = f32::from(bounds.size.height); + let cols = (width / CELL).ceil() as i32; + let rows = (height / CELL).ceil() as i32; + let light = Hsla::from(rgb(0xe8e8e8)); + let dark = Hsla::from(rgb(0xc0c0c0)); + for y in 0..rows { + for x in 0..cols { + let cell = Bounds::new( + point( + bounds.left() + px(x as f32 * CELL), + bounds.top() + px(y as f32 * CELL), + ), + size(px(CELL), px(CELL)), + ); + let shade = if (x + y) % 2 == 0 { light } else { dark }; + window.paint_quad(fill(cell, shade)); + } + } + // The colour on top; a transparent alpha blends over the checkerboard. + window.paint_quad(fill(bounds, Hsla::from(color))); +} + +#[cfg(test)] +mod tests { + use super::*; + use gpui::TestAppContext; + + #[test] + fn hex_parsing_and_formatting() { + // #RRGGBB parses with opaque alpha; #RRGGBBAA keeps the alpha. + let c = parse_hex("#1A80E6").expect("6-digit hex parses"); + assert!((c.r - 0x1A as f32 / 255.0).abs() < 1e-6); + assert!((c.g - 0x80 as f32 / 255.0).abs() < 1e-6); + assert!((c.b - 0xE6 as f32 / 255.0).abs() < 1e-6); + assert!((c.a - 1.0).abs() < 1e-6); + let c = parse_hex("#1A80E6FF").expect("8-digit opaque hex parses"); + assert!((c.a - 1.0).abs() < 1e-6); + let c = parse_hex("#1A80E67F").expect("8-digit alpha hex parses"); + assert!((c.a - 0x7F as f32 / 255.0).abs() < 1e-6); + + // format -> parse round-trips exactly (both sides are 8-bit). + for hex in ["#102030", "#0A0B0C", "#11223344", "#FF000080"] { + let parsed = parse_hex(hex).expect("round-trip source parses"); + assert_eq!(format_hex(parsed), hex.to_ascii_uppercase()); + } + + // Opaque colours format to 6 digits, translucent to 8. + assert_eq!( + format_hex(Rgba { + r: 0.0, + g: 0.0, + b: 0.0, + a: 1.0, + }), + "#000000" + ); + assert_eq!( + format_hex(Rgba { + r: 1.0, + g: 1.0, + b: 1.0, + a: 0.5, + }), + "#FFFFFF80" + ); + } + + #[test] + fn hex_parse_rejects_malformed() { + for bad in [ + "", // empty + "102030", // missing '#' + "#12345", // too short + "#1234567", // 7 digits + "#GGHHII", // non-hex digits + "#123456789", // too long + "# 123456", // whitespace inside + ] { + assert!(parse_hex(bad).is_none(), "expected {bad:?} to be rejected"); + } + } + + /// The colour picker (and its four sliders + hex editor) constructs + /// without panicking and paints a swatch. + #[gpui::test] + async fn color_picker_constructs_without_panicking(cx: &mut TestAppContext) { + struct Host { + picker: Entity, + } + impl Render for Host { + fn render(&mut self, _window: &mut Window, _cx: &mut Context) -> impl IntoElement { + div().size_full().child(self.picker.clone()) + } + } + + cx.update(|cx| cx.init_colors()); + let window = cx.open_window(size(px(320.0), px(200.0)), |window, cx| { + let picker = cx.new(|cx| { + OfxColorPicker::new( + 1, + Rgba { + r: 0.4, + g: 0.2, + b: 0.8, + a: 0.5, + }, + window, + cx, + ) + }); + Host { picker } + }); + cx.run_until_parked(); + + let mut visual = gpui::VisualTestContext::from_window(window.into(), cx).into_mut(); + visual.update(|window, cx| { + window.draw(cx).clear(); + }); + assert!( + visual.debug_bounds("ofx-color-swatch").is_some(), + "the colour swatch should be painted" + ); + } +} diff --git a/src/panels/program_viewer.rs b/src/panels/program_viewer.rs index 0dde89d23..23a2ca532 100644 --- a/src/panels/program_viewer.rs +++ b/src/panels/program_viewer.rs @@ -23,15 +23,16 @@ use gpui::colors::DefaultColors; use gpui::dock::{DockPanel, PanelEvent}; use gpui::{ - div, prelude::*, px, AnyElement, App, ClickEvent, Context, Entity, EventEmitter, MouseButton, - Render, SharedString, Window, + div, prelude::*, px, size, AnyElement, App, ClickEvent, Context, Entity, EventEmitter, + MouseButton, Point, Render, SharedString, Window, }; use gpui_widgets::audio_meter::AudioLevelMeter; use gpui_widgets::scopes::{ChromaDataSource, Histogram, LumaDataSource, Vectorscope, Waveform}; -use gpui_widgets::viewer::{ViewerEvent, ViewerWidget}; +use gpui_widgets::viewer::{InteractPointerKind, PlaybackClock, ViewerEvent, ViewerWidget}; use crate::actions::ActionId; use crate::menus::context::{ContextMenuHandle, ContextMenuTriggered}; +use crate::oakui::ofx::InteractViewport; use crate::oakui::timecode::{format_fps, format_resolution}; use crate::oakui::{AppEngine, Monitor}; use crate::panels::commands::{self as panel_commands, PanelCommandHandler}; @@ -72,6 +73,22 @@ impl ChromaDataSource for ScopeState { } } +/// The cached overlay composite the viewer displays while an OFX interact +/// is active (redrawn only when the frame/time/viewport changed or the +/// plugin requested a repaint). +struct OverlayComposite { + /// The base frame the composite was built from (Arc identity compare). + frame: std::sync::Arc, + /// The plugin instance the overlay came from. + instance: u64, + /// The viewport the overlay was drawn at. + viewport: InteractViewport, + /// The playhead seconds used for the draw. + time: f64, + /// The composited image shown in the viewer. + image: std::sync::Arc, +} + /// The program viewer panel. pub struct ProgramViewerPanel { viewer: Entity>, @@ -83,6 +100,12 @@ pub struct ProgramViewerPanel { /// The last CPU frame handed to the viewer (compared by `Arc` identity so /// a paused playhead does not re-upload the picture every frame). last_cpu_frame: Option>, + /// The last base frame the scope samples were analyzed from (the scopes + /// follow the raw frame, not the overlay composite). + scope_frame: Option>, + /// The cached overlay composite (active OFX interact only); `None` when + /// the viewer shows the raw engine frame. + overlay: Option, /// The active body tab. tab: ProgramViewTab, /// The scope samples backing the three scope widgets. @@ -108,18 +131,51 @@ impl ProgramViewerPanel { cx: &mut Context, ) -> Self { let viewer = cx.new(|cx| ViewerWidget::new(3, clock.clone(), window, cx)); - // Route every transport request to the engine's program monitor. - cx.subscribe(&viewer, |this, _viewer, event: &ViewerEvent, cx| { - let monitor = Monitor::Program; - this.engine.update(cx, |engine, cx| match event { - ViewerEvent::PlayRequested { .. } => engine.play(monitor, cx), - ViewerEvent::PauseRequested { .. } => engine.pause(monitor, cx), - ViewerEvent::StepRequested { delta, .. } => engine.step(monitor, *delta, cx), - other => println!("[program viewer] request: {other:?}"), - }); + // Route every transport request to the engine's program monitor, and + // forward the picture's pointer/key events to the active OFX interact + // (no-op when none is live). + cx.subscribe(&viewer, |this, _viewer, event: &ViewerEvent, cx| match event { + ViewerEvent::InteractPointer { + kind, + position, + button, + pressed, + } => this.forward_interact_pointer(*kind, *position, *button, *pressed, cx), + ViewerEvent::InteractKey { down, keystroke } => { + this.forward_interact_key(*down, keystroke, cx) + } + event => { + let monitor = Monitor::Program; + this.engine.update(cx, |engine, cx| match event { + ViewerEvent::PlayRequested { .. } => engine.play(monitor, cx), + ViewerEvent::PauseRequested { .. } => engine.pause(monitor, cx), + ViewerEvent::StepRequested { delta, .. } => engine.step(monitor, *delta, cx), + other => println!("[program viewer] request: {other:?}"), + }); + } }) .detach(); + // A throttled idle pump for the OFX interact: the plugin's UI work + // loop is served on the viewer's own timer (the app tick is + // shell-owned), so an active interact's `idle` action runs without + // coupling to the shell. + let this = cx.weak_entity(); + window + .spawn(cx, async move |cx: &mut gpui::AsyncWindowContext| loop { + cx.background_executor() + .timer(std::time::Duration::from_millis(50)) + .await; + let _ = cx.update(|_window, app| { + if let Some(this) = this.upgrade() { + this.update(app, |_this, _cx| { + crate::oakui::ofx::pump_interact_idle(); + }); + } + }); + }) + .detach(); + let context_menu = ContextMenuHandle::new(Self::on_local_menu_item, window, cx); @@ -137,6 +193,8 @@ impl ProgramViewerPanel { engine, clock, last_cpu_frame: None, + scope_frame: None, + overlay: None, tab: ProgramViewTab::Picture, scope_state, histogram, @@ -175,23 +233,191 @@ impl ProgramViewerPanel { /// Pushes the engine's current frame into the viewer and the scopes, but /// only when it actually changed (the engine caches one image per /// playhead frame, with the scope samples analyzed in the same pass). + /// + /// When an OFX interact is live, the displayed picture is the engine + /// frame with the interact's GL-drawn overlay composited over it; the + /// composite is cached and only redrawn when the frame / playhead / + /// viewport changed or the plugin requested a repaint (so a paused + /// viewer stays inert). fn sync_frame(&mut self, cx: &mut Context) { + // Keep the main-process interact in sync with the inspector's + // current selection (creates/destroys the interact as the target + // moves; a no-op while it is unchanged). + let target = self.engine.read(cx).ofx_interact_target(cx); + crate::oakui::ofx::sync_active_interact(target); + let frame = self.engine.read(cx).cpu_frame(Monitor::Program, cx); + + // The scopes follow the *base* frame (the overlay is not part of the + // analysed picture). if self - .last_cpu_frame + .scope_frame .as_ref() .is_none_or(|last| !std::sync::Arc::ptr_eq(last, &frame)) { - self.last_cpu_frame = Some(frame.clone()); + self.scope_frame = Some(frame.clone()); let scope = self.engine.read(cx).scope_data(Monitor::Program, cx); self.scope_state.update(cx, |state, cx| { state.luma = (*scope.luma).clone(); state.chroma = (*scope.chroma).clone(); cx.notify(); }); - let frame = frame.clone(); + } + + // The picture actually shown: the raw frame, or the overlay + // composite while an interact is live. + let displayed = self.displayed_frame(&frame, cx); + + if self + .last_cpu_frame + .as_ref() + .is_none_or(|last| !std::sync::Arc::ptr_eq(last, &displayed)) + { + self.last_cpu_frame = Some(displayed.clone()); + let displayed = displayed.clone(); self.viewer - .update(cx, |viewer, cx| viewer.set_cpu_frame(Some(frame), cx)); + .update(cx, |viewer, cx| viewer.set_cpu_frame(Some(displayed), cx)); + } + } + + /// The picture for the current engine `frame`: the frame itself, or — + /// when an interact is live — the cached overlay composite, redrawing it + /// when any redraw condition changed. + fn displayed_frame( + &mut self, + frame: &std::sync::Arc, + cx: &mut Context, + ) -> std::sync::Arc { + use std::sync::atomic::Ordering; + + let Some((instance, interact, _)) = crate::oakui::ofx::active_interact() else { + self.overlay = None; + return frame.clone(); + }; + let frame_size = frame.size(0); + let viewport = InteractViewport::at_frame_size( + frame_size.width.0 as u32, + frame_size.height.0 as u32, + ); + let time = self.playhead_seconds(cx); + + // Redraw when the base frame, the target instance, the playhead, or + // the viewport changed, or when the plugin asked for a repaint via + // interactRedraw / interactSwapBuffers (polled and cleared here — + // outside the cache predicate so a redraw request during a cache-miss + // frame is not replayed). + let plugin_redraw = interact.redraw_requested.swap(false, Ordering::Relaxed) + || interact.swap_requested.swap(false, Ordering::Relaxed); + let stale = plugin_redraw + || self.overlay.as_ref().is_none_or(|o| { + !std::sync::Arc::ptr_eq(&o.frame, frame) + || o.instance != instance + || o.time != time + || o.viewport != viewport + }); + if stale { + if let Some(image) = crate::oakui::ofx::draw_interact_composite( + instance, + &interact, + &viewport, + time, + frame, + ) { + self.overlay = Some(OverlayComposite { + frame: frame.clone(), + instance, + viewport, + time, + image: image.clone(), + }); + return image; + } + // The interact is inert (no GL / the plugin failed to draw): + // show the base frame. The next frame re-probes (cheap when GL + // is unavailable). + self.overlay = None; + return frame.clone(); + } + self.overlay.as_ref().unwrap().image.clone() + } + + /// The program monitor's playhead in seconds (the interact's draw/pen + /// `time`). + fn playhead_seconds(&self, cx: &App) -> f64 { + let clock = self.clock.read(cx); + gpui::timeline::frame_to_seconds(clock.current_frame(), clock.frame_rate()) + } + + /// Forwards a picture pointer event to the active OFX interact, mapping + /// the picture-local position into the interact's viewport pixels (the + /// frame's pixel grid). Only the primary (left) button drives + /// pen_down/pen_up (the OFX pen is a two-state device); moves forward + /// pen_motion with the pen-down state reflecting whether a button is + /// held. No-op without an active interact or outside the frame rect. + fn forward_interact_pointer( + &mut self, + kind: InteractPointerKind, + position: Point, + button: Option, + pressed: bool, + cx: &mut Context, + ) { + let Some((_, interact, _)) = crate::oakui::ofx::active_interact() else { + return; + }; + let Some(bounds) = self.viewer.read(cx).picture_bounds() else { + return; + }; + let area = size(f32::from(bounds.size.width), f32::from(bounds.size.height)); + let frame_size = self + .engine + .read(cx) + .cpu_frame(Monitor::Program, cx) + .size(0); + let frame = size(frame_size.width.0 as f32, frame_size.height.0 as f32); + let Some((px, py)) = crate::oakui::ofx::viewport_pixel_to_pen(position, area, frame) + else { + // The pointer is in the letterbox (outside the frame rect). + return; + }; + let time = self.playhead_seconds(cx); + match kind { + InteractPointerKind::Move => { + let _ = interact.pen_motion((px, py), pressed, time); + } + InteractPointerKind::Down if button == Some(MouseButton::Left) => { + let _ = interact.pen_down((px, py), time); + } + InteractPointerKind::Up if button == Some(MouseButton::Left) => { + let _ = interact.pen_up((px, py), time); + } + _ => {} + } + } + + /// Forwards a key event to the active OFX interact. No-op without an + /// active interact. + /// + /// # Consumption semantics + /// + /// gpui dispatches the global keybindings *before* the focused element's + /// key handlers, so a key consumed by the app's action system (space = + /// play/pause, J/K/L = shuttle, …) never reaches the picture's key + /// handler and therefore never reaches the interact — the existing + /// shortcut system keeps priority. A key that reaches here was not + /// consumed by any binding. The plugin's return status is deliberately + /// not acted on: the app does not steal key repeat or other listeners, + /// so the interact is a passive consumer of otherwise-unused keys. + fn forward_interact_key(&mut self, down: bool, keystroke: &gpui::Keystroke, cx: &mut Context) { + let Some((_, interact, _)) = crate::oakui::ofx::active_interact() else { + return; + }; + let (sym, key_string) = crate::oakui::ofx::key_symbol(keystroke); + let time = self.playhead_seconds(cx); + if down { + let _ = interact.key_down(sym, &key_string, time); + } else { + let _ = interact.key_up(sym, &key_string, time); } } diff --git a/src/panels/source_viewer.rs b/src/panels/source_viewer.rs index 06364663b..0f7f19aa2 100644 --- a/src/panels/source_viewer.rs +++ b/src/panels/source_viewer.rs @@ -63,6 +63,10 @@ impl SourceViewerPanel { ViewerEvent::PlayRequested { .. } => engine.play(monitor, cx), ViewerEvent::PauseRequested { .. } => engine.pause(monitor, cx), ViewerEvent::StepRequested { delta, .. } => engine.step(monitor, *delta, cx), + // The source monitor hosts no OFX interact: the picture's + // pointer/key events are not forwarded here (the program + // viewer's panel forwards them when an interact is live). + ViewerEvent::InteractPointer { .. } | ViewerEvent::InteractKey { .. } => {} other => println!("[source viewer] request: {other:?}"), }); })