From 1b0a15192e722960f361b4bf4ec652bd5806404a Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Tue, 1 Sep 2026 20:27:59 +0800 Subject: [PATCH] multicam: wizard + AFV + node-graph preview + performance - wizard: angle multi-select, sync modes, auto-align, create sequence; keeps the host sequence current (host clip = the multicam clip) - build_multicam_sequence: per-angle clip -> SOURCES_INPUT[element], array slot growth, audio angle tracks for AFV - AFV: host linked audio clip follows the switched source (one undo), muted host audio track disables it - node-graph preview: inline producer renders through the traverser (viewer/project-matched graph frames); sequence viewer uses the graph - multicam node value(): element-tagged row keys (sources_in[i]), reads the current source; build_row keys array inputs by element - angle grid: viewer=0 (single-track montage, not whole-graph) - project explorer: rename (dialog) + delete (undoable) real items - performance: decoded-frame LRU, per-process NVDEC quota (1 session, evict before open), GPU composite fail-once fallback, snapshot upload debounced on the engine tick, worker vram budget headroom - timeline clip: multicam overlay via ClipDecorator - wizard menu item moved to Sequence menu --- assets/i18n/de.yaml | 14 + assets/i18n/en-US.yaml | 12 + assets/i18n/es.yaml | 14 + assets/i18n/fr.yaml | 14 + assets/i18n/ja.yaml | 14 + assets/i18n/pt.yaml | 14 + assets/i18n/ru.yaml | 14 + assets/i18n/zh-CN.yaml | 12 + crates/oak-app/src/actions.rs | 3 + crates/oak-app/src/app.rs | 280 +++- crates/oak-app/src/dialogs.rs | 232 ++- crates/oak-app/src/oakui/engine.rs | 107 +- crates/oak-app/src/oakui/graphops.rs | 93 ++ crates/oak-app/src/oakui/mock.rs | 77 +- crates/oak-app/src/oakui/multicam.rs | 452 +++++- crates/oak-app/src/oakui/real.rs | 1430 ++++++++++++++++- crates/oak-app/src/oakui/renderops.rs | 439 ++++- crates/oak-app/src/oakui/waveform.rs | 35 + crates/oak-app/src/panels/project_explorer.rs | 61 +- crates/oak-node/src/nodes/multicamnode.rs | 29 +- crates/oak-node/src/traverser.rs | 12 +- crates/oak-render/src/eval.rs | 149 +- crates/oak-render/src/manager.rs | 73 +- crates/oak-render/src/procpool.rs | 11 +- gpui | 2 +- 25 files changed, 3528 insertions(+), 65 deletions(-) diff --git a/assets/i18n/de.yaml b/assets/i18n/de.yaml index 2e7ff45e0..177bf0699 100644 --- a/assets/i18n/de.yaml +++ b/assets/i18n/de.yaml @@ -145,6 +145,20 @@ "panel.timeline": "Zeitleiste" "panel.effect_library": "Effektbibliothek" "panel.multicam": "Multikamera" +"panel.multicam": "Multikamera" +"menu.sequence.multicam_wizard": "Multikamera-Assistent" +"multicam.wizard.title": "Multikamera-Assistent" +"multicam.wizard.default_name": "Multikamera" +"multicam.wizard.name": "Sequenzname" +"multicam.wizard.sync": "Synchronisierung" +"multicam.sync.audio": "Audiowellenform" +"multicam.sync.timecode": "Quellzeitcode" +"multicam.sync.none": "Keine (keine Ausrichtung)" +"multicam.wizard.angles_label": "Kameras (mindestens zwei auswählen):" +"multicam.wizard.no_footage": "Kein Material im Projekt für eine Multikamera." +"multicam.wizard.need_two": "Bitte mindestens zwei Kameras auswählen." +"multicam.wizard.create": "Erstellen" +"multicam.no_multicam": "Kein Multikamera-Clip erkannt" "multicam.no_multicam": "Kein Multikamera-Clip erkannt" "multicam.switch_1": "Zu Kamera 1 wechseln" "multicam.switch_2": "Zu Kamera 2 wechseln" diff --git a/assets/i18n/en-US.yaml b/assets/i18n/en-US.yaml index ae8d9c1bf..cefe27f77 100644 --- a/assets/i18n/en-US.yaml +++ b/assets/i18n/en-US.yaml @@ -145,6 +145,18 @@ "panel.timeline": "Timeline" "panel.effect_library": "Effect Library" "panel.multicam": "Multi-Cam" +"menu.sequence.multicam_wizard": "Multi-Cam Wizard" +"multicam.wizard.title": "Multi-Cam Wizard" +"multicam.wizard.default_name": "Multi-Cam" +"multicam.wizard.name": "Sequence Name" +"multicam.wizard.sync": "Sync Method" +"multicam.sync.audio": "Audio Waveform" +"multicam.sync.timecode": "Source Timecode" +"multicam.sync.none": "None (No Alignment)" +"multicam.wizard.angles_label": "Angles (select at least two):" +"multicam.wizard.no_footage": "No footage in the project to create a multicam from." +"multicam.wizard.need_two": "Select at least two angles." +"multicam.wizard.create": "Create" "multicam.no_multicam": "No multi-camera clip detected" "multicam.switch_1": "Switch to Camera 1" "multicam.switch_2": "Switch to Camera 2" diff --git a/assets/i18n/es.yaml b/assets/i18n/es.yaml index 19b8da027..65d9e00c3 100644 --- a/assets/i18n/es.yaml +++ b/assets/i18n/es.yaml @@ -145,6 +145,20 @@ "panel.timeline": "Línea de tiempo" "panel.effect_library": "Biblioteca de efectos" "panel.multicam": "Multicámara" +"panel.multicam": "Multicámara" +"menu.sequence.multicam_wizard": "Asistente multicámara" +"multicam.wizard.title": "Asistente multicámara" +"multicam.wizard.default_name": "Multicámara" +"multicam.wizard.name": "Nombre de la secuencia" +"multicam.wizard.sync": "Método de sincronización" +"multicam.sync.audio": "Forma de onda de audio" +"multicam.sync.timecode": "Código de tiempo de origen" +"multicam.sync.none": "Ninguna (sin alineación)" +"multicam.wizard.angles_label": "Cámaras (selecciona al menos dos):" +"multicam.wizard.no_footage": "No hay material en el proyecto para crear una multicámara." +"multicam.wizard.need_two": "Selecciona al menos dos cámaras." +"multicam.wizard.create": "Crear" +"multicam.no_multicam": "No se ha detectado ningún clip multicámara" "multicam.no_multicam": "No se ha detectado ningún clip multicámara" "multicam.switch_1": "Cambiar a cámara 1" "multicam.switch_2": "Cambiar a cámara 2" diff --git a/assets/i18n/fr.yaml b/assets/i18n/fr.yaml index ea2944fde..381ca1de0 100644 --- a/assets/i18n/fr.yaml +++ b/assets/i18n/fr.yaml @@ -145,6 +145,20 @@ "panel.timeline": "Montage" "panel.effect_library": "Bibliothèque d'effets" "panel.multicam": "Multicam" +"panel.multicam": "Multicam" +"menu.sequence.multicam_wizard": "Assistant multicaméra" +"multicam.wizard.title": "Assistant multicaméra" +"multicam.wizard.default_name": "Multicaméra" +"multicam.wizard.name": "Nom de la séquence" +"multicam.wizard.sync": "Méthode de synchronisation" +"multicam.sync.audio": "Forme d’onde audio" +"multicam.sync.timecode": "Code temporel source" +"multicam.sync.none": "Aucune (pas d’alignement)" +"multicam.wizard.angles_label": "Caméras (sélectionnez au moins deux) :" +"multicam.wizard.no_footage": "Aucun média dans le projet pour créer une multicaméra." +"multicam.wizard.need_two": "Sélectionnez au moins deux caméras." +"multicam.wizard.create": "Créer" +"multicam.no_multicam": "Aucun clip multicaméra détecté" "multicam.no_multicam": "Aucun clip multicaméra détecté" "multicam.switch_1": "Passer à la caméra 1" "multicam.switch_2": "Passer à la caméra 2" diff --git a/assets/i18n/ja.yaml b/assets/i18n/ja.yaml index 88dde8a9f..7edad9756 100644 --- a/assets/i18n/ja.yaml +++ b/assets/i18n/ja.yaml @@ -145,6 +145,20 @@ "panel.timeline": "タイムライン" "panel.effect_library": "エフェクトライブラリ" "panel.multicam": "マルチカム" +"panel.multicam": "マルチカム" +"menu.sequence.multicam_wizard": "マルチカムウィザード" +"multicam.wizard.title": "マルチカムウィザード" +"multicam.wizard.default_name": "マルチカム" +"multicam.wizard.name": "シーケンス名" +"multicam.wizard.sync": "同期方法" +"multicam.sync.audio": "オーディオ波形" +"multicam.sync.timecode": "ソースタイムコード" +"multicam.sync.none": "なし（整列しない）" +"multicam.wizard.angles_label": "カメラ（2つ以上選択）：" +"multicam.wizard.no_footage": "プロジェクトにマルチカムを作成できる素材がありません。" +"multicam.wizard.need_two": "少なくとも2つのカメラを選択してください。" +"multicam.wizard.create": "作成" +"multicam.no_multicam": "マルチカメラクリップが検出されませんでした" "multicam.no_multicam": "マルチカメラクリップが検出されませんでした" "multicam.switch_1": "カメラ1に切り替え" "multicam.switch_2": "カメラ2に切り替え" diff --git a/assets/i18n/pt.yaml b/assets/i18n/pt.yaml index b63079679..136a1fa15 100644 --- a/assets/i18n/pt.yaml +++ b/assets/i18n/pt.yaml @@ -145,6 +145,20 @@ "panel.timeline": "Linha do tempo" "panel.effect_library": "Biblioteca de efeitos" "panel.multicam": "Multicâmera" +"panel.multicam": "Multicâmera" +"menu.sequence.multicam_wizard": "Assistente multicâmera" +"multicam.wizard.title": "Assistente multicâmera" +"multicam.wizard.default_name": "Multicâmera" +"multicam.wizard.name": "Nome da sequência" +"multicam.wizard.sync": "Método de sincronização" +"multicam.sync.audio": "Forma de onda de áudio" +"multicam.sync.timecode": "Código de tempo de origem" +"multicam.sync.none": "Nenhuma (sem alinhamento)" +"multicam.wizard.angles_label": "Câmeras (selecione pelo menos duas):" +"multicam.wizard.no_footage": "Nenhum material no projeto para criar uma multicâmera." +"multicam.wizard.need_two": "Selecione pelo menos duas câmeras." +"multicam.wizard.create": "Criar" +"multicam.no_multicam": "Nenhum clipe multicâmera detectado" "multicam.no_multicam": "Nenhum clipe multicâmera detectado" "multicam.switch_1": "Alternar para a câmera 1" "multicam.switch_2": "Alternar para a câmera 2" diff --git a/assets/i18n/ru.yaml b/assets/i18n/ru.yaml index 9a9813282..3bd6fca66 100644 --- a/assets/i18n/ru.yaml +++ b/assets/i18n/ru.yaml @@ -145,6 +145,20 @@ "panel.timeline": "Таймлайн" "panel.effect_library": "Библиотека эффектов" "panel.multicam": "Мультикамера" +"panel.multicam": "Мультикамера" +"menu.sequence.multicam_wizard": "Мастер мультикамеры" +"multicam.wizard.title": "Мастер мультикамеры" +"multicam.wizard.default_name": "Мультикамера" +"multicam.wizard.name": "Имя последовательности" +"multicam.wizard.sync": "Метод синхронизации" +"multicam.sync.audio": "Аудиосигнал (форма волны)" +"multicam.sync.timecode": "Исходный таймкод" +"multicam.sync.none": "Нет (без выравнивания)" +"multicam.wizard.angles_label": "Камеры (выберите не менее двух):" +"multicam.wizard.no_footage": "В проекте нет материала для создания мультикамеры." +"multicam.wizard.need_two": "Выберите не менее двух камер." +"multicam.wizard.create": "Создать" +"multicam.no_multicam": "Не обнаружен клип мультикамеры" "multicam.no_multicam": "Не обнаружен клип мультикамеры" "multicam.switch_1": "Переключиться на камеру 1" "multicam.switch_2": "Переключиться на камеру 2" diff --git a/assets/i18n/zh-CN.yaml b/assets/i18n/zh-CN.yaml index d173969ed..22a57f6fd 100644 --- a/assets/i18n/zh-CN.yaml +++ b/assets/i18n/zh-CN.yaml @@ -145,6 +145,18 @@ "panel.timeline": "时间线" "panel.effect_library": "效果库" "panel.multicam": "多机位" +"menu.sequence.multicam_wizard": "多机位向导" +"multicam.wizard.title": "多机位向导" +"multicam.wizard.default_name": "多机位" +"multicam.wizard.name": "序列名称" +"multicam.wizard.sync": "同步方式" +"multicam.sync.audio": "音频波形" +"multicam.sync.timecode": "源时间码" +"multicam.sync.none": "无（不自动对齐）" +"multicam.wizard.angles_label": "机位（至少选择两个）：" +"multicam.wizard.no_footage": "项目中没有可用于创建多机位的素材。" +"multicam.wizard.need_two": "请至少选择两个机位。" +"multicam.wizard.create": "创建" "multicam.no_multicam": "未检测到多机位片段" "multicam.switch_1": "切换到机位 1" "multicam.switch_2": "切换到机位 2" diff --git a/crates/oak-app/src/actions.rs b/crates/oak-app/src/actions.rs index dbba2c27c..a413a1e1f 100644 --- a/crates/oak-app/src/actions.rs +++ b/crates/oak-app/src/actions.rs @@ -207,6 +207,9 @@ define_actions! { FocusTimeline { cpp: "focustimeline", i18n: "menu.window.timeline", keys: [], route: Global, menu_id: 607 }; FocusEffectLibrary { cpp: "focuseffectlibrary", i18n: "menu.window.effect_library", keys: [], route: Global, menu_id: 608 }; FocusMulticam { cpp: "focusmulticam", i18n: "menu.window.multicam", keys: [], route: Global, menu_id: 609 }; + // The multicam wizard (窗口 → 多机位向导; create a multi-angle + // sequence from selected footage with auto-sync alignment). + MulticamWizard { cpp: "multicamwizard", i18n: "menu.sequence.multicam_wizard", keys: [], route: Global, menu_id: 613 }; MaximizePanel { cpp: "maximizepanel", i18n: "menu.window.maximize_panel", keys: ["`"], route: Global, menu_id: 1070 }; ResetDefaultLayout { cpp: "resetdefaultlayout", i18n: "menu.window.reset_layout", keys: [], route: Global, menu_id: 1071 }; diff --git a/crates/oak-app/src/app.rs b/crates/oak-app/src/app.rs index e46f32c16..246ab262f 100644 --- a/crates/oak-app/src/app.rs +++ b/crates/oak-app/src/app.rs @@ -136,6 +136,10 @@ mod modal_ids { /// The drop-onto-empty-timeline choice (probe the footage's params as /// the sequence's, or set them up manually). pub const DROP_SEQUENCE_CHOICE: usize = 16; + /// The multicam wizard (窗口 → 多机位向导). + pub const MULTICAM_WIZARD: usize = 17; + /// The rename dialog (project explorer 重命名). + pub const RENAME: usize = 18; } /// What a picked platform-dialog path should do. @@ -221,6 +225,20 @@ enum ModalState { /// sequence exists (the paused drop lives in [`OakApp::pending_drop`]). /// The buttons carry the decision, so the state keeps only the modal. DropSequenceChoice { modal: Entity }, + /// The multicam wizard (窗口 → 多机位向导): pick the angle footage, + /// choose the sync mode, then create the multicam sequence. The OK + /// button runs the sync + creation through the content. + MulticamWizard { + modal: Entity, + content: Entity>, + }, + /// The project-explorer rename dialog (right-click → 重命名): the + /// content's field is prefilled with the entry's current name. + EntryRename { + modal: Entity, + content: Entity, + entry_id: u64, + }, } /// A running export: the session the tick loop drains for progress. @@ -245,7 +263,9 @@ impl ModalState { | ModalState::About { modal } | ModalState::NewSequence { modal, .. } | ModalState::SequenceProperties { modal, .. } - | ModalState::DropSequenceChoice { modal, .. } => Some(modal.clone()), + | ModalState::DropSequenceChoice { modal, .. } + | ModalState::MulticamWizard { modal, .. } + | ModalState::EntryRename { modal, .. } => Some(modal.clone()), } } } @@ -558,6 +578,30 @@ impl OakApp { ) .detach(); + // The project explorer's 重命名 / 删除 context items operate on + // the project's entry (rename prompts; delete removes the node). + cx.subscribe( + &panels.project, + |this, + _panel, + event: &crate::panels::project_explorer::RenameRequested, + cx| { + this.open_entry_rename(event.0, cx); + }, + ) + .detach(); + cx.subscribe( + &panels.project, + |this, + _panel, + event: &crate::panels::project_explorer::DeleteRequested, + cx| { + this.engine + .update(cx, |engine, cx| engine.delete_entry(event.0, cx)); + }, + ) + .detach(); + // The project explorer's 新建序列 button opens the new-sequence // dialog (the same one File > New > Sequence… uses). cx.subscribe( @@ -663,13 +707,16 @@ impl OakApp { }), cx, ); - // The multicam panel tabs behind the program viewer (the C++ + // The multicam panel tabs behind the SOURCE viewer (the C++ // default is hidden; the 窗口 menu's Focus Multicam brings it - // forward). The program viewer stays the group's active tab. + // forward). Sharing the source viewer's pane lets the user + // toggle between inspecting source media and switching angles + // in the same window-half — the multicam is NOT tabbed with + // the node editor. dock.add_panel( PanelHandle::new(multicam_panel, cx), Some(DropTarget { - panel: Some(PROGRAM_VIEWER), + panel: Some(SOURCE_VIEWER), zone: DropZone::Center, }), cx, @@ -1157,6 +1204,7 @@ impl OakApp { } A::ProxySettings => self.open_proxy_dialog(cx), A::ProjectProperties => self.open_project_properties(cx), + A::MulticamWizard => self.open_multicam_wizard(cx), A::NewSequence => self.open_new_sequence(cx), A::NewFolder => { if let Err(err) = self @@ -2458,6 +2506,155 @@ impl OakApp { }); } + /// Opens the rename dialog for the project-entry `id` (the project + /// explorer's 重命名 context item), prefilled with the current name. + fn open_entry_rename(&mut self, id: u64, cx: &mut Context) { + if !matches!(self.modal, ModalState::None) { + return; + } + let current = self + .engine + .read(cx) + .project_entry_name(id) + .unwrap_or_default(); + self.spawn_modal(cx, move |window, app| { + let content = app.new(|cx| { + crate::dialogs::RenameContent::new(current.clone(), window, cx) + }); + let modal = app.new(|cx| { + Modal::new( + modal_ids::RENAME, + ModalOptions::new( + crate::i18n::tr("project.context.rename"), + px(420.0), + ) + .with_button(DialogButton::primary(crate::i18n::tr("dialog.ok"))) + .with_button(DialogButton::new( + crate::i18n::tr("dialog.cancel"), + gpui_widgets::dialog::DialogButtonRole::Secondary, + )), + window, + cx, + ) + .with_content(content.clone()) + }); + ModalState::EntryRename { modal, content, entry_id: id } + }); + } + + /// Opens the multicam wizard (窗口 → 多机位向导): pick the angle + /// footage, choose the sync mode, then create the multicam sequence. + fn open_multicam_wizard(&mut self, cx: &mut Context) { + if !matches!(self.modal, ModalState::None) { + return; + } + let engine = self.engine.clone(); + self.spawn_modal(cx, move |window, app| { + let content = app.new(|cx| { + crate::dialogs::MulticamWizardContent::new(engine, window, cx) + }); + let modal = app.new(|cx| { + Modal::new( + modal_ids::MULTICAM_WIZARD, + ModalOptions::new( + crate::i18n::tr("multicam.wizard.title"), + px(480.0), + ) + .with_button(DialogButton::primary(crate::i18n::tr("multicam.wizard.create"))) + .with_button(DialogButton::new( + crate::i18n::tr("dialog.cancel"), + gpui_widgets::dialog::DialogButtonRole::Secondary, + )), + window, + cx, + ) + .with_content(content.clone()) + }); + ModalState::MulticamWizard { modal, content } + }); + } + + /// The wizard's OK: run the sync (or use timecode/no-align per the + /// chosen mode), create the multicam sequence, then close. A rejected + /// create keeps the dialog open with a message row. + fn commit_multicam_wizard( + &mut self, + content: &Entity>, + cx: &mut Context, + ) -> Result<(), String> { + let selection = content.read(cx).selection(); + if selection.len() < 2 { + return Err(crate::i18n::tr("multicam.wizard.need_two").to_string()); + } + let sync_mode = content.read(cx).sync_mode(cx); + let name = content.read(cx).name(cx).to_string(); + // Sync: audio-waveform correlation (0), source timecode (1), or + // no alignment (2 — all angles start at 0). The engine's sync + // fails when the reference angle has no decodable audio: the + // wizard then falls back to source timecode deltas (absolute + // timestamps, so the first angle is the alignment reference). + let offsets: Vec = match sync_mode { + 0 => { + let synced = self + .engine + .read(cx) + .multicam_wizard_sync_offsets(&selection); + match synced { + Ok(offsets) => { + let reference_tc = selection[0].source_timecode; + let mut by_id = std::collections::HashMap::new(); + for off in &offsets { + by_id.insert(off.footage, off.offset_s); + } + selection + .iter() + .map(|f| { + by_id.get(&f.id).copied().unwrap_or_else(|| { + // No audio-sync answer for this angle: + // the source timecode delta relative to + // the reference (the fallback). + let ref_tc = reference_tc.unwrap_or(0); + f.source_timecode.unwrap_or(0) as f64 / 1000.0 + - ref_tc as f64 / 1000.0 + }) + }) + .collect() + } + Err(_) => { + let reference_tc = selection[0].source_timecode. + map(|t| t as f64 / 1000.0).unwrap_or(0.0); + selection + .iter() + .map(|f| { + f.source_timecode.map(|t| t as f64 / 1000.0) + .unwrap_or(0.0) + - reference_tc + }) + .collect() + } + } + } + 1 => { + let reference_tc = selection[0].source_timecode.map(|t| t as f64 / 1000.0) + .unwrap_or(0.0); + selection + .iter() + .map(|f| { + f.source_timecode.map(|t| t as f64 / 1000.0).unwrap_or(0.0) + - reference_tc + }) + .collect() + } + _ => vec![0.0; selection.len()], + }; + let result = self + .engine + .update(cx, |engine, cx| { + engine.multicam_create_sequence(selection, offsets, name, cx) + }); + result.map(|_| ()) + } + /// Opens the export dialog. fn open_export_dialog(&mut self, cx: &mut Context) { if self.engine.read(cx).current_sequence().is_none() { @@ -2716,6 +2913,39 @@ impl OakApp { } } } + modal_ids::MULTICAM_WIZARD => { + if let ModalState::MulticamWizard { content, .. } = &self.modal { + let content = content.clone(); + if *button == 0 { + // OK: create the multicam; a rejected create + // keeps the dialog open. + match self.commit_multicam_wizard(&content, cx) { + Ok(()) => self.close_modal(cx), + Err(err) => { + println!("[multicam wizard] {err}"); + } + } + } else { + self.close_modal(cx); + } + } + } + modal_ids::RENAME => { + if let ModalState::EntryRename { content, entry_id, .. } = &self.modal { + if *button == 0 { + // OK: apply the new name. + let name = content.read(cx).value(cx); + let id = *entry_id; + let engine = self.engine.clone(); + engine.update(cx, |engine, cx| { + engine.rename_entry(id, name.to_string(), cx) + }); + self.close_modal(cx); + } else { + self.close_modal(cx); + } + } + } modal_ids::SEQUENCE_PROPERTIES => { if let ModalState::SequenceProperties { content, .. } = &self.modal { let content = content.clone(); @@ -2867,7 +3097,7 @@ fn default_dock_target(id: PanelId) -> Option { panel: None, zone: DropZone::Bottom, }), - MULTICAM => Some(t(PROGRAM_VIEWER, DropZone::Center)), + MULTICAM => Some(t(SOURCE_VIEWER, DropZone::Center)), _ => Some(t(PROJECT, DropZone::Center)), } } @@ -3114,6 +3344,7 @@ fn make_menus(state: MenuState) -> Vec { menu_item(A::SeqCacheInOut), menu_item(A::SeqCacheClear).separated(), menu_item(A::SequenceSettings).disabled(), + menu_item(A::MulticamWizard).separated(), ]), ), MenuBarEntry::new( @@ -3132,7 +3363,7 @@ fn make_menus(state: MenuState) -> Vec { last.separator_after = true; } window_items.push(menu_item(A::MaximizePanel)); - window_items.push(menu_item(A::ResetDefaultLayout)); + window_items.push(menu_item(A::ResetDefaultLayout).separated()); window_items }), ), @@ -4455,6 +4686,43 @@ mod tests { ); } + /// The 窗口/Window → 多机位向导 menu item opens the wizard modal with + /// the engine's wizard footage rows (the mock offers a demo list), and + /// the created modal carries the content entity (the "click freezes" + /// regression guard). + #[gpui::test] + async fn multicam_wizard_menu_opens_the_modal(cx: &mut TestAppContext) { + let _guard = crate::actions::shortcuts_test_lock() + .lock() + .unwrap_or_else(|e| e.into_inner()); + let _lang = crate::i18n::lang_test_lock() + .lock() + .unwrap_or_else(|e| e.into_inner()); + let (window, root) = mock_shell(cx); + + cx.update(|app| { + root.update(app, |app, cx| { + app.on_menu(ActionId::MulticamWizard.menu_id(), cx) + }) + }); + cx.run_until_parked(); + cx.update_window(window.into(), |_root, window, cx| { + window.draw(cx).clear(); + }) + .expect("window is still open"); + let modal_rows = cx.read(|app| match &root.read(app).modal { + ModalState::MulticamWizard { content, .. } => content.read(app).selection().len(), + _ => panic!("the multicam wizard modal should be open"), + }); + // The mock offers four demo footage rows. + let footage = cx.read(|app| { + let engine = root.read(app).engine.read(app); + engine.multicam_wizard_footage().unwrap_or_default() + }); + assert_eq!(footage.len(), 4, "the mock exposes the demo footage list"); + assert_eq!(modal_rows, 0, "no angles are pre-checked"); + } + /// Arrow keys move the search selection and Enter runs the selected action /// through the same dispatch path the menu clicks take (here the dialog /// closes because the action dispatched successfully). diff --git a/crates/oak-app/src/dialogs.rs b/crates/oak-app/src/dialogs.rs index 703825f72..378891199 100644 --- a/crates/oak-app/src/dialogs.rs +++ b/crates/oak-app/src/dialogs.rs @@ -36,8 +36,8 @@ use gpui::colors::DefaultColors; use gpui::prelude::*; use gpui::timeline::FrameRate; use gpui::{ - div, px, App, Context, ElementId, Entity, EventEmitter, FocusHandle, Focusable, Keystroke, - PathPromptOptions, Render, SharedString, Window, + div, px, App, ClickEvent, Context, ElementId, Entity, EventEmitter, FocusHandle, Focusable, + InteractiveElement, Keystroke, PathPromptOptions, Render, SharedString, Window, }; use gpui_elements::editable_text::{EditableTextState, StringStorage, TextChanged}; @@ -3630,10 +3630,236 @@ impl Render for DropSequenceChoiceContent { } } +/// The multicam wizard dialog content: pick the angle footage, choose +/// the sync mode, name the multicam sequence. The host reads the inputs +/// through [`Self::selection`] / [`Self::sync_mode`] / [`Self::name`] +/// when the OK button fires. +pub struct MulticamWizardContent { + engine: Entity, + /// The name field (prefilled "Multi-Cam"). + name: Entity, + /// Sync mode combo (0 = audio waveform, 1 = source timecode, 2 = no + /// alignment). + sync: Entity, + /// Selection state per footage row: `(entry, checked)`. + rows: Vec<(crate::oakui::engine::WizardFootage, bool)>, + /// The wheel to scroll the long angle list. + scrolled: bool, +} + +/// The wizard's sync mode combo values (display order). +const WIZARD_SYNC_MODES: &[&str] = &[ + "multicam.sync.audio", + "multicam.sync.timecode", + "multicam.sync.none", +]; + +impl MulticamWizardContent { + /// Builds the content seeded with the engine's wizard footage. + pub fn new( + engine: Entity, + window: &mut Window, + cx: &mut Context, + ) -> Self { + let name = cx.new(|cx| { + let editor = cx.new(|cx| EditableTextState::new(StringStorage::default(), cx)); + TextValue { editor } + }); + name.update(cx, |field, cx| { + field.set_value(i18n::tr("multicam.wizard.default_name"), cx) + }); + let sync = cx.new(|cx| { + let options = WIZARD_SYNC_MODES + .iter() + .enumerate() + .map(|(i, key)| ComboBoxOption::new(i, i18n::tr(key))) + .collect(); + ComboBox::new(60, options, window, cx) + }); + sync.update(cx, |combo, cx| combo.set_selected(Some(0), cx)); + let rows = engine + .read(cx) + .multicam_wizard_footage() + .unwrap_or_default() + .into_iter() + .map(|entry| (entry, false)) + .collect(); + Self { + engine, + name, + sync, + rows, + scrolled: false, + } + } + + /// Toggles the checked state of row `index`. + pub fn toggle_row(&mut self, index: usize, cx: &mut Context) { + if let Some(row) = self.rows.get_mut(index) { + row.1 = !row.1; + cx.notify(); + } + } + + /// The angle entries currently checked, in row order. + pub fn selection(&self) -> Vec { + self.rows + .iter() + .filter(|row| row.1) + .map(|row| row.0.clone()) + .collect() + } + + /// The selected sync mode (0 = audio, 1 = timecode, 2 = none). + pub fn sync_mode(&self, cx: &App) -> usize { + self.sync.read(cx).selected().unwrap_or(0) + } + + /// The sequence name entered. + pub fn name(&self, cx: &App) -> SharedString { + self.name.read(cx).value(cx) + } + + /// Whether the engine offers any wizard footage (drives the empty + /// hint row). + pub fn has_rows(&self) -> bool { + !self.rows.is_empty() + } + + /// The engine entity (the host needs it to run the sync + create). + pub fn engine(&self) -> &Entity { + &self.engine + } +} + +impl Render for MulticamWizardContent { + fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { + let colors = cx.default_colors().clone(); + let mut rows_div = div().flex().flex_col().gap_1().w_full(); + for (index, (entry, checked)) in self.rows.iter().enumerate() { + let is_checked = *checked; + let row = div() + .id(ElementId::named_usize("multicam-wizard-angle", index)) + .flex() + .items_center() + .gap_2() + .px_1() + .py_0p5() + .rounded_sm() + .hover(|style| style.bg(colors.container)) + .cursor_pointer() + .on_click(cx.listener( + move |this: &mut Self, _event: &ClickEvent, _window, cx| { + this.toggle_row(index, cx); + }, + )) + .child( + div() + .size(px(12.0)) + .flex() + .items_center() + .justify_center() + .bg(if is_checked { colors.selected } else { colors.background }) + .text_color(colors.text) + .child(if is_checked { "✓" } else { "" }), + ) + .child( + div() + .flex_1() + .text_color(colors.text) + .child(entry.name.clone()), + ) + .child( + div() + .text_xs() + .text_color(colors.disabled) + .child(entry.duration_s.map(|d| format!("{d:.1}s")).unwrap_or_default()), + ); + rows_div = rows_div.child(row); + } + let angles = div() + .id("multicam-wizard-angles") + .max_h(px(220.0)) + .overflow_y_scroll() + .child(rows_div); + let stock = if self.rows.is_empty() { + div() + .text_color(colors.disabled) + .text_xs() + .child(i18n::tr("multicam.wizard.no_footage")) + .into_any_element() + } else { + div().into_any_element() + }; + div() + .flex() + .flex_col() + .gap_3() + .w_full() + .child(form_row( + &colors, + i18n::tr("multicam.wizard.name").into(), + self.name.clone(), + )) + .child(form_row( + &colors, + i18n::tr("multicam.wizard.sync").into(), + self.sync.clone(), + )) + .child( + div() + .text_sm() + .text_color(colors.text) + .child(i18n::tr("multicam.wizard.angles_label")), + ) + .child(angles) + .child(stock) + } +} + +/// The rename dialog: a single text field prefilled with the entry's +/// current name (the project-explorer 重命名 context item; the host reads +/// [`Self::value`] on OK and calls the engine's `rename_entry`). +pub struct RenameContent { + field: Entity, +} + +impl RenameContent { + /// Builds the dialog seeded with the current name. + pub fn new(current: SharedString, window: &mut Window, cx: &mut Context) -> Self { + let field = cx.new(|cx| { + let editor = cx.new(|cx| EditableTextState::new(StringStorage::default(), cx)); + TextValue { editor } + }); + field.update(cx, |field, cx| field.set_value(current, cx)); + Self { field } + } + + /// The name to apply. + pub fn value(&self, cx: &App) -> SharedString { + self.field.read(cx).value(cx) + } +} + +impl Render for RenameContent { + fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { + let colors = cx.default_colors().clone(); + div() + .flex() + .flex_col() + .gap_3() + .w_full() + .child(form_row( + &colors, + i18n::tr("project.context.rename").into(), + self.field.clone(), + )) + } +} + #[cfg(test)] mod tests { use super::*; - fn keystroke(key: &str) -> Keystroke { gpui::Keystroke::parse(key).unwrap() } diff --git a/crates/oak-app/src/oakui/engine.rs b/crates/oak-app/src/oakui/engine.rs index 2961a531c..f8e04a560 100644 --- a/crates/oak-app/src/oakui/engine.rs +++ b/crates/oak-app/src/oakui/engine.rs @@ -40,7 +40,7 @@ use gpui::node_graph::{NodeGraphDataSource, NodeGraphEvent}; use gpui::timeline::{ ClipId, Frame, FrameRate, TimelineDataSource, TimelineEvent, TrackData, TrackKind, }; -use gpui::{App, Context, Entity, Pixels, Point, Rgba, RenderImage}; +use gpui::{App, Context, Entity, Pixels, Point, Rgba, RenderImage, SharedString}; use gpui_widgets::audio_meter::AudioMeterDataSource; use gpui_widgets::project_explorer::ProjectDataSource; use gpui_widgets::viewer::PlaybackClock; @@ -532,6 +532,12 @@ pub trait AppEngine: None } + /// The display name of project-entry `id` (the rename dialog's seed). + fn project_entry_name(&self, id: u64) -> Option { + let _ = id; + None + } + /// Replaces the footage of project-explorer entry `id` with the media /// file at `path` (the C++ `ReplaceFootage` flow). Default: /// unsupported. @@ -545,6 +551,19 @@ pub trait AppEngine: Err("replace footage not supported".into()) } + /// Renames the project-explorer entry `id` (footage / folder / + /// sequence) in the project model. Default: unsupported. + fn rename_entry(&mut self, id: u64, new_name: String, cx: &mut Context) { + let _ = (id, new_name, cx); + } + + /// Deletes the project-explorer entry `id` (footage / folder / + /// sequence) and its graph neighborhoods — ONE undoable entry. Default: + /// unsupported. + fn delete_entry(&mut self, id: u64, cx: &mut Context) { + let _ = (id, cx); + } + // ------------------------------------------------------------------- // Sequence markers & work area (M12 P4): the facade surfaces are // undoable, mirroring Olive (MarkerAdd/MarkerRemove/WorkareaSet*). @@ -974,6 +993,13 @@ pub trait AppEngine: false } + /// Opens the sequence `id` in the timeline (the project-explorer + /// double-click on a sequence entry). Default no-op for engines + /// without sequence tabs. + fn open_sequence_id(&mut self, id: u64, cx: &mut Context) { + let _ = (id, cx); + } + /// The sequence parameters of `id` (the sequence-properties dialog /// seed), or `None` when the entry is not a sequence. fn sequence_parameters(&self, id: u64) -> Option { @@ -1210,6 +1236,50 @@ pub trait AppEngine: /// the status bar. fn backend_name(&self) -> &'static str; + // ------------------------------------------------------------------- + // Multicam wizard (the C++ Multicam dialog + auto-sync): creating a + // new multicam sequence from a set of footage angles, with optional + // audio-waveform auto alignment. Defaults degrade to "unsupported", + // so engines without the real project surface keep compiling. + // ------------------------------------------------------------------- + + /// The footage entries the multicam wizard can offer (project-root + /// footage not already inside a multicam). `None` = no wizard data + /// source (the dialog's angle picker stays empty). + fn multicam_wizard_footage(&self) -> Option> { + None + } + + /// Estimate the audio alignment offsets between the wizard's selected + /// angles (the first selected footage is the reference; each other + /// angle gets a sample offset relative to it, estimated by the RMS + /// envelope correlation). Returns `Err` when the engine cannot run + /// the correlation (missing audio / too short) — the wizard then falls + /// back to timecode alignment. + fn multicam_wizard_sync_offsets( + &self, + _selected: &[WizardFootage], + ) -> Result, String> { + Err("offline sync not supported".to_string()) + } + + /// Creates the multicam sequence: a new sequence with one video track + /// per selected angle (top track = angle 0), each track carrying a + /// clip of the corresponding footage placed at `offsets[i]` seconds + /// (waveform-sync offsets, or the source timecode delta, or 0), plus a + /// multicam node feeding a fresh clip on the CURRENT timeline's top + /// video track at the playhead. One undo entry. Returns the sequence + /// identity (the app opens it). + fn multicam_create_sequence( + &mut self, + _selected: Vec, + _offsets: Vec, + _name: String, + _cx: &mut Context, + ) -> Result { + Err("multicam wizard not supported".to_string()) + } + /// Arms or disarms the program viewer's eyedropper. While armed the /// viewer samples the pixel under the cursor on click and reports it via /// [`Self::eyedropper_picked`]; the OFX color picker drives both. @@ -1260,6 +1330,41 @@ pub struct MulticamState { pub current_source: i32, } +/// One angle candidate of the multicam wizard: a root-level footage entry +/// the user can pick for the new multicam sequence. +#[derive(Debug, Clone, PartialEq)] +pub struct WizardFootage { + /// The project-explorer entry id (the footage node identity). + pub id: u64, + /// Display name (the explorer label — usually the file name). + pub name: SharedString, + /// The footage's source timecode, if the media carries one (the + /// absolute reference for the no-audio sync fallback). + pub source_timecode: Option, + /// The footage's duration in seconds (`None` = not probed). + pub duration_s: Option, + /// Whether the footage has a decodable audio stream (sync via RMS + /// envelope requires one; `None` = not probed). + pub has_audio: Option, +} + +/// The audio-sync alignment result of one angle relative to the wizard's +/// reference angle (`estimate_offset` semantics: the candidate's media +/// time that matches the reference's media time 0 — a NEGATIVE offset +/// means the candidate starts earlier). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct WizardSyncOffset { + /// The angle's project-explorer entry id (the same value the wizard + /// selected). + pub footage: u64, + /// Offset in seconds relative to the reference (the first selected + /// footage gets 0.0). + pub offset_s: f64, + /// Normalized correlation confidence in `[0, 1]` (`None` when the + /// offset came from timecode alignment, not the audio correlation). + pub confidence: Option, +} + /// The lifecycle state of one footage's proxy (the UI mirror of /// `oak_codec::proxymanager::ProxyState`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/oak-app/src/oakui/graphops.rs b/crates/oak-app/src/oakui/graphops.rs index e759b6451..488787cd2 100644 --- a/crates/oak-app/src/oakui/graphops.rs +++ b/crates/oak-app/src/oakui/graphops.rs @@ -389,6 +389,19 @@ pub fn node_type_id(g: &Graph, id: NodeId) -> String { .unwrap_or_default() } +/// The project's multicam nodes (the wizard/sequence drop rewire uses the +/// one whose source sequence matches). +pub fn multicam_nodes(g: &Graph) -> Vec { + g.node_ids() + .into_iter() + .filter(|&id| { + g.get(id) + .map(|e| e.behavior.type_id() == "org.olivevideoeditor.Olive.multicam") + .unwrap_or(false) + }) + .collect() +} + /// The sequence's video format `(width, height, rate)` from its first /// video stream. pub fn sequence_video_params(g: &Graph, seq: NodeId) -> Option<(i32, i32, Rational)> { @@ -1492,6 +1505,86 @@ pub fn place_footage_clip( Ok(clip) } +/// Places ONE video clip fed by a sequence node — a nested-sequence clip +/// (drag a sequence entry from the project explorer onto the timeline). +/// The clip reads the sequence's output during playback; the sequence's +/// own tracks stay in the source sequence (a frame, not a folder of +/// tracks, on the host timeline). +pub fn place_nested_sequence_clip( + p: &ProjectRef, + host_seq: NodeId, + source_seq: NodeId, + track_index: usize, + in_ts: i64, + out_ts: i64, +) -> Result { + if in_ts < 0 || out_ts <= in_ts { + return Err("invalid clip range (need 0 <= in < out)".to_string()); + } + let (tb, list, source_behavior) = { + let g = lock(p); + let s = sequence_behavior(&g.graph, source_seq) + .ok_or_else(|| "the source sequence is not in the project".to_string())?; + let tb = sequence_time_base(&g.graph, host_seq) + .ok_or_else(|| "host sequence has no valid frame rate".to_string())?; + let list = track_list_of(&g.graph, host_seq, TrackType::Video) + .ok_or_else(|| "host sequence has no video track list".to_string())?; + let label = format!("{}（序列）", node_label(&g.graph, source_seq)); + (tb, list, label) + }; + let track_count = { + let g = lock(p); + track_list_behavior(&g.graph, list) + .map(|l| l.tracks.len()) + .unwrap_or(0) + }; + if track_index >= track_count { + return Err(format!( + "track index {track_index} out of range ({track_count} tracks)" + )); + } + + let in_r = ts_to_rational(in_ts, tb); + let out_r = ts_to_rational(out_ts, tb); + let length = out_r - in_r; + + let clip = { + let mut g = lock(p); + let (ccore, cbehavior) = oak_node::block::clip_create(); + let mut core = ccore; + core.label = source_behavior; + g.graph.add_node(core, cbehavior) + }; + { + let mut g = lock(p); + let Some(clip_behavior) = g + .graph + .get_mut(clip) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + else { + return Err("clip node is not a clip".to_string()); + }; + clip_behavior.core.range = oak_core::TimeRange::new(in_r, length); + } + + let place = oak_timeline::undopointer::TrackPlaceBlockCommand::new( + node_ref(p, list), + track_index as i32, + node_ref(p, clip), + in_r, + ) + .to_command(); + let edge = connect_command( + p, + source_seq, + clip, + oak_node::block::clip_input::TEXTURE_INPUT, + )?; + push_multi(vec![place, edge], "Add Clip")?; + Ok(clip) +} + /// Place linked clips of one footage on several tracks in ONE undoable /// "Add Clip" entry (the NLE A/V-drop: a video-with-audio file lands as /// a video clip plus a linked audio clip). `placements` lists the diff --git a/crates/oak-app/src/oakui/mock.rs b/crates/oak-app/src/oakui/mock.rs index 4c74a29a5..7ff7e6bbc 100644 --- a/crates/oak-app/src/oakui/mock.rs +++ b/crates/oak-app/src/oakui/mock.rs @@ -68,7 +68,7 @@ use oak_timeline::util::{block_clip_create, track_append_block}; use super::engine::{ AppEngine, EngineGateway, ExportEvent, ExportSession, LibraryProject, Monitor, MulticamState, - Project, ScopeData, Sequence, VideoFormat, + Project, ScopeData, Sequence, VideoFormat, WizardFootage, WizardSyncOffset, }; use super::graphops; use super::transport::TransportState; @@ -2339,6 +2339,81 @@ impl AppEngine for MockEngine { cx.notify(); } + // --- multicam wizard (mock: the demo footage entries) --------------- + + fn multicam_wizard_footage(&self) -> Option> { + Some(vec![ + WizardFootage { + id: 3, + name: "第一稿.mp4".into(), + source_timecode: Some(0), + duration_s: Some(30.0), + has_audio: Some(true), + }, + WizardFootage { + id: 10, + name: "intro.mov".into(), + source_timecode: Some(1000), + duration_s: Some(45.0), + has_audio: Some(true), + }, + WizardFootage { + id: 11, + name: "b-roll.mp4".into(), + source_timecode: Some(2000), + duration_s: Some(60.0), + has_audio: Some(true), + }, + WizardFootage { + id: 12, + name: "interview.mov".into(), + source_timecode: Some(500), + duration_s: Some(90.0), + has_audio: Some(true), + }, + ]) + } + + fn multicam_wizard_sync_offsets( + &self, + selected: &[WizardFootage], + ) -> Result, String> { + // The mock's correlation is a demo: each angle's offset is derived + // from its source timecode relative to the reference (angle 0). + let reference_tc = selected + .first() + .and_then(|f| f.source_timecode) + .unwrap_or(0) as f64; + Ok(selected + .iter() + .map(|f| WizardSyncOffset { + footage: f.id, + offset_s: f.source_timecode.map(|t| t as f64 / 1000.0).unwrap_or(0.0) + - reference_tc / 1000.0, + confidence: f.has_audio.unwrap_or(false).then_some(0.95), + }) + .collect()) + } + + fn multicam_create_sequence( + &mut self, + selected: Vec, + offsets: Vec, + name: String, + cx: &mut Context, + ) -> Result { + let _ = (&selected, offsets); + // The mock has no project graph for a NEW sequence outside its + // demo: report the creation by switching the demo's name and + // returning a stable identity (the panel already shows the demo + // multicam — the wizard's create is a verbose success). + if self.sequence.name.is_empty() { + self.sequence.name = name; + } + cx.notify(); + Ok(1) + } + fn backend_name(&self) -> &'static str { "mock" } diff --git a/crates/oak-app/src/oakui/multicam.rs b/crates/oak-app/src/oakui/multicam.rs index 02054bd95..caeb284e8 100644 --- a/crates/oak-app/src/oakui/multicam.rs +++ b/crates/oak-app/src/oakui/multicam.rs @@ -40,8 +40,10 @@ use oak_node::nodes::multicamnode::{SEQUENCE_INPUT, SEQUENCE_TYPE_INPUT}; use oak_node::sequence::SequenceBehavior; use oak_node::track::TrackType; -use super::engine::MulticamState; -use super::graphops::{self, lock, ProjectRef}; +use super::engine::{MulticamState, WizardFootage}; +use super::graphops::{ + self, lock, sequence_behavior, track_list_behavior, track_list_of, ProjectRef, +}; /// Whether `id` names a sequence node (C++ `dynamic_cast` / /// the facade's `oakengine_node_is_sequence`). @@ -57,6 +59,19 @@ fn connected_output(g: &Graph, node: NodeId, input: &str, element: i32) -> Optio g.connected_output(node, input, element) } +/// Whether the clip's texture chain feeds directly from a multicam node — +/// the timeline overlay's flag (`true` for the wizard host clip and the +/// dropped multicam source clip; both get the accented treatment). +pub fn clip_is_multicam(g: &Graph, clip: NodeId) -> bool { + connected_output(g, clip, clip_input::TEXTURE_INPUT, -1) + .map(|source| { + g.get(source) + .map(|e| e.behavior.type_id() == "org.olivevideoeditor.Olive.multicam") + .unwrap_or(false) + }) + .unwrap_or(false) +} + /// The C++ `find_input_node_internal` walk: check `node`'s input /// connections for a match, recursing into each source. Collects the first /// sequence found (stopping at `maximum` matches, `0` = unlimited). @@ -198,6 +213,377 @@ pub fn clip_at_playhead_with_multicam(p: &ProjectRef, seq: NodeId, time: Rationa }) } +/// The footage entries the multicam wizard offers: root-level footage +/// nodes (skipping folders). Display labels are the node labels (file +/// names); source timecode and duration come from the probed behavior. +pub fn wizard_footage(p: &ProjectRef) -> Vec { + // The root identity first, with the guard dropped BEFORE children + // runs: `children` takes the project lock again internally (through + // `find_by_identity`), and a still-live guard from the argument + // expression re-enters the non-recursive std Mutex — the wizard + // freeze regression. + let root_id = graphops::lock(p).root.identity(); + let entries = super::projectbrowser::children(p, root_id); + let mut out = Vec::new(); + for entry in entries { + if entry.is_dir { + continue; + } + let Some(node) = graphops::id_of(entry.id) else { + continue; + }; + let g = graphops::lock(p); + let Some(f) = graphops::footage_behavior(&g.graph, node) else { + continue; + }; + if f.filename.is_empty() { + continue; + } + let name = graphops::node_label(&g.graph, node); + let source_timecode = f.has_source_start_time.then(|| { + let t = f.source_start_time; + (t.numerator() as f64 / t.denominator().max(1) as f64).round() as i64 + }); + let duration_s = { + let d = f.duration(); + (d.denominator() != 0).then(|| d.numerator() as f64 / d.denominator() as f64) + }; + let has_audio = Some(f.audio_stream_count() > 0); + out.push(WizardFootage { + id: entry.id, + name: name.into(), + source_timecode, + duration_s, + has_audio, + }); + } + out +} + +/// Builds the multicam sequence graph for the wizard (non-undoable — +/// the caller pushes the whole construction as one undo entry). +/// +/// * a new sequence `source` with one VIDEO track per selected footage +/// (top track = angle 0); the clip on each track starts at +/// `offsets[i]` seconds (media-in 0 — the sync offset lands on the +/// timeline, so switching angles aligns the clips); +/// * a multicam node wired `sequence_in` = the source sequence and the +/// source's video track list as its `sources_in`. +/// +/// The timeline clip that FEEDS the multicam output is the caller's job +/// (the app places it on the current sequence at the playhead and +/// pushes everything as one undo entry). Returns +/// `(source_sequence, multicam_node)`. +pub fn build_multicam_sequence( + p: &ProjectRef, + selected: &[WizardFootage], + offsets: &[f64], + name: &str, +) -> Result<(NodeId, NodeId), String> { + if selected.is_empty() { + return Err("no angles selected".to_string()); + } + if selected.len() != offsets.len() { + return Err("angle/offset count mismatch".to_string()); + } + + let mut seq: Option = None; + let mut mc: Option = None; + { + let mut g = lock(p); + let (score, sbehavior) = oak_node::sequence::SequenceBehavior::create(); + let mut core = oak_node::node::NodeCore::new(); + core.label = name.to_string(); + let seq_id = g.graph.add_node(core, sbehavior); + seq = Some(seq_id); + + // The video track list (the vec-shape course: `track_lists` is a + // Vec on the sequence, `tracks` a Vec on the list — not + // graph edges; mirror `find_or_create_track_list`). + let list = if let Some(list) = track_list_of(&g.graph, seq_id, TrackType::Video) { + list + } else { + let (lcore, lbehavior) = oak_node::track::TrackListBehavior::create(); + let mut behavior = lbehavior; + if let Some(a) = behavior.as_any_mut() { + if let Some(list) = a.downcast_mut::() { + list.kind = TrackType::Video; + list.array_base = + sequence_behavior(&g.graph, seq_id).map(|s| s.track_lists.len()).unwrap_or(0) + as i32; + } + } + let list = g.graph.add_node(lcore, behavior); + if let Some(s) = g + .graph + .get_mut(seq_id) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + s.track_lists.push(list); + } + list + }; + // One video track per angle, pushed onto the list's `tracks`. + let mut track_ids = Vec::new(); + for _ in 0..selected.len() { + let (tcore, tbehavior) = oak_node::track::TrackBehavior::create(); + let track = g.graph.add_node(tcore, tbehavior); + if let Some(l) = g + .graph + .get_mut(list) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + l.tracks.push(track); + } + track_ids.push(track); + } + + let tb_den = 25i64; // 25 fps default wizard rate + // The source's AUDIO track list (built only when one or more + // angles carries audio — the AFV half of the multicam: each + // angle's audio clip sits on its own track, and the host audio + // clip re-points at the CURRENT source's audio on switch). + let mut audio_list: Option = None; + let mut audio_track_ids: Vec = Vec::new(); + if selected.iter().any(|e| e.has_audio.unwrap_or(false)) { + audio_list = Some(if let Some(list) = + track_list_of(&g.graph, seq_id, TrackType::Audio) + { + list + } else { + let (lcore, lbehavior) = oak_node::track::TrackListBehavior::create(); + let mut behavior = lbehavior; + if let Some(a) = behavior.as_any_mut() { + if let Some(list) = a.downcast_mut::() { + list.kind = TrackType::Audio; + list.array_base = + sequence_behavior(&g.graph, seq_id).map(|s| s.track_lists.len()).unwrap_or(0) + as i32; + } + } + let list = g.graph.add_node(lcore, behavior); + if let Some(s) = g + .graph + .get_mut(seq_id) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + s.track_lists.push(list); + } + list + }); + let list = audio_list.unwrap(); + for _ in 0..selected.len() { + let (tcore, tbehavior) = oak_node::track::TrackBehavior::create(); + let track = g.graph.add_node(tcore, tbehavior); + if let Some(l) = g + .graph + .get_mut(list) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + l.tracks.push(track); + } + audio_track_ids.push(track); + } + } + + let mut angle_clip_ids: Vec = Vec::new(); + for (i, entry) in selected.iter().enumerate() { + let footage = graphops::id_of(entry.id) + .ok_or_else(|| format!("angle {i} is not a footage node"))?; + if graphops::footage_behavior(&g.graph, footage).is_none() { + return Err(format!("angle {i} is not a footage node")); + } + let offset_ticks = ((offsets[i].max(0.0)) * tb_den as f64).round() as i64; + let length_ticks = ((entry.duration_s.unwrap_or(10.0) * tb_den as f64).round() as i64) + .max(1); + let in_r = Rational::new(offset_ticks, tb_den); + let out_r = Rational::new(offset_ticks + length_ticks, tb_den); + + let (ccore, cbehavior) = oak_node::block::clip_create(); let clip = g.graph.add_node(ccore, cbehavior); + if let Some(clip_behavior) = g + .graph + .get_mut(clip) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + clip_behavior.core.range = oak_core::TimeRange::new(in_r, out_r); + } + // The clip is owned by its track's `blocks` vec (not an edge). + if let Some(track) = track_ids.get(i) { + let idx = &mut g + .graph + .get_mut(*track) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()); + if let Some(t) = idx { + t.index = i as i32; + } + if let Some(t) = g + .graph + .get_mut(*track) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + t.blocks.push(clip); + } + } + // The footage feeds the clip through the texture-input EDGE. + g.graph + .connect( + footage, + clip, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ) + .map_err(|e| format!("connect footage to clip: {e:?}"))?; + + // AFV: the angle's audio clip on its own source audio track — + // same range as the video angle (the sync offset aligns them), + // fed from the footage's audio stream. + if let (Some(audio_list), Some(audio_track)) = (audio_list, audio_track_ids.get(i)) { + if entry.has_audio.unwrap_or(false) { + let (acore, abehavior) = oak_node::block::clip_create(); + let aclip = g.graph.add_node(acore, abehavior); + if let Some(clip_behavior) = g + .graph + .get_mut(aclip) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + clip_behavior.core.range = oak_core::TimeRange::new(in_r, out_r); + } + if let Some(t) = g + .graph + .get_mut(*audio_track) + .and_then(|e| e.behavior.as_any_mut()) + .and_then(|a| a.downcast_mut::()) + { + t.blocks.push(aclip); + } + g.graph + .connect( + footage, + aclip, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ) + .map_err(|e| format!("connect footage to audio clip: {e:?}"))?; + // The angle's audio + video clips are linked (grouped + // edits stay together). + if let Some(a) = g.graph.get_mut(aclip) { + a.core.links.push(clip); + } + if let Some(v) = g.graph.get_mut(clip) { + v.core.links.push(aclip); + } + } + } + angle_clip_ids.push(clip); + let _ = audio_list; + } + + let (mcore, mbehavior) = oak_node::nodes::multicamnode::create(); + let mc_id = g.graph.add_node(mcore, mbehavior); + g.graph + .connect( + seq_id, + mc_id, + oak_node::nodes::multicamnode::SEQUENCE_INPUT, + -1, + ) + .map_err(|e| format!("connect multicam sequence: {e:?}"))?; + // The multicam's source array input picks the CURRENT source's + // video track clip: one edge per angle, at the element index (the + // C++ shape — `sources_in[element]` is the angle's texture, and + // the traverser's `active_elements_at_time` retains ONLY the + // current source's element, which multi's `value()` forwards). The + // sequence + track-list connections alone cannot feed a texture + // (see the "value() always read None" debug report). + for (i, clip) in angle_clip_ids.iter().enumerate() { + // Grow the array slot so the element index is addressable + // (C++ `MakeArraySlot`; `source_count` reads the array size). + g.graph + .input_array_insert(mc_id, oak_node::nodes::multicamnode::SOURCES_INPUT, i as i32) + .map_err(|e| format!("multicam array slot {i}: {e:?}"))?; + g.graph + .connect( + *clip, + mc_id, + oak_node::nodes::multicamnode::SOURCES_INPUT, + i as i32, + ) + .map_err(|e| format!("connect multicam source {i}: {e:?}"))?; + } + mc = Some(mc_id); + } + let (seq, mc) = (seq.ok_or("sequence")?, mc.ok_or("multicam")?); + Ok((seq, mc)) +} + +/// Normalized cross-correlation over two windowed peak envelopes: for +/// each candidate, slide it against the reference over a ±5 s window in +/// 0.1 s steps and pick the shift with the best score. Pure wrapper +/// over [`oak_audio::waveformsync`]-style correlation, kept testable +/// without audio decoding. +pub fn estimate_wizard_offsets( + reference: &[f64], + candidates: &[Vec], +) -> Vec { + const WINDOW_S: f64 = 0.1; + let mut out = Vec::with_capacity(candidates.len()); + for candidate in candidates { + let (offset_s, _confidence) = best_correlation_offset(reference, candidate, WINDOW_S); + out.push(offset_s); + } + out +} + +/// Raw sliding correlation: the candidate shifted by `offset_s * 10` +/// windows (integer windows) against the reference. Returns +/// `(offset_seconds, score)`. +fn best_correlation_offset(reference: &[f64], candidate: &[f64], window_s: f64) -> (f64, f64) { + if reference.len() < 2 || candidate.len() < 2 { + return (0.0, 0.0); + } + let mut best = (0.0f64, f64::NEG_INFINITY); + let max_shift = (5.0 / window_s).round() as i64; + for shift in -max_shift..=max_shift { + let mut dot = 0.0; + let mut ref_norm = 0.0; + let mut cand_norm = 0.0; + let mut count = 0usize; + let ref_slice = reference; + // `j = i + shift` (the candidate sample aligning to reference i). + for i in 0..ref_slice.len() { + let j = i as i64 + shift; + if j < 0 { + continue; + } + let Some(&c) = candidate.get(j as usize) else { + continue; + }; + let r = ref_slice[i]; + dot += r * c; + ref_norm += r * r; + cand_norm += c * c; + count += 1; + } + if count < 2 { + continue; + } + let denom = (ref_norm * cand_norm).sqrt(); + let score = if denom > 1e-9 { dot / denom } else { 0.0 }; + if score > best.1 { + best = (-(shift as f64) * window_s, score); + } + } + best +} + #[cfg(test)] mod tests { use super::*; @@ -424,4 +810,66 @@ mod tests { let _ = plain; assert_eq!(block_in(&mc_clip), Rational::new(0, 1)); } + + /// `estimate_wizard_offsets` recovers each candidate's lag against the + /// reference: a noise envelope shifted by +2 s must come back as +2 s + /// (the candidate's clip then starts 2 s later on the timeline). + #[test] + fn wizard_offsets_recover_known_lags() { + // Deterministic pseudo-random envelope (positive vs. silence). + let window_s = 0.1; + let n = (20.0 / window_s) as usize; + let mut state = 0x2545_F491_4F6C_DD1Du64; + let reference: Vec = (0..n) + .map(|_| { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + let v = (state >> 33) as f64 / (1u64 << 31) as f64; + if v < 0.5 { + 1.0 + } else { + 0.0 + } + }) + .collect(); + + // candidate[i] = reference[i + k] (content runs k windows ahead, + // e.g. the candidate started k windows earlier in real time). + let shift_of = |k: isize| { + let mut s = state; + let mut shifted: Vec = (0..n) + .map(|_| { + s = s + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + ((((s >> 33) as f64) / ((1u64 << 31) as f64)) < 0.5) as u8 as f64 + }) + .collect(); + for i in 0..n { + let src = i as isize + k; + shifted[i] = if (0..n as isize).contains(&src) { + reference[src as usize] + } else { + shifted[i] + }; + } + shifted + }; + let cand_plus2 = shift_of(20); + let cand_minus3 = shift_of(-30); + + let offsets = estimate_wizard_offsets(&reference, &[cand_plus2, cand_minus3]); + assert_eq!(offsets.len(), 2); + assert!((offsets[0] - 2.0).abs() < 0.35, "got {}", offsets[0]); + assert!((offsets[1] + 3.0).abs() < 0.35, "got {}", offsets[1]); + } + + /// `estimate_wizard_offsets` is robust to empty or tiny inputs: it + /// returns 0 offsets rather than panicking. + #[test] + fn wizard_offsets_tolerate_short_inputs() { + assert_eq!(estimate_wizard_offsets(&[], &[vec![]]), vec![0.0]); + assert_eq!(estimate_wizard_offsets(&[1.0], &[vec![2.0, 3.0]]), vec![0.0]); + } } diff --git a/crates/oak-app/src/oakui/real.rs b/crates/oak-app/src/oakui/real.rs index e33e4108d..4a2b477c2 100644 --- a/crates/oak-app/src/oakui/real.rs +++ b/crates/oak-app/src/oakui/real.rs @@ -87,7 +87,7 @@ use oak_timeline::util::NodeRef; use super::engine::{ AppEngine, EngineGateway, ExportSession, LibraryProject, Monitor, MulticamState, Project, - ScopeData, Sequence, SequenceParameters, VideoFormat, + ScopeData, Sequence, SequenceParameters, VideoFormat, WizardFootage, WizardSyncOffset, }; use super::frames::{bgra_bytes_to_render_image, f32_rgba_to_bgra_image, synthetic_frame_samples}; use super::graphops::{self, ProjectRef}; @@ -859,6 +859,9 @@ pub struct RealClip { color: Hsla, /// The block node in the project graph. block: NodeId, + /// The clip's texture chain contains a multicam node (the timeline + /// overlay + the panel's detection read the same truth). + multicam: bool, } impl ClipData for RealClip { @@ -881,6 +884,10 @@ impl ClipData for RealClip { fn color(&self) -> Option { Some(self.color) } + + fn is_multicam(&self) -> bool { + self.multicam + } } /// A running proxy transcode the engine drains on the tick loop (the @@ -1197,9 +1204,137 @@ pub struct RealEngine { /// (without it a resync retriggers every tick while the fresh chunks /// are still rendering). last_audio_resync: Option, + /// Debounced graph-snapshot upload: the last undo revision the workers + /// were told about, and the time the latest push was requested. Rapid + /// edits (multicam switching) coalesce into ONE serialization + worker + /// load instead of blocking the UI tick on every edit (the "switch + /// several times and everything grinds to a halt" report). + pending_snapshot: Option, + last_snapshot_push: Option, } impl RealEngine { + /// The AFV half of a switch: commands that re-point the host AFV audio + /// clip (the clip linked to the multicam `clip`) to the CURRENT source's + /// audio stream. `None` when the clip has no AFV audio (no linked audio + /// clip, or the multicam has no audio angles). + /// + /// The audio clip's own range stays; only its TEXTURE_INPUT edge moves + /// to the new angle's audio footage. Two commands (disconnect the old, + /// connect the new) join the caller's multi: one undo restores the + /// previous source. + fn afv_audio_switch_commands( + &self, + project_ref: &ProjectRef, + clip: NodeId, + source: i32, + ) -> Option> { + let project = project_ref.clone(); + let (audio_clip, old_footage) = { + let g = graphops::lock(&project); + let mut audio_clip = None; + for &link in &g.graph.links_of(clip) { + let Some(track) = graphops::clip_track(&g.graph, link) else { + continue; + }; + if !graphops::track_behavior(&g.graph, track) + .map(|t| t.kind == TrackType::Audio) + .unwrap_or(false) + { + continue; + } + audio_clip = Some(link); + break; + } + let audio_clip = audio_clip?; + let old_footage = graphops::find_input_footage(&g.graph, audio_clip); + (audio_clip, old_footage) + }; + let old_footage = old_footage?; + // The multicam node for `clip` and its source sequence — then the + // source's AUDIO track at `source`, the angle audio clip, and its + // footage as the re-point target. + let target_footage = { + let g = graphops::lock(&project); + let mc = g + .graph + .connected_output(clip, oak_node::block::clip_input::TEXTURE_INPUT, -1)?; + let seq = g + .graph + .connected_output(mc, oak_node::nodes::multicamnode::SEQUENCE_INPUT, -1)?; + let list = graphops::track_list_of(&g.graph, seq, TrackType::Audio)?; + let list_behavior = graphops::track_list_behavior(&g.graph, list)?; + let audio_track = list_behavior.tracks.get(source as usize)?; + let track_behavior = graphops::track_behavior(&g.graph, *audio_track)?; + let angle_audio_clip = track_behavior.blocks.first()?; + graphops::find_input_footage(&g.graph, *angle_audio_clip)? + }; + if old_footage == target_footage { + return Some(Vec::new()); + } + // Disconnect the old edge + connect the new (undoable pair). + let p_undo = project.clone(); + let p_redo = project.clone(); + let aclip = audio_clip; + let old = old_footage; + let new_footage = target_footage; + let disconnect = oak_undo::undocommand::UndoCommand::from_closures( + move || { + let mut g = graphops::lock(&p_undo); + for (from, input, element) in g.graph.input_connections(aclip) { + if input.as_str() == oak_node::block::clip_input::TEXTURE_INPUT { + let _ = g.graph.disconnect(from, aclip, &input, element); + } + } + }, + move || { + let mut g = graphops::lock(&p_redo); + let _ = g.graph.connect( + old, + aclip, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ); + }, + ); + let p_connect = project.clone(); + let connect = oak_undo::undocommand::UndoCommand::from_closures( + move || { + let mut g = graphops::lock(&p_connect); + for (from, input, element) in g.graph.input_connections(aclip) { + if input.as_str() == oak_node::block::clip_input::TEXTURE_INPUT { + let _ = g.graph.disconnect(from, aclip, &input, element); + } + } + let _ = g.graph.connect( + new_footage, + aclip, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ); + }, + { + let p = project.clone(); + let nf = new_footage; + move || { + let mut g = graphops::lock(&p); + for (from, input, element) in g.graph.input_connections(aclip) { + if input.as_str() == oak_node::block::clip_input::TEXTURE_INPUT { + let _ = g.graph.disconnect(from, aclip, &input, element); + } + } + let _ = g.graph.connect( + old, + aclip, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ); + } + }, + ); + Some(vec![disconnect, connect]) + } + /// Render one tick's worth of audio at the program playhead and queue /// it for playback (M12 P1; M15 S3: worker-pool prefetch; M16 S3: each /// chunk is queued on the dedicated audio-render thread — see @@ -1438,6 +1573,8 @@ impl RealEngine { audio_prefetch: Mutex::new(AudioPrefetch::new()), audio_playback: None, last_audio_resync: None, + pending_snapshot: None, + last_snapshot_push: None, } } @@ -2948,14 +3085,37 @@ impl RealEngine { /// M16 S1 graph mode: pushes the current project state to the worker /// pool (snapshot serialized once per undo-stack revision; the worker /// renders node-graph tickets from it). - fn push_graph_snapshot(&self) { + fn push_graph_snapshot(&mut self) { + // Debounced: record the wanted revision, flush on the next tick + // after the debounce window (or immediately when no edit raced + // recently). Rapid edits coalesce instead of serializing + + // uploading the whole project snapshot on every keystroke. + let revision = oak_undo::global::index().unwrap_or(0).max(0) as u64; + self.pending_snapshot = Some(revision); + } + + /// Flushes the pending snapshot upload when the debounce window passed + /// (called from the engine tick; the window lets a burst of edits + /// coalesce into one serialization + one worker load). + fn flush_snapshot_if_due(&mut self, cx: &mut Context) { + let Some(revision) = self.pending_snapshot.take() else { + return; + }; + let due = match self.last_snapshot_push { + Some(last) => last.elapsed().as_millis() >= 150, + None => true, + }; + if !due { + return; + } let Some(project) = self.project.clone() else { return; }; if let Some(m) = RenderManager::global() { - let revision = oak_undo::global::index().unwrap_or(0).max(0) as u64; let _ = m.set_graph_snapshot(&project, revision); } + self.last_snapshot_push = Some(std::time::Instant::now()); + cx.notify(); } /// Adopts a newly created/loaded project, dropping any previous one, @@ -2996,6 +3156,12 @@ impl RealEngine { guard.output_color_spec(), ); } + // The inline (test) render backend's node-graph source: the live + // project, so preview frames evaluate through the traverser just + // like the worker pool's process snapshot does. + if let Some(m) = oak_render::manager::RenderManager::global() { + m.set_inline_project(project.clone()); + } // Footage loaded from a file may lack stream metadata (C++ projects // have no `` segment; older Rust saves predate the probe @@ -3261,6 +3427,8 @@ impl RealEngine { } else { entry.core.label.clone() }; + let multicam = + super::multicam::clip_is_multicam(graph, block); Some(RealClip { id: ClipId(block.identity()), range: FrameRange::new(Frame(to_ts(in_r)), Frame(to_ts(out_r))), @@ -3268,6 +3436,7 @@ impl RealEngine { label: label.into(), color, block, + multicam, }) }) .collect(); @@ -3719,6 +3888,9 @@ impl EngineGateway for RealEngine { self.drain_multicam_frames(cx); self.schedule_full_res(Monitor::Source, cx); self.schedule_full_res(Monitor::Program, cx); + // Debounced snapshot upload: burst edits (multicam switching) + // coalesce into one worker load instead of per-edit serialization. + self.flush_snapshot_if_due(cx); cx.notify(); } } @@ -4848,6 +5020,19 @@ impl AppEngine for RealEngine { Some(PathBuf::from(&behavior.filename)) } + fn project_entry_name(&self, id: u64) -> Option { + let project = self.project_ref()?; + let node = graphops::id_of(id)?; + let guard = graphops::lock(project); + if !guard.graph.is_valid(node) { + return None; + } + guard + .graph + .get(node) + .map(|e| e.core.label.clone().into()) + } + fn replace_footage( &mut self, id: u64, @@ -4887,6 +5072,43 @@ impl AppEngine for RealEngine { Ok(()) } + fn rename_entry(&mut self, id: u64, new_name: String, cx: &mut Context) { + let Some(project) = self.project.clone() else { + return; + }; + let Some(node) = graphops::id_of(id) else { + return; + }; + let result = { + let mut guard = graphops::lock(&project); + let Some(entry) = guard.graph.get_mut(node) else { + return; + }; + entry.core.label = new_name; + Ok::<(), String>(()) + }; + self.apply_edit(result, "rename entry", cx); + } + + fn delete_entry(&mut self, id: u64, cx: &mut Context) { + let Some(project) = self.project.clone() else { + return; + }; + let Some(node) = graphops::id_of(id) else { + return; + }; + // A folder removal is already one command; a footage/sequence + // entry removes the node itself (the C++ Delete behaviour: the + // node and its connections; timeline clips referencing a removed + // footage keep their blocks but their decode source is gone — + // matching the C++ path, which is undoable). + let result = graphops::push_command( + oak_task::nodeops::remove_node_command(project, node), + "Delete Entry", + ); + self.apply_edit(result, "delete entry", cx); + } + fn drop_footage( &mut self, id: u64, @@ -4904,6 +5126,20 @@ impl AppEngine for RealEngine { println!("[real engine] drop footage: entry {id} is not a footage node"); return; }; + // Sequence entry drop: a NESTED-SEQUENCE clip (one row, not the + // sequence's tracks — a sequence is a frame, not a folder of + // tracks, on the timeline). The dropped clip feeds from the + // sequence node (TEXTURE_INPUT, like `clip_connected_sequence` + // resolves), so a multicam source sequence drops as a single + // multi-cam clip the panel detects. + let is_sequence = { + let guard = graphops::lock(&project); + graphops::sequence_behavior(&guard.graph, footage).is_some() + }; + if is_sequence { + self.drop_sequence_entry(&project, footage, track_index, time, cx); + return; + } let (filename, video_streams, total_streams, seconds) = { let guard = graphops::lock(&project); let Some(f) = graphops::footage_behavior(&guard.graph, footage) else { @@ -4970,8 +5206,41 @@ impl AppEngine for RealEngine { let Some(video_target) = ensure_track(self, TrackKind::Video, cx) else { return; }; - let Some(audio_target) = ensure_track(self, TrackKind::Audio, cx) else { - return; + // The A/V drop pairs the audio with the SAME track number as + // the video (V2 → A2, V1 → A1): the group lands together and + // later drops no longer pile every audio clip onto A1. A + // missing same-numbered audio track (e.g. A2 was removed) is + // recreated so the pair stays aligned; the user is free to + // rearrange afterwards. + let audio_target = loop { + let video_number = self.tracks[video_target].track_index; + if let Some(index) = self.tracks.iter().position(|t| { + t.kind == TrackKind::Audio && t.track_index == video_number + }) { + break index; + } + let audio_count = self.tracks.iter().filter(|t| t.kind == TrackKind::Audio).count(); + if audio_count > video_number { + // Same-numbered audio tracks have run out without a + // hit (should not happen: counting is contiguous). + println!( + "[real engine] drop footage: no audio track A{} for \"{}\"", + video_number + 1, + filename + ); + return; + } + self.add_track(TrackKind::Audio, cx); + let after = self.tracks.iter().filter(|t| t.kind == TrackKind::Audio).count(); + if after == audio_count { + // The track add failed: give up this drop. + println!( + "[real engine] drop footage: could not create audio track A{} for \"{}\"", + video_number + 1, + filename + ); + return; + } }; let (video_index, audio_index) = ( self.tracks[video_target].track_index, @@ -5326,6 +5595,35 @@ impl AppEngine for RealEngine { graphops::sequence_behavior(&guard.graph, node).is_some() } + fn open_sequence_id(&mut self, id: u64, cx: &mut Context) { + let Some(project) = self.project.clone() else { + return; + }; + let Some(node) = graphops::id_of(id) else { + return; + }; + let is_seq = { + let guard = graphops::lock(&project); + graphops::sequence_behavior(&guard.graph, node).is_some() + }; + if !is_seq { + return; + } + // Switch the current sequence: the timeline rebuilds against the + // opened sequence (the "open a sequence entry" behavior — same + // refresh as project open). + if self.sequence == Some(node) { + return; + } + self.sequence = Some(node); + graphops::ensure_sequences_mounted(&project); + self.refresh_sequence_info(); + self.rebuild_timeline(); + self.invalidate_rendered_frames(); + self.push_graph_snapshot(); + cx.notify(); + } + fn sequence_parameters(&self, id: u64) -> Option { let project = self.project_ref()?; let node = graphops::id_of(id)?; @@ -6016,6 +6314,16 @@ impl AppEngine for RealEngine { if source < 0 || source >= state.source_count { return; } + // No-op guard: switching to the ALREADY current source must not + // push an undo command. The playback-time "revision advances every + // frame" report traced back to the multicam panel queue replaying + // the same switch on every tick — each replay pushed a useless + // undo entry, re-uploaded the whole 130 KB graph snapshot to every + // worker and re-serialized it per frame (the "picture barely + // updates" freeze). + if state.current_source == source { + return; + } let Some(clip) = graphops::id_of(state.clip_id) else { return; }; @@ -6026,15 +6334,494 @@ impl AppEngine for RealEngine { split_clip, playhead, ); - let result = graphops::push_command(cmd, oak_timeline::multicam::SWITCH_LABEL); + let afv_cmds = self.afv_audio_switch_commands(&project, clip, source); + let result = match afv_cmds { + // ONE undo entry: video switch + AFV audio re-point. + Some(children) if !children.is_empty() => { + let mut all = vec![cmd]; + all.extend(children); + graphops::push_multi_command(all, oak_timeline::multicam::SWITCH_LABEL) + } + _ => graphops::push_command(cmd, oak_timeline::multicam::SWITCH_LABEL), + }; self.apply_edit(result, "multicam switch", cx); } + // --- multicam wizard ------------------------------------------------ + + fn multicam_wizard_footage(&self) -> Option> { + let project = self.project_ref()?; + Some(super::multicam::wizard_footage(&project)) + } + + fn multicam_wizard_sync_offsets( + &self, + selected: &[WizardFootage], + ) -> Result, String> { + let Some(reference) = selected.first() else { + return Ok(Vec::new()); + }; + // One angle is enough: nothing to align. + if selected.len() == 1 { + return Ok(vec![WizardSyncOffset { + footage: reference.id, + offset_s: 0.0, + confidence: Some(1.0), + }]); + } + // Extract each angle's RMS envelope (100 ms windows) through the + // audio waveform extractor over the footage's media file. The + // reference angle is angle 0. + let source_paths: Vec<(u64, String, i32)> = { + let project = self.project_ref().ok_or("no project open")?; + let g = graphops::lock(&project); + let mut out = Vec::new(); + for entry in selected { + let node = graphops::id_of(entry.id).ok_or("angle missing")?; + let f = graphops::footage_behavior(&g.graph, node).ok_or("angle is not footage")?; + out.push(( + entry.id, + f.filename.clone(), + f.streams + .iter() + .find(|s| !s.is_video) + .map(|s| s.index) + .unwrap_or(1), + )); + } + out + }; + let mut envelopes: Vec> = Vec::new(); + for (_, filename, stream) in &source_paths { + let envelope = crate::oakui::waveform::extract_audio_envelope(filename, *stream) + .ok_or_else(|| format!("failed to extract audio from {filename}"))?; + envelopes.push(envelope); + } + let reference_envelope = envelopes[0].clone(); + let offsets = super::multicam::estimate_wizard_offsets( + &reference_envelope, + &envelopes[1..], + ); + let mut out = vec![WizardSyncOffset { + footage: reference.id, + offset_s: 0.0, + confidence: Some(1.0), + }]; + for (entry, offset_s) in selected[1..].iter().zip(offsets.iter()) { + out.push(WizardSyncOffset { + footage: entry.id, + offset_s: *offset_s, + confidence: Some(0.0), // the wizard's confidence is UI-only for now + }); + } + Ok(out) + } + + fn multicam_create_sequence( + &mut self, + selected: Vec, + offsets: Vec, + name: String, + cx: &mut Context, + ) -> Result { + let Some(project) = self.project.clone() else { + return Err(crate::i18n::tr("seqprops.error.no_project").to_string()); + }; + let name = if name.trim().is_empty() { + "Multi-Cam".to_string() + } else { + name.trim().to_string() + }; + // Build the source sequence (one clip per angle on its own track, + // offset by the sync result) plus the multicam node over it. + let (source_seq, mc_node) = super::multicam::build_multicam_sequence( + &project, + &selected, + &offsets, + &name, + )?; + // Mount the source sequence under the root folder (non-undoable, + // like `create_sequence` does). + let root = graphops::lock(&project).root; + oak_task::nodeops::folder_add_child_command( + (project.clone(), root), + (project.clone(), source_seq), + ) + .redo_now(); + + // The host clip: the clip under the program playhead on the + // current sequence. A clip whose texture input is rewired to the + // multicam output (the switch source feeds it). When the host + // sequence has no video clip at or anywhere on the video tracks + // (a fresh sequence), one is PLACED at the playhead from the + // first selected footage first — the wizard needs no pre-existing + // clip on an otherwise empty timeline. + let playhead = self + .sequence + .map(|seq| graphops::sequence_playhead(&graphops::lock(&project).graph, seq)); + let host = match self.find_existing_multicam_host_clip(&project, playhead) { + Ok(host_id) => Ok(host_id), + Err(no_clip) => { + // No candidate: place the first angle's footage at the + // playhead (one video clip on the pointed/first video + // track), then re-resolve. + let angle = selected + .first() + .cloned() + .ok_or_else(|| "no selected angles".to_string())?; + self.place_host_clip_for_wizard(&project, playhead, angle) + .map_err(|e| format!("{no_clip} — placing skipped: {e}")) + } + }; + let host_id = match &host { + Ok(id) => *id, + Err(e) => { + self.apply_edit(Err(e.clone()), "create multicam sequence", cx); + return Err(e.clone()); + } + }; + // Rewire: host clip texture input -> multicam output. + let result = { + let mut g = graphops::lock(&project); + // Disconnect the raw footage edge first (place_footage_clip + // wired footage->clip). + for (from, input, element) in g.graph.input_connections(host_id) { + if input.as_str() == oak_node::block::clip_input::TEXTURE_INPUT { + let _ = g.graph.disconnect(from, host_id, &input, element); + } + } + g.graph + .connect( + mc_node, + host_id, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ) + .map_err(|e| format!("connect multicam to host clip: {e:?}")) + }; + match result { + Ok(()) => { + // The wizard's payload stays on the SOURCE sequence (the + // angles' clips + the multicam node); the user's editing + // sequence is UNCHANGED — its host clip (under the + // playhead) now feeds from the multicam output and IS the + // "multi-cam clip" the panel detects. Switching the current + // sequence to the source one used to open the angles' rows + // (N tracks for N cameras) and left the panel with no clip + // resolution ("未检测到多机位片段"). + // AFV: when the angles carry audio, the host clip's LINKED + // audio clip is placed on the host sequence (same range, + // linked both ways); its source starts at angle 0's audio + // and follows every switch. Muting the host audio track + // disables the follow-through audio. + if let Err(e) = + self.ensure_host_afv_audio(&project, host_id, &selected, cx) + { + println!("[real engine] AFV audio placement skipped: {e}"); + } + self.apply_edit(Ok(()), "create multicam sequence", cx); + Ok(self.sequence.unwrap_or(source_seq).identity()) + } + Err(e) => { + self.apply_edit(Err(e.clone()), "create multicam sequence", cx); + Err(e) + } + } + } + fn backend_name(&self) -> &'static str { "real" } } +impl RealEngine { + /// AFV: places the audio clip paired with the multicam `host_clip` on + /// the host sequence — same range, linked both ways (grouped edits + /// move them together). Its media source is ANGLE 0's audio stream: + /// `multicam_switch_to` re-points it to the switching target's audio + /// clip. The host audio track keeps the regular mute/solo controls, so + /// muting that track disables the follow-through audio. + fn ensure_host_afv_audio( + &mut self, + project: &ProjectRef, + host_clip: NodeId, + selected: &[WizardFootage], + cx: &mut Context, + ) -> Result { + let Some(host_seq) = self.sequence else { + return Err("no sequence open".to_string()); + }; + let Some(reference) = selected.first() else { + return Err("no selected angles".to_string()); + }; + if !reference.has_audio.unwrap_or(false) { + return Err("the reference angle has no audio".to_string()); + } + let angle_footage = graphops::id_of(reference.id) + .ok_or_else(|| "angle 0 footage is not in the project".to_string())?; + // The host audio track: the first audio track, created when the + // host sequence has none. + let audio_index = match self.tracks.iter().position(|t| t.kind == TrackKind::Audio) { + Some(i) => self.tracks[i].track_index, + None => { + self.add_track(TrackKind::Audio, cx); + self.tracks + .iter() + .find(|t| t.kind == TrackKind::Audio) + .map(|t| t.track_index) + .ok_or_else(|| "could not add an audio track".to_string())? + } + }; + // The clip range on the host timeline (in host sequence frames). + let (host_in_ts, host_out_ts) = { + let g = graphops::lock(project); + let in_r = graphops::clip_range(&g.graph, host_clip) + .ok_or_else(|| "host clip has no range".to_string())? + .0; + let out_r = graphops::clip_range(&g.graph, host_clip) + .ok_or_else(|| "host clip has no range".to_string())? + .1; + let tb = graphops::sequence_time_base(&g.graph, host_seq) + .unwrap_or((1, 25)); + (graphops::rational_to_ts(in_r, tb), graphops::rational_to_ts(out_r, tb)) + }; + let audio_clip = graphops::place_footage_clip( + project, + host_seq, + angle_footage, + TrackType::Audio, + audio_index, + host_in_ts, + host_out_ts, + 0, + )?; + // Link the A/V pair both ways (grouped edits move them together). + graphops::set_clips_linked(project, &[host_clip, audio_clip], true)?; + Ok(audio_clip) + } + + /// Drops a sequence-project entry on the timeline as a NESTED-SEQUENCE + /// clip: ONE video clip on the video track (not the sequence's tracks — + /// a frame, not a folder, of tracks), fed from the sequence node. This + /// is also how a multicam source sequence lands in the timeline: the + /// clip is the single multi-cam row the panel detects. Undoable as one + /// "Add Clip". + fn drop_sequence_entry( + &mut self, + project: &ProjectRef, + source_seq: NodeId, + track_index: usize, + time: Frame, + cx: &mut Context, + ) { + let Some(host_seq) = self.sequence else { + println!("[real engine] drop sequence: no sequence open"); + return; + }; + // The target video track: the pointed display row when it is a + // video track, else the first video track (the footage policy). + let video_target = if self + .tracks + .get(track_index) + .is_some_and(|t| t.kind == TrackKind::Video) + { + track_index + } else if let Some(index) = self.tracks.iter().position(|t| t.kind == TrackKind::Video) + { + index + } else { + println!("[real engine] drop sequence: no video track"); + return; + }; + let video_index = self.tracks[video_target].track_index; + // The clip length: the source sequence's length converted into the + // host sequence's frame timestamps (the drop ghost's extent). + let (in_ts, length_ts) = { + let guard = graphops::lock(project); + let Ok(tb) = graphops::sequence_time_base(&guard.graph, host_seq) + .ok_or_else(|| "host sequence has no frame rate".to_string()) + else { + println!("[real engine] drop sequence: host has no frame rate"); + return; + }; + let fps_f = tb.1.max(1) as f64 / tb.0.max(1) as f64; + let source_seconds = graphops::sequence_length(&guard.graph, source_seq).to_f64(); + let length_ts = ((source_seconds * fps_f).round() as i64).max(1); + (time.0.max(0), length_ts) + }; + let placed = graphops::place_nested_sequence_clip( + project, + host_seq, + source_seq, + video_index, + in_ts, + in_ts + length_ts, + ); + let placed = match placed { + Ok(clip) => clip, + Err(e) => { + self.apply_edit(Err(e.clone()), "drop sequence", cx); + return; + } + }; + // A source sequence with a multicam node drops as the multi-cam + // clip itself (its texture feeds from the multicam output, so the + // panel detects it — "未检测到多机位片段" when it nur fed the + // raw source sequence). + // Collect the candidate multicam ids under the lock FIRST, then + // match each with `multicam_sequence` OUTSIDE the guard (that + // helper locks the project itself — a live guard re-enters the + // non-recursive mutex, exactly the wizard-freeze shape). + let candidates = { + let guard = graphops::lock(project); + graphops::multicam_nodes(&guard.graph) + }; + let mc_for_source = candidates + .iter() + .copied() + .find(|&mc| super::multicam::multicam_sequence(project, mc) == Some(source_seq)); + if let Some(mc) = mc_for_source { + let result = { + let mut g = graphops::lock(project); + for (from, input, element) in g.graph.input_connections(placed) { + if input.as_str() == oak_node::block::clip_input::TEXTURE_INPUT { + let _ = g.graph.disconnect(from, placed, &input, element); + } + } + g.graph + .connect( + mc, + placed, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ) + .map_err(|e| format!("connect multicam to dropped clip: {e:?}")) + }; + self.apply_edit(result, "drop sequence (multicam)", cx); + } else { + self.apply_edit(Ok(()), "drop sequence", cx); + } + } + + /// The clip that will host the multicam output: the clip under the + /// current sequence's playhead (on any video track), or the first + /// clip on any video track when the playhead sits in a gap. Does NOT + /// create a clip ([`Self::place_host_clip_for_wizard`] does). + fn find_existing_multicam_host_clip( + &self, + project: &ProjectRef, + playhead: Option, + ) -> Result { + let Some(seq) = self.sequence else { + return Err("no sequence open".to_string()); + }; + // 1. The clip covering the playhead on the topmost video track. + let covering = { + let g = graphops::lock(project); + // Walk the video tracks topmost-first for a clip covering the + // playhead. + let mut found = None; + let tracks = graphops::track_ids(&g.graph, seq, TrackType::Video); + for &track_id in tracks.iter().rev() { + let Some(track) = graphops::track_behavior(&g.graph, track_id) else { + continue; + }; + for &block in &track.blocks { + let Some(clip) = graphops::clip_behavior(&g.graph, block) else { + continue; + }; + if playhead.is_none_or(|t| t >= clip.core.in_() && t < clip.core.out()) { + found = Some(block); + break; + } + } + if found.is_some() { + break; + } + } + found + }; + if let Some(block) = covering { + return Ok(block); + } + // 2. No clip at the playhead: fall back to the FIRST clip on any + // video track, so the wizard still works when the playhead + // sits in a gap. + let any_clip = { + let g = graphops::lock(project); + graphops::track_ids(&g.graph, seq, TrackType::Video) + .into_iter() + .find_map(|track_id| { + graphops::track_behavior(&g.graph, track_id) + .and_then(|t| t.blocks.first().copied()) + }) + }; + any_clip.ok_or_else(|| "no clip under the playhead to host the multicam".to_string()) + } + + /// Places a fresh host clip for the wizard on an empty (or clip-less) + /// host sequence: `angle`'s footage at the program playhead on the top + /// video track, standard-length — then the wizard rewire slot runs. + /// Returns the placed clip's node. + fn place_host_clip_for_wizard( + &mut self, + project: &ProjectRef, + playhead: Option, + angle: WizardFootage, + ) -> Result { + let Some(seq) = self.sequence else { + return Err("no sequence open".to_string()); + }; + let Some(angle) = graphops::id_of(angle.id) else { + return Err("the angle footage is not in the project".to_string()); + }; + let video_target = if let Some(index) = + self.tracks.iter().position(|t| t.kind == TrackKind::Video) + { + index + } else { + return Err("no video track on the host sequence".to_string()); + }; + let video_index = self.tracks[video_target].track_index; + let (length, in_ts) = { + let tb = self.frame_rate(); + let fps_f = tb.num.max(1) as f64 / tb.den.max(1) as f64; + let seconds = { + let guard = graphops::lock(project); + graphops::footage_duration_seconds(&guard.graph, angle) + }; + let length = match seconds { + Some(s) => (s * fps_f).round().max(1.0) as i64, + None => (10.0 * fps_f).round().max(1.0) as i64, + }; + let in_ts = match playhead { + Some(t) => { + let (num, den) = (t.numerator(), t.denominator().max(1)); + let fps_f = fps_f as i64; + ((num * fps_f) / den).max(0) + } + None => 0, + }; + (length, in_ts) + }; + let clip = graphops::place_footage_clip( + project, + seq, + angle, + TrackType::Video, + video_index, + in_ts, + in_ts + length, + 0, + ) + .map_err(|e| format!("place host clip: {e}"))?; + self.refresh_sequence_info(); + self.rebuild_timeline(); + self.invalidate_rendered_frames(); + Ok(clip) + } +} + // --------------------------------------------------------------------------- // Format dispatch (pure, unit tested) // --------------------------------------------------------------------------- @@ -7049,6 +7836,39 @@ mod tests { let _ = std::fs::remove_file(&media); } + /// The multicam wizard's data path on the REAL engine: the footage + /// enumeration must return the imported entry (an A/V file appearing in + /// the wizard as one angle), no deadlock. This mirrors what the wizard + /// modal runs on open ("click 窗口→多机位向导 freezes" regression). + #[gpui::test] + async fn real_engine_wizard_footage_lists_imported_media(cx: &mut gpui::TestAppContext) { + let _media = media_lock(); + let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx))); + cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx))); + + // Use the repo's committed fixture media (the same file the other + // engine tests import): write_test_clip is omitted because this + // test targets the WIZARD path, not media generation. + let media = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/demo.mp4"); + let imported = cx.update(|app| { + engine.update(app, |engine, cx| engine.import_footage(media.clone(), cx)) + }); + assert!(imported.is_ok(), "import succeeds: {imported:?}"); + + // The same read the wizard modal performs on open. + let footage = cx.read(|app| engine.read(app).multicam_wizard_footage()); + let footage = footage.expect("wizard footage exists"); + let name = media.file_name().unwrap().to_string_lossy().into_owned(); + assert!( + footage.iter().any(|f| f.name.as_ref() == name), + "the imported file is offered as a wizard angle" + ); + assert!( + footage.iter().all(|f| f.has_audio.is_some()), + "audio presence is probed" + ); + } + /// The material-bin thumbnail pipeline end to end: importing a real media /// file (tests/demo.mp4) lists a footage entry whose icon-view thumbnail /// is rendered on a background worker, drained on the tick, and cached as @@ -8033,6 +8853,135 @@ mod tests { ); } + /// The A/V drop pairs the audio with the SAME track number as the + /// video: dropping onto the V2 row places the video on V2 and the + /// linked audio on A2, not on A1 (the multi-drop piles onto A1 + /// regression). + #[gpui::test] + async fn drop_av_footage_same_numbered_tracks(cx: &mut gpui::TestAppContext) { + let _media = media_lock(); + let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx))); + cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx))); + + let media = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/demo.mp4"); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.import_footage(media.clone(), cx).expect("import") + }) + }); + let name = media.file_name().unwrap().to_string_lossy().into_owned(); + let entry = cx + .read(|app| { + engine + .read(app) + .roots() + .into_iter() + .find(|e| e.name.as_ref() == name) + }) + .expect("imported footage is listed"); + + // The default layout is [V2, V1, A1, A2]: display row 0 is V2. + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.drop_footage(entry.id, TrackKind::Video, 0, Frame(40), cx) + }) + }); + + let (video_row, audio_row) = cx.read(|app| { + let engine = engine.read(app); + let v = engine + .tracks + .iter() + .position(|t| t.kind == TrackKind::Video && !t.clips.is_empty()); + let a = engine + .tracks + .iter() + .position(|t| t.kind == TrackKind::Audio && !t.clips.is_empty()); + (v.expect("video row"), a.expect("audio row")) + }); + let (video_index, audio_index) = cx.read(|app| { + let engine = engine.read(app); + (engine.tracks[video_row].track_index, engine.tracks[audio_row].track_index) + }); + assert_eq!( + video_index, 1, + "the video clip lands on V2 (per-type index 1)" + ); + assert_eq!( + audio_index, 1, + "the audio clip lands on A2 (same track number), not A1" + ); + } + + /// When the same-numbered audio track is missing (A2 removed), the A/V + /// drop recreates it so the pair stays aligned; the first audio track + /// does not absorb the clip. + #[gpui::test] + async fn drop_av_footage_recreates_missing_same_numbered_audio_track( + cx: &mut gpui::TestAppContext, + ) { + let _media = media_lock(); + let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx))); + cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx))); + + let media = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/demo.mp4"); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.import_footage(media.clone(), cx).expect("import") + }) + }); + let name = media.file_name().unwrap().to_string_lossy().into_owned(); + let entry = cx + .read(|app| { + engine + .read(app) + .roots() + .into_iter() + .find(|e| e.name.as_ref() == name) + }) + .expect("imported footage is listed"); + + // Remove A2 (display row 3 in the default [V2, V1, A1, A2] layout). + cx.update(|app| { + engine.update(app, |engine, cx| engine.remove_track(3, cx)) + }); + // Drop onto V2 (row 0): the audio has no A2 to go to. + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.drop_footage(entry.id, TrackKind::Video, 0, Frame(40), cx) + }) + }); + + let (audio_rows, audio_clip_row) = cx.read(|app| { + let engine = engine.read(app); + let rows: Vec = engine + .tracks + .iter() + .enumerate() + .filter(|(_, t)| t.kind == TrackKind::Audio) + .map(|(i, _)| i) + .collect(); + let clip = engine + .tracks + .iter() + .position(|t| t.kind == TrackKind::Audio && !t.clips.is_empty()); + (rows, clip) + }); + assert_eq!( + audio_rows.len(), + 2, + "A2 was recreated (default A1 + new A2)" + ); + let audio_index = cx.read(|app| { + let engine = engine.read(app); + engine.tracks[audio_clip_row.expect("audio clip row")].track_index + }); + assert_eq!( + audio_index, 1, + "the audio clip lands on the recreated A2" + ); + } + /// Dragging a clip of a linked A/V pair drags its partner in lockstep: /// the linked clip stays on its own track and shifts by the same frame /// offset, both for a same-track drag and a cross-track drag, and ONE @@ -9050,6 +9999,33 @@ mod tests { (24000, 1001) ); assert!(params.interlaced, "the interlaced flag round-trips"); + + // Double-click opens the sequence: the current sequence switches + // to the double-clicked one (the "explorer double-click on a + // sequence entry has no effect" regression). The 4K sequence's + // format becomes the timeline's baseline. + cx.update(|app| { + engine.update(app, |engine, cx| engine.open_sequence_id(seq, cx)); + }); + let current = cx.read(|app| engine.read(app).current_sequence().cloned()); + assert_eq!( + current.map(|s| s.format.width), + Some(3840), + "the double-clicked sequence opens in the timeline" + ); + // A folder or stale id is ignored. + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.open_sequence_id(folder, cx); + engine.open_sequence_id(0xdead_beef, cx); + }) + }); + let still = cx.read(|app| engine.read(app).current_sequence().cloned()); + assert_eq!( + still.map(|s| s.format.width), + Some(3840), + "non-sequence ids do not switch the current sequence" + ); } /// `update_sequence_parameters` rewrites the name, the format and the @@ -9243,4 +10219,446 @@ mod tests { let _ = std::fs::remove_file(&media); } + + /// The wizard's create keeps the USER's sequence open (its playhead + /// clip becomes the multi-cam clip; switching to the source sequence + /// left N camera rows on the timeline and an "未检测到多机位片段" panel). + #[gpui::test] + async fn wizard_create_keeps_host_sequence_and_panel_detects( + cx: &mut gpui::TestAppContext, + ) { + let _media = media_lock(); + let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx))); + cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx))); + + let media = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/demo.mp4"); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.import_footage(media.clone(), cx).expect("import") + }) + }); + let footage = cx + .read(|app| engine.read(app).multicam_wizard_footage()) + .expect("wizard footage"); + let name = media.file_name().unwrap().to_string_lossy(); + let entry = footage.iter().find(|f| f.name.as_ref() == name).cloned(); + let entry = entry.expect("the imported file is a wizard angle"); + let host_before = cx.read(|app| { + engine.read(app).current_sequence().map(|s| s.name.clone()) + }); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine + .multicam_create_sequence(vec![entry], vec![0.0], "MultiCam".to_string(), cx) + .expect("create multicam") + }) + }); + let host_after = cx.read(|app| { + engine.read(app).current_sequence().map(|s| s.name.clone()) + }); + assert_eq!( + host_after, host_before, + "the user's sequence stays current (no camera rows on the timeline)" + ); + assert_eq!( + host_after, Some("Sequence 1".to_string()), + "the fresh project's default sequence stays open" + ); + // The panel resolves the playhead clip to the multicam state now. + let state = cx.read(|app| engine.read(app).multicam_state()); + assert!( + state.is_some(), + "the panel detects the multi-cam clip at the playhead" + ); + assert_eq!(state.map(|s| s.source_count), Some(1)); + + // The timeline snapshot flags the host clip as a multi-cam clip (the + // special overlay drives off `ClipData::is_multicam`). + let multicam_flags = cx.read(|app| { + let engine = engine.read(app); + engine + .tracks + .iter() + .filter(|t| t.kind == TrackKind::Video) + .flat_map(|t| t.clips.iter().map(|c| c.is_multicam())) + .collect::>() + }); + assert_eq!( + multicam_flags, + vec![true], + "the single video clip is the multi-cam clip" + ); + + // AFV: the host clip's linked audio clip was placed on the host + // sequence (same range, linked both ways) so the audio follows the + // switched camera. + let (audio_clips, audio_linked) = cx.read(|app| { + let engine = engine.read(app); + let audio: Vec<_> = engine + .tracks + .iter() + .filter(|t| t.kind == TrackKind::Audio) + .flat_map(|t| t.clips.iter().map(|c| c.block)) + .collect(); + let project = engine.project.as_ref().cloned(); + let linked = match (project, audio.first()) { + (Some(p), Some(audio_clip)) => { + let g = graphops::lock(&p); + let host = engine + .tracks + .iter() + .filter(|t| t.kind == TrackKind::Video) + .flat_map(|t| t.clips.iter().map(|c| c.block)) + .next() + .map(|b| b); + host.map(|h| g.graph.links_of(h).contains(audio_clip)) + .unwrap_or(false) + } + _ => false, + }; + (audio.len(), linked) + }); + assert_eq!( + audio_clips, 1, + "the AFV audio clip was placed on the host sequence" + ); + assert!( + audio_linked, + "the AFV audio clip links to the multi-cam host clip" + ); + + // Muting the host audio track disables the AFV audio (the track + // edit path the user controls). The graph flag reads back directly + // (the in-memory snapshot refreshes on the next engine event). + cx.update(|app| { + engine.update(app, |engine, _cx| { + let Some(project) = engine.project.clone() else { + return; + }; + if let Some(audio_row) = + engine.tracks.iter().position(|t| t.kind == TrackKind::Audio) + { + let track = engine.tracks[audio_row].track; + let _ = graphops::set_track_muted(&project, track, true); + } + }) + }); + let muted = cx.read(|app| { + let engine = engine.read(app); + let project = engine.project.clone().unwrap(); + let g = graphops::lock(&project); + engine + .tracks + .iter() + .find(|t| t.kind == TrackKind::Audio) + .map(|t| graphops::track_behavior(&g.graph, t.track).map(|b| b.muted).unwrap_or(false)) + .unwrap_or(false) + }); + assert!(muted, "the AFV audio track is muted — no audio follows"); + } + + /// Dragging a multicam SOURCE sequence entry from the project explorer + /// onto the timeline places ONE nested-sequence clip on the video track + /// (no camera rows) and rewires it to the multicam node, so the panel + /// resolves that clip to the multi-cam state — the "not detected" + /// report's drag shape. + #[gpui::test] + async fn drop_multicam_sequence_places_one_clip_and_panel_detects( + cx: &mut gpui::TestAppContext, + ) { + let _media = media_lock(); + let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx))); + cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx))); + + // The wizard create (host clip placed on the default sequence). + let media = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/demo.mp4"); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.import_footage(media.clone(), cx).expect("import") + }) + }); + let footage = cx + .read(|app| engine.read(app).multicam_wizard_footage()) + .expect("wizard footage"); + let name = media.file_name().unwrap().to_string_lossy(); + let entry = footage.iter().find(|f| f.name.as_ref() == name).cloned(); + let entry = entry.expect("the imported file is a wizard angle"); + let source_seq = cx.update(|app| { + engine.update(app, |engine, cx| { + engine + .multicam_create_sequence( + vec![entry.clone()], + vec![0.0], + "MultiCam".to_string(), + cx, + ) + .expect("create multicam") + }) + }); + let clip_count = |engine: &RealEngine| -> usize { + engine + .tracks + .iter() + .filter(|t| t.kind == TrackKind::Video) + .map(|t| t.clips.len()) + .sum() + }; + // After the wizard create the default (host) sequence has ONE + // video clip. + let before = cx.read(|app| { + let engine = engine.read(app); + clip_count(engine) + }); + assert_eq!(before, 1, "host clip on the default sequence"); + + // Simulate the project-explorer drag: drop the SOURCE sequence's + // entry at frame 80. The wizard source sequence lists as + // "MultiCam" under the project root (id is its own identity, NOT + // the return value — the return value is the host sequence kept + // current). + let project = cx.read(|app| engine.read(app).project.clone().expect("project")); + let source_entry_id = cx.read(|app| { + let engine = engine.read(app); + engine + .roots() + .into_iter() + .find(|e| e.name.as_ref() == "MultiCam") + .map(|e| e.id) + .expect("the multicam source sequence lists in the project") + }); + assert_ne!(source_entry_id, source_seq, "the return value is the host sequence id"); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.drop_footage(source_entry_id, TrackKind::Video, 0, Frame(80), cx) + }) + }); + let after = cx.read(|app| { + let engine = engine.read(app); + clip_count(engine) + }); + assert_eq!(after, 2, "the drop added ONE nested-sequence clip"); + + // The dropped clip feeds from the multicam node (NOT the raw + // source sequence), so the panel detects it. + let (dropped_clip, state) = cx.read(|app| { + let engine = engine.read(app); + let track = engine + .tracks + .iter() + .find(|t| t.kind == TrackKind::Video) + .expect("video track"); + let clip = track + .clips + .iter() + .find(|c| c.range.start.0 >= 80) + .expect("the dropped clip at frame 80") + .block; + let state = engine.multicam_state(); + (clip.identity(), state) + }); + let guard = graphops::lock(&project); + let dropped_node = graphops::id_of(dropped_clip).expect("clip node"); + let src = guard + .graph + .connected_output(dropped_node, oak_node::block::clip_input::TEXTURE_INPUT, -1); + assert!( + src.is_some(), + "the dropped clip has a texture source" + ); + let is_multicam = src.is_some_and(|s| { + graphops::node_type_id(&guard.graph, s) == "org.olivevideoeditor.Olive.multicam" + }); + assert!( + is_multicam, + "the drop rewire feeds from the multicam node" + ); + drop(guard); + assert!( + state.is_some(), + "the dropped multi-cam clip is detected by the panel" + ); + + // The sequence-viewer montage must resolve playable media for the + // dropped clip (the "dragging a multicam sequence into another + // sequence shows black" report): the montage carries the CURRENT + // angle's footage. NOTE: the project guard is NOT held across + // `video_montage` (it locks the project itself — the same + // non-recursive-mutex reentry that froze the wizard). + let seq = cx.read(|app| engine.read(app).sequence).expect("current sequence node"); + let montage_files = { + let time = { + let g = graphops::lock(&project); + graphops::sequence_playhead(&g.graph, seq) + }; + let montage = crate::oakui::renderops::video_montage(&project, seq, time); + montage + .iter() + .map(|m| m.filename.clone()) + .collect::>() + }; + assert!( + !montage_files.is_empty(), + "the dropped multi-cam clip's montage resolves media (not black)" + ); + assert_eq!( + montage_files.len(), 1, + "the host clip's current angle is the montage's only entry" + ); + } + + /// AFV switch-follow: with two audio-bearing angles, switching to + /// source 1 re-points the host audio clip to angle 1's audio footage — + /// one undo restores the previous source's audio. + #[gpui::test] + async fn multicam_switch_follows_audio_to_the_new_source(cx: &mut gpui::TestAppContext) { + let _media = media_lock(); + let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx))); + cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx))); + + let media_a = + std::env::temp_dir().join(format!("oakapp_afv_a_{}.mp4", std::process::id())); + let media_b = + std::env::temp_dir().join(format!("oakapp_afv_b_{}.mp4", std::process::id())); + oak_codec::testmedia::write_test_clip(&media_a, 64, 64, 10, 10).expect("media A"); + oak_codec::testmedia::write_test_clip(&media_b, 64, 64, 10, 10).expect("media B"); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.import_footage(media_a.clone(), cx).expect("import A"); + engine.import_footage(media_b.clone(), cx).expect("import B"); + }) + }); + let footage = cx + .read(|app| engine.read(app).multicam_wizard_footage()) + .expect("wizard footage"); + let names = [&media_a, &media_b]; + let angles: Vec<_> = names + .iter() + .map(|m| { + let n = m.file_name().unwrap().to_string_lossy(); + footage + .iter() + .find(|f| f.name.as_ref() == n) + .cloned() + .expect("angle listed") + }) + .collect(); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine + .multicam_create_sequence( + angles.clone(), + vec![0.0, 0.0], + "AFV".to_string(), + cx, + ) + .expect("create multicam") + }) + }); + + let audio_source = |engine: &RealEngine| -> Option { + let project = engine.project.clone()?; + let g = graphops::lock(&project); + let audio = engine + .tracks + .iter() + .find(|t| t.kind == TrackKind::Audio)? + .clips + .first()? + .block; + let footage = graphops::find_input_footage(&g.graph, audio)?; + let f = graphops::footage_behavior(&g.graph, footage)?; + Some(f.filename.clone()) + }; + let before = cx.read(|app| audio_source(engine.read(app))).expect("AFV audio before"); + + // Switch to source 1 (the second angle). + cx.update(|app| { + engine.update(app, |engine, cx| engine.multicam_switch_to(1, false, cx)); + }); + let after = cx.read(|app| audio_source(engine.read(app))).expect("AFV audio after switch"); + let b_name = media_b.to_string_lossy().into_owned(); + assert_eq!(after, b_name, "the AFV audio follows to angle 1's media"); + assert_ne!(before, after, "the audio source actually changed"); + + // ONE undo returns to angle 0's audio. + cx.update(|app| { + engine.update(app, |engine, cx| engine.undo(cx)); + }); + let undone = + cx.read(|app| audio_source(engine.read(app))).expect("AFV audio after undo"); + assert_eq!(undone, before, "undo restores the previous source's audio"); + + let _ = std::fs::remove_file(&media_a); + let _ = std::fs::remove_file(&media_b); + } + + /// The project explorer's rename + delete work on real entries: the + /// imported footage renames through the engine (the label lands on the + /// node) and deletes with ONE undoable entry (the footage node is gone, + /// one undo restores it). + #[gpui::test] + async fn entry_rename_and_delete_are_real(cx: &mut gpui::TestAppContext) { + let _media = media_lock(); + let engine = cx.update(|cx| cx.new(|cx| RealEngine::create(cx))); + cx.update(|app| engine.update(app, |engine, cx| engine.new_project(cx))); + + let media = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/demo.mp4"); + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.import_footage(media.clone(), cx).expect("import") + }) + }); + let entry = cx + .read(|app| { + engine + .read(app) + .roots() + .into_iter() + .find(|e| e.name.as_ref() == "demo.mp4") + .expect("imported") + }); + + // Rename: the node label changes. + cx.update(|app| { + engine.update(app, |engine, cx| { + engine.rename_entry(entry.id, "renamed-clip".to_string(), cx) + }) + }); + let name = cx.read(|app| engine.read(app).project_entry_name(entry.id)); + assert_eq!( + name.as_deref(), + Some(SharedString::from("renamed-clip").as_ref()), + "the entry's display name changes" + ); + + // Delete: the entry leaves the project browser; ONE undo brings it + // back with the rename still applied. + cx.update(|app| { + engine.update(app, |engine, cx| engine.delete_entry(entry.id, cx)) + }); + let gone = cx.read(|app| { + engine + .read(app) + .roots() + .iter() + .any(|e| e.id == entry.id) + }); + assert!(!gone, "the deleted entry leaves the project browser"); + cx.update(|app| { + engine.update(app, |engine, cx| engine.undo(cx)); + }); + let back = cx.read(|app| { + engine + .read(app) + .roots() + .iter() + .find(|e| e.id == entry.id) + .map(|e| e.name.clone()) + }); + assert_eq!( + back, + Some(SharedString::from("renamed-clip")), + "one undo restores the entry (with its rename)" + ); + } } diff --git a/crates/oak-app/src/oakui/renderops.rs b/crates/oak-app/src/oakui/renderops.rs index 9d0caf0bb..0b738a175 100644 --- a/crates/oak-app/src/oakui/renderops.rs +++ b/crates/oak-app/src/oakui/renderops.rs @@ -147,6 +147,67 @@ fn clip_preview_media( Some(preview_footage_media(f, is_video)) } +/// A multi-cam host clip's playable media: the CURRENT source's angle clip +/// (footage + media_in) rendered across the host clip's span. +/// +/// `None` when `clip` is not fed from a multicam node (or the resolved +/// angle has no footage). The montage clip's `media_in` maps the host +/// timeline position into the angle's media: `angle_media_in + (host_in - +/// angle_in)`. +/// +/// CALLER HOLDS THE PROJECT LOCK: this is pure `Graph` walking — the +/// `multicam::*` project-locking helpers MUST NOT be called from here +/// (non-recursive mutex). +fn clip_multicam_media( + g: &oak_node::graph::Graph, + clip: NodeId, + host_in: oak_core::Rational, +) -> Option<(String, i32, oak_core::Rational)> { + use oak_node::nodes::multicamnode::{CURRENT_INPUT, SEQUENCE_INPUT, SEQUENCE_TYPE_INPUT}; + // The clip's texture source must be a multicam node. + let mc = g.connected_output(clip, oak_node::block::clip_input::TEXTURE_INPUT, -1)?; + let mc_entry = g.get(mc)?; + if mc_entry.behavior.type_id() != "org.olivevideoeditor.Olive.multicam" { + return None; + } + let source = mc_entry.core.standard_value(CURRENT_INPUT, -1).to_double() as i32; + if source < 0 { + return None; + } + // The source sequence and the current source's track. + let seq = g.connected_output(mc, SEQUENCE_INPUT, -1)?; + let kind = { + let t = mc_entry.core.standard_value(SEQUENCE_TYPE_INPUT, -1).to_double() as i32; + oak_node::track::TrackType::from_c(t).unwrap_or(oak_node::track::TrackType::Video) + }; + let seq_pos = super::graphops::track_list_of(g, seq, kind)?; + let list = super::graphops::track_list_behavior(g, seq_pos)?; + let track = *list.tracks.get(source as usize)?; + let track_behavior = super::graphops::track_behavior(g, track)?; + // The angle clip covering the host position. + let angle = track_behavior.blocks.iter().find_map(|&block_id| { + let clip = super::graphops::clip_behavior(g, block_id)?; + let in_ = clip.core.in_(); + let out = clip.core.out(); + if host_in < in_ || host_in >= out { + return None; + } + let footage_id = super::graphops::find_input_footage(g, block_id)?; + let f = super::graphops::footage_behavior(g, footage_id)?; + if f.filename.is_empty() { + return None; + } + let (filename, stream_index) = preview_footage_media(f, true); + // Host position → angle media position: the host sits at angle + // media_in when the host's in == the angle's in; the offset shifts + // it by the difference. + let host_delta = host_in - in_; + let media_in = clip.core.media_in + host_delta; + Some((filename, stream_index, media_in)) + })?; + Some(angle) +} + /// The clip block's effect stack as montage effect descriptors /// (source-first order — the chain walk's signal order, so the renderer /// applies them media-side first). The chain walk runs all the way to @@ -214,17 +275,30 @@ pub fn video_montage(p: &ProjectRef, seq: NodeId, time: Rational) -> Vec= out { continue; } - let Some((filename, stream_index)) = - clip_preview_media(&g.graph, block_id, true) - else { - continue; + // A multi-cam host clip decodes through its multicam node's + // CURRENT source: the montage carries that angle's angle-clip + // media (footage + the angle clip's media_in), rendered at the + // host clip's timeline span. Without this the host clip's + // texture chain has no footage to preview (the sequence viewer + // painted black). + let montage_media = clip_multicam_media(&g.graph, block_id, in_); + let (filename, stream_index, media_in) = match montage_media { + Some(mc) => mc, + None => { + let Some((filename, stream_index)) = + clip_preview_media(&g.graph, block_id, true) + else { + continue; + }; + (filename, stream_index, clip.core.media_in) + } }; clips.push(MontageClip { filename, stream_index, in_time: in_, out_time: out, - media_in: clip.core.media_in, + media_in, gain: 1.0, effects: clip_effects(&g.graph, block_id), }); @@ -309,7 +383,16 @@ pub fn multicam_angle_frame_params( // initializers and deadlock the reentrant lock in `single_track_video_montage`. let project = lock(p).uuid.clone(); Ok(VideoTicketParams { - viewer: seq.identity(), + // The angle grid is a single-track MONTAGE render, not a full + // sequence-graph frame: viewer stays 0 so the worker takes the + // montage path. A nonzero viewer (the source sequence identity) + // made every angle request re-evaluate the WHOLE source sequence + // through the traverser — three grid cells × every tick of + // playback ground the machine to a halt ("switch then everything + // crawls"), and the graph render of the source sequence returned + // the stacked composite (all angles at once) rather than this + // one angle, which is why the cells showed the wrong/no picture. + viewer: 0, project, time, force_size: Some((width, height)), @@ -1359,7 +1442,349 @@ mod tests { let _ = std::fs::remove_file(&media); } - /// The original media decodes from the footage's actual first stream of + /// A multi-cam HOST clip's montage media is the CURRENT source's angle + /// clip (footage + media-in mapped from the host position) — without + /// this the host clip's texture chain has no footage and the sequence + /// viewer paints black. + #[test] + fn clip_multicam_media_resolves_the_current_angle() { + let _media = media_lock(); + let media = + std::env::temp_dir().join(format!("oakapp_mcmedia_{}.mp4", std::process::id())); + oak_codec::testmedia::write_test_clip(&media, 64, 64, 10, 10).expect("generate test media"); + + let (project, seq, footage) = project_with_clip(&media); + // Build the multicam shape: source seq (the angle track), multicam + // node, host seq with a clip fed from the multicam output. + let source_seq = graphops::create_sequence(&project, "Angles"); + graphops::add_track(&project, source_seq, TrackType::Video).expect("angle track"); + graphops::place_footage_clip(&project, source_seq, footage, TrackType::Video, 0, 0, 10, 0) + .expect("place the angle clip"); + let angle_clip = { + let g2 = graphops::lock(&project); + let track = graphops::track_ids(&g2.graph, source_seq, TrackType::Video)[0]; + graphops::track_behavior(&g2.graph, track) + .expect("angle track") + .blocks + .first() + .copied() + .expect("angle clip") + }; + let mc = { + let mut g = graphops::lock(&project); + let (core, behavior) = oak_node::nodes::multicamnode::create(); + let id = g.graph.add_node(core, behavior); + // Default current source: 0. + if let Some(core) = g.graph.get_mut(id).map(|e| &mut e.core) { + core.set_standard_value( + oak_node::nodes::multicamnode::CURRENT_INPUT, + -1, + oak_node::value::NodeValue::Combo(0), + ); + } + g.graph + .connect(source_seq, id, oak_node::nodes::multicamnode::SEQUENCE_INPUT, -1) + .unwrap(); + // The angle CLIP feeds the source array at element 0 (the + // traverser's `active_elements_at_time` keeps only the current + // source, so multi `value()` forwards exactly it). + g.graph + .input_array_insert( + id, + oak_node::nodes::multicamnode::SOURCES_INPUT, + 0, + ) + .unwrap(); + g.graph + .connect( + angle_clip, + id, + oak_node::nodes::multicamnode::SOURCES_INPUT, + 0, + ) + .unwrap(); + id + }; + // A host clip on the host sequence (default layout V1) fed from the + // multicam output; the clip's own input must first be connected. + let host_clip = graphops::place_footage_clip(&project, seq, footage, TrackType::Video, 0, 0, 10, 0) + .expect("place the host clip"); + { + let mut g = graphops::lock(&project); + let _ = g + .graph + .disconnect(footage, host_clip, oak_node::block::clip_input::TEXTURE_INPUT, -1); + g.graph + .connect(mc, host_clip, oak_node::block::clip_input::TEXTURE_INPUT, -1) + .unwrap(); + } + // The montage at host time 0 resolves to the angle's media. + let tb = graphops::sequence_time_base(&lock(&project).graph, seq).unwrap(); + let montage = { + let g = graphops::lock(&project); + clip_multicam_media(&g.graph, host_clip, graphops::ts_to_rational(0, tb)) + }; + let (filename, _stream, _media_in) = montage.expect("the current angle resolves"); + assert_eq!( + filename, + media.to_string_lossy(), + "the angle's footage decodes for the host clip" + ); + + // The FULL sequence-viewer path: video_montage of the host sequence + // -> render_montage_frame_into must produce a non-black frame (the + // host clip's angle media reaches the pixels — the "multi-cam clip + // shows black in the sequence viewer" report). + let montage = video_montage(&project, seq, graphops::ts_to_rational(0, tb)); + assert_eq!(montage.len(), 1, "the host clip is the only video entry"); + let params = VideoTicketParams { + viewer: 0, + project: String::new(), + time: graphops::ts_to_rational(0, tb), + force_size: Some((64, 64)), + force_format: Some(oak_core::PixelFormat::F32), + cache: None, + cache_dir: None, + cache_id: None, + cache_timebase: None, + footage: None, + montage, + }; + let mut dst = vec![0u8; 64 * 64 * 16]; + oak_render::eval::render_montage_frame_into( + graphops::ts_to_rational(0, tb), + ¶ms, + (64, 64), + &mut dst, + 64 * 16, + ) + .expect("montage render"); + let off = (8 * 64 + 8) * 16; + let r = f32::from_le_bytes(dst[off..off + 4].try_into().unwrap()); + assert!( + r > 0.05, + "the sequence viewer shows the angle's frame, not black (r={r})" + ); + + // The NODE-GRAPH evaluation path: `render_graph_frame` must also + // produce the current angle (footage → angle clip → multi → + // host clip → composite, all through `traverser::evaluate` + the + // value() chain — the "preview bypasses the node graph" fix). + // Needs the render manager initialized? No — this inline eval is + // pure (render_footage_frame decodes directly). + let graph_frame = oak_render::eval::render_graph_frame( + &project, + seq, + graphops::ts_to_rational(0, tb), + (64, 64), + oak_core::PixelFormat::F32, + ) + .expect("graph render"); + let grow; + let gdata; + let goff; + { + let oak_render::texture::Texture::Cpu(ref gf) = &graph_frame else { + panic!("graph render produced a non-CPU frame"); + }; + grow = gf.linesize_bytes(); + gdata = gf.data.clone(); + } + goff = (8 * grow as usize + 8 * 16) as usize; + let gr = f32::from_le_bytes(gdata[goff..goff + 4].try_into().unwrap()); + assert!( + gr > 0.05, + "the node-graph preview shows the angle's frame, not black (r={gr})" + ); + + oak_undo::global::clear().unwrap(); + let _ = std::fs::remove_file(&media); + } + + /// The multi-cam NODE-GRAPH source switch: two solid-color angles + /// (red/blue), the traverser evaluates `sources_in[source]` — the + /// rendered frame turns blue when `current_in` switches to source 1 + /// (the "value() always read None / the node graph is bypassed" + /// report's full regression). + #[test] + fn graph_render_switches_multicam_source() { + let _media = media_lock(); + let red = std::env::temp_dir().join(format!("oakapp_mcsw_r_{}.mp4", std::process::id())); + let blue = std::env::temp_dir().join(format!("oakapp_mcsw_b_{}.mp4", std::process::id())); + oak_codec::testmedia::write_test_clip_solid(&red, 64, 64, 10, 10, [1.0, 0.0, 0.0, 1.0]) + .expect("red angle"); + oak_codec::testmedia::write_test_clip_solid(&blue, 64, 64, 10, 10, [0.0, 0.0, 1.0, 1.0]) + .expect("blue angle"); + + let (project, seq, _rfoot) = project_with_clip(&red); + let bfoot = graphops::import_footage(&project, &blue).expect("import blue"); + graphops::add_track(&project, seq, TrackType::Video).expect("track 1"); + graphops::place_footage_clip(&project, seq, _rfoot, TrackType::Video, 1, 0, 10, 0) + .expect("red angle slot"); + + // The multicam: order matters — source 0 = red, source 1 = blue. + let track_red = { + let g = graphops::lock(&project); + graphops::track_ids(&g.graph, seq, TrackType::Video)[0] + }; + let red_clip = { + let g = graphops::lock(&project); + graphops::track_behavior(&g.graph, track_red) + .expect("track 0") + .blocks + .first() + .copied() + .expect("red clip") + }; + let blue_clip = graphops::place_footage_clip(&project, seq, bfoot, TrackType::Video, 1, 0, 10, 0) + .expect("blue angle slot"); + let mc = { + let mut g = graphops::lock(&project); + let (core, behavior) = oak_node::nodes::multicamnode::create(); + let id = g.graph.add_node(core, behavior); + if let Some(core) = g.graph.get_mut(id).map(|e| &mut e.core) { + core.set_standard_value( + oak_node::nodes::multicamnode::CURRENT_INPUT, + -1, + oak_node::value::NodeValue::Combo(0), + ); + } + g.graph + .input_array_insert(id, oak_node::nodes::multicamnode::SOURCES_INPUT, 0) + .unwrap(); + g.graph + .connect(red_clip, id, oak_node::nodes::multicamnode::SOURCES_INPUT, 0) + .unwrap(); + g.graph + .input_array_insert(id, oak_node::nodes::multicamnode::SOURCES_INPUT, 1) + .unwrap(); + g.graph + .connect(blue_clip, id, oak_node::nodes::multicamnode::SOURCES_INPUT, 1) + .unwrap(); + id + }; + // Host clip fed from the multicam output. + let host_clip = + graphops::place_footage_clip(&project, seq, _rfoot, TrackType::Video, 2, 0, 10, 0) + .expect("host clip"); + { + let mut g = graphops::lock(&project); + let _ = g.graph.disconnect( + _rfoot, + host_clip, + oak_node::block::clip_input::TEXTURE_INPUT, + -1, + ); + g.graph + .connect(mc, host_clip, oak_node::block::clip_input::TEXTURE_INPUT, -1) + .unwrap(); + } + + let tb = graphops::sequence_time_base(&lock(&project).graph, seq).unwrap(); + let at = graphops::ts_to_rational(0, tb); + let render_rgb = |mc_id: u64| -> (f32, f32) { + let pixel = { + let texture = oak_render::eval::render_graph_frame( + &project, + seq, + at, + (64, 64), + oak_core::PixelFormat::F32, + ) + .expect("graph render"); + let oak_render::texture::Texture::Cpu(ref gf) = texture else { + panic!("non-CPU frame"); + }; + let stride = gf.linesize_bytes(); + let off = (8 * stride as usize + 8 * 16) as usize; + ( + f32::from_le_bytes(gf.data[off..off + 4].try_into().unwrap()), + f32::from_le_bytes(gf.data[off + 8..off + 12].try_into().unwrap()), + ) + }; + let _ = mc_id; + pixel + }; + // Source 0 = red angle. + let (r0, b0) = render_rgb(0); + assert!(r0 > 0.4 && b0 < 0.4, "source 0 is the red angle (r={r0} b={b0})"); + + // Switch to source 1: current_in drives the element the multi + // `value()` forwards — the frame must turn blue. + { + let mut g = graphops::lock(&project); + if let Some(core) = g.graph.get_mut(mc).map(|e| &mut e.core) { + core.set_standard_value( + oak_node::nodes::multicamnode::CURRENT_INPUT, + -1, + oak_node::value::NodeValue::Combo(1), + ); + } + } + let (r1, b1) = render_rgb(0); + assert!( + b1 > 0.4 && r1 < 0.4, + "source 1 switches the node-graph frame to blue (r={r1} b={b1})" + ); + + // Playback load profile: 30 sequential graph frames vs the montage + // shortcut — the "now that the graph renders, previews crawl" + // report's baseline (in-process, no worker). A wide gap means the + // graph path needs a frame/decoder cache, not just correctness. + let measure_frames = |graph_mode: bool| -> f64 { + let start = std::time::Instant::now(); + let mut frames = 0; + for f in 1..=30 { + if graph_mode { + let texture = oak_render::eval::render_graph_frame( + &project, + seq, + graphops::ts_to_rational(f, tb), + (64, 64), + oak_core::PixelFormat::F32, + ) + .expect("graph frame"); + let _ = texture; + } else { + let montage = video_montage(&project, seq, graphops::ts_to_rational(f, tb)); + let params = VideoTicketParams { + viewer: 0, + project: String::new(), + time: graphops::ts_to_rational(f, tb), + force_size: Some((64, 64)), + force_format: Some(oak_core::PixelFormat::F32), + cache: None, + cache_dir: None, + cache_id: None, + cache_timebase: None, + footage: None, + montage, + }; + let mut dst = vec![0u8; 64 * 64 * 16]; + oak_render::eval::render_montage_frame_into( + graphops::ts_to_rational(f, tb), + ¶ms, + (64, 64), + &mut dst, + 64 * 16, + ) + .expect("montage frame"); + } + frames += 1; + } + start.elapsed().as_secs_f64() / frames as f64 + }; + let graph_ms = measure_frames(true) * 1000.0; + let montage_ms = measure_frames(false) * 1000.0; + eprintln!( + "[perf] graph {:?} ms/frame vs montage {:?} ms/frame (30 frames, 64x64)", + graph_ms, montage_ms + ); + + oak_undo::global::clear().unwrap(); + let _ = std::fs::remove_file(&red); + let _ = std::fs::remove_file(&blue); + } /// the kind (not the hardcoded 0/1 of a typical layout): a file whose /// video stream is not stream 0 must still decode its own video when /// the proxy switch is off or the proxy is not ready. diff --git a/crates/oak-app/src/oakui/waveform.rs b/crates/oak-app/src/oakui/waveform.rs index 44a3cdecf..2b56a914b 100644 --- a/crates/oak-app/src/oakui/waveform.rs +++ b/crates/oak-app/src/oakui/waveform.rs @@ -143,6 +143,41 @@ impl WaveformCache { }) } +/// Extract an audio RMS peak envelope directly from a media file's +/// audio stream — the multicam wizard's sync correlation input. Uses +/// the oakaudio waveform extractor's min/max points (per 256 source +/// samples), keeps the first channel, and maps each point onto a tiny +/// norm that the envelope correlator can compare across angles (the +/// point's peak = max(|min|, |max|), windowed 1/10 s). +/// +/// Returns `None` when the file has no decodable audio (the wizard then +/// falls back to timecode alignment). +pub fn extract_audio_envelope(filename: &str, stream_index: i32) -> Option> { + let cname = std::ffi::CString::new(filename).ok()?; + const SAMPLES_PER_POINT: i32 = 256; + const WINDOW_POINTS: usize = 10; // ~100 ms windows at 48 kHz + let outcome = oak_audio::waveform::extract(&cname, stream_index, SAMPLES_PER_POINT).ok()?; + if outcome.channels <= 0 || outcome.points.is_empty() { + return None; + } + let channel_count = outcome.channels.max(1) as usize; + let count = outcome.points.len() / channel_count; + // Peaks of the first channel. + let mut peaks: Vec = (0..count) + .map(|i| { + let p = outcome.points[i * channel_count]; + f64::max(f64::from(p.min).abs(), f64::from(p.max).abs()) + }) + .collect(); + // Window: every WINDOW_POINTS points collapses to one max (the + // envelope the correlator slides). + let mut envelope = Vec::with_capacity(count.div_ceil(WINDOW_POINTS)); + for chunk in peaks.chunks_mut(WINDOW_POINTS) { + envelope.push(chunk.iter().copied().fold(0.0, f64::max)); + } + Some(envelope) +} + /// The timeline's clip decorator: draws extracted waveforms into audio /// clips (M12 P4). Reads the engine-populated cache. pub struct OakClipDecorator { diff --git a/crates/oak-app/src/panels/project_explorer.rs b/crates/oak-app/src/panels/project_explorer.rs index 14fb07b62..c78640482 100644 --- a/crates/oak-app/src/panels/project_explorer.rs +++ b/crates/oak-app/src/panels/project_explorer.rs @@ -56,10 +56,16 @@ impl ProjectExplorerPanel { &explorer, |this, _explorer, event: &ProjectExplorerEvent, cx| match event { ProjectExplorerEvent::OpenRequested { id, .. } => { - // Open the item in the engine's model (double-click on a - // footage entry selects it for the source viewer). - this.engine - .update(cx, |engine, cx| engine.select_item(*id, cx)); + // A sequence entry opens in the timeline (the + // double-click on a sequence acts like opening it); + // footage entries select into the source viewer. + if this.engine.read(cx).entry_is_sequence(*id) { + this.engine + .update(cx, |engine, cx| engine.open_sequence_id(*id, cx)); + } else { + this.engine + .update(cx, |engine, cx| engine.select_item(*id, cx)); + } } ProjectExplorerEvent::FileDropRequested { paths, .. } => { // Drag-and-drop import: probe and add each dropped file @@ -152,7 +158,19 @@ impl ProjectExplorerPanel { }) .detach(); } - LOCAL_RENAME | LOCAL_DELETE | LOCAL_OPEN_IN_NEW_TAB => { + LOCAL_RENAME => { + let Some(id) = self.context_entry else { + return; + }; + cx.emit(RenameRequested(id)); + } + LOCAL_DELETE => { + let Some(id) = self.context_entry else { + return; + }; + cx.emit(DeleteRequested(id)); + } + LOCAL_OPEN_IN_NEW_TAB => { println!("[project explorer] menu action {item} (not implemented yet)"); } LOCAL_PROPERTIES => { @@ -289,6 +307,18 @@ pub struct NewSequenceRequested; impl EventEmitter for ProjectExplorerPanel {} +/// The project explorer asked the shell to rename entry `id`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RenameRequested(pub u64); + +impl EventEmitter for ProjectExplorerPanel {} + +/// The project explorer asked the shell to delete entry `id`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeleteRequested(pub u64); + +impl EventEmitter for ProjectExplorerPanel {} + impl DockPanel for ProjectExplorerPanel { fn panel_id(&self) -> gpui::dock::PanelId { PROJECT @@ -395,15 +425,22 @@ pub(crate) fn footage_menu( } /// A non-footage entry's context menu (folder / sequence): open-in-new-tab, -/// then rename / delete / properties. +/// then rename / delete / properties. Open-in-new-tab and the window +/// variant are pending the sequence tabs. pub(crate) fn entry_menu() -> Menu { + let open_tab = MenuItem::new( + LOCAL_OPEN_IN_NEW_TAB, + crate::i18n::tr("project.context.open_in_new_tab"), + ) + .disabled(); + let open_window = MenuItem::new( + LOCAL_OPEN_IN_NEW_WINDOW, + crate::i18n::tr("project.context.open_in_new_window"), + ) + .disabled(); Menu::new(vec![ - MenuItem::new(LOCAL_OPEN_IN_NEW_TAB, crate::i18n::tr("project.context.open_in_new_tab")), - MenuItem::new( - LOCAL_OPEN_IN_NEW_WINDOW, - crate::i18n::tr("project.context.open_in_new_window"), - ) - .separated(), + open_tab, + open_window.separated(), MenuItem::new(LOCAL_RENAME, crate::i18n::tr("project.context.rename")).separated(), MenuItem::new(LOCAL_DELETE, crate::i18n::tr("project.context.delete")), MenuItem::new(LOCAL_PROPERTIES, crate::i18n::tr("menu.context.properties")).separated(), diff --git a/crates/oak-node/src/nodes/multicamnode.rs b/crates/oak-node/src/nodes/multicamnode.rs index 7a4047639..bec0dcca7 100644 --- a/crates/oak-node/src/nodes/multicamnode.rs +++ b/crates/oak-node/src/nodes/multicamnode.rs @@ -18,6 +18,7 @@ //! `src/node/src/input/multicam/multicamnode.{h,cpp}`, //! `olive::MultiCamNode`). +use std::any::Any; use crate::factory::NodeMeta; use crate::id::NodeId; use crate::node::{Category, NodeBehavior, NodeCore}; @@ -256,14 +257,12 @@ impl NodeBehavior for MultiCamNode { } } - /// Evaluate outputs (C++ `value()`): pushes the first value of the - /// `sources_in` value array (which, per - /// `active_elements_at_time`, is the currently selected source); - /// pushes nothing when the array is empty. - /// - /// The Rust row carries no array payload (the `sources_in` value is - /// the single active element), so the connected value is pushed - /// through as-is; when the input is absent nothing is pushed. + /// Evaluate outputs (C++ `value()`): pushes the ARRAY element of + /// `sources_in` selected by `current_in` (the current source) — the + /// traverser stores array elements under `{input}[{element}]` keys + /// ([`crate::traverser`]), so the selected source's texture is the + /// value at `sources_in[source]`. A missing element (or an absent + /// array) pushes nothing. fn value( &self, core: &NodeCore, @@ -271,7 +270,19 @@ impl NodeBehavior for MultiCamNode { time: Rational, table: &mut NodeValueTable, ) { - let _ = (core, time); + let _ = time; + let source = core.standard_value(CURRENT_INPUT, -1).to_double() as i32; + let key = if source >= 0 { + format!("{SOURCES_INPUT}[{source}]") + } else { + SOURCES_INPUT.to_string() + }; + if let Some(v) = inputs.get(&key) { + table.push(v.value_type(), v.clone(), None); + return; + } + // Fallback: an unindexed source value (single-source wiring from + // before the element-tagged rows). if let Some(v) = inputs.get(SOURCES_INPUT) { table.push(v.value_type(), v.clone(), None); } diff --git a/crates/oak-node/src/traverser.rs b/crates/oak-node/src/traverser.rs index f41966cf4..10f3784f1 100644 --- a/crates/oak-node/src/traverser.rs +++ b/crates/oak-node/src/traverser.rs @@ -254,6 +254,11 @@ fn build_row( // which elements are live at this time (C++ // `GetActiveElementsAtTime` — a track pulls only the blocks // covering the frame). An empty answer means "no restriction". + // Element-tagged keys: an array input's per-element values coexist + // in the row under `{input}[{element}]` (C++ `GetValueAtTime` + // indexes the array; the multi-cam node reads exactly the element + // of its current source — a plain `id` key would collapse the + // array to its last element). if conns.iter().any(|(_, e)| *e >= 0) { let active = entry.behavior.active_elements_at_time(id, time); if !active.is_empty() { @@ -271,7 +276,12 @@ fn build_row( .get(&(from, upstream_time)) .map(|t| pick_value(t, entry.core.input_data_type(id))) .unwrap_or(NodeValue::None); - row.insert(id.to_string(), value); + let key = if element >= 0 { + format!("{id}[{element}]") + } else { + id.to_string() + }; + row.insert(key, value); } } row diff --git a/crates/oak-render/src/eval.rs b/crates/oak-render/src/eval.rs index 31159a145..1a717edfb 100644 --- a/crates/oak-render/src/eval.rs +++ b/crates/oak-render/src/eval.rs @@ -801,11 +801,48 @@ static DECODERS: std::sync::OnceLock< /// "4K 切换后大量 CUDA_ERROR_OUT_OF_MEMORY 报错" log flood). Software /// sessions (system RAM only) are evicted only when nothing else is /// available. -const MAX_CACHED_DECODERS: usize = 16; +const MAX_CACHED_DECODERS: usize = 6; /// LRU tick source for [`DECODERS`]. static DECODER_TICK: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); +/// Cap on decoded FRAMES cached per process (release builds of the graph +/// renderer re-decode every frame the pre-render window pulls — each is a +/// seek+decode on the shared decoder session, which serializes the graph +/// path behind the montage baseline. A small frame LRU lets playback pull +/// the forward window from cache instead of thrashing the decoder: the +/// window is sequential, so a short FIFO of recently decoded frames hits +/// on every pre-render restart and on repeat plays). +const MAX_CACHED_FRAMES: usize = 24; + +/// Decoded-frame LRU: `(filename, stream, time, w, h)` -> F32 CPU frame. +/// Frame data is the expensive part (a 1080p frame ≈ 31 MB); the decoder +/// session cache alone still re-decodes every `render_footage_frame`. +/// The size is part of the key: the same media at a different target +/// resolution is a different frame (an interleaved source-monitor/proxy +/// request must not reuse a wrongly-sized pixel buffer). +static DECODED_FRAMES: std::sync::OnceLock< + std::sync::Mutex< + std::collections::HashMap<(String, i32, (i64, i64), i32, i32), (Frame, u64)>, + >, +> = std::sync::OnceLock::new(); + +fn decoded_frames( +) -> std::sync::MutexGuard< + 'static, + std::collections::HashMap<(String, i32, (i64, i64), i32, i32), (Frame, u64)>, +> { + DECODED_FRAMES + .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new())) + .lock() + .unwrap_or_else(|e| e.into_inner()) +} + +/// Time-key rational (the frame-LRU's deterministic key half). +fn time_key(t: &Rational) -> (i64, i64) { + (t.numerator(), t.denominator()) +} + fn decoders( ) -> std::sync::MutexGuard< 'static, @@ -853,6 +890,33 @@ fn open_decoder(filename: &str, stream_index: i32) -> Result = Arc::new(FFmpegDecoder::new()); let stream = CodecStream::with_block(filename.to_string(), stream_index, None); + // NVDEC surface-pool budget BEFORE opening: each HARDWARE session + // pins big GPU memory chunks (a 1080p-4K HEVC pool ≈ 300-1000 MB). + // Several worker processes × sessions each exceed even a 16 GB card — + // the `CUDA_ERROR_OUT_OF_MEMORY` log flood. Per process O N L Y one + // hardware session may coexist (the video being played is shared, so + // one NVDEC context serves every montage/graph frame); when one is + // already open, evict it now so the new open has the VRAM budget. + // Eviction before open matters: post-open eviction is too late — the + // failing open was already trying to allocate. + { + let mut cache = decoders(); + const MAX_HARDWARE_SESSIONS: usize = 1; + let hw_count = cache + .iter() + .filter(|(_, (d, _))| d.hardware_decoding()) + .count(); + if hw_count >= MAX_HARDWARE_SESSIONS { + if let Some(victim) = cache + .iter() + .filter(|(_, (d, _))| d.hardware_decoding()) + .min_by_key(|(_, (_, t))| *t) + .map(|(k, _)| k.clone()) + { + cache.remove(&victim); + } + } + } decoder .open(&stream) .map_err(|e| Error::Failed(format!("footage decode open: {e:?}")))?; @@ -874,15 +938,32 @@ fn open_decoder(filename: &str, stream_index: i32) -> Result Result { - let decoder = open_decoder(filename, stream_index)?; - let (w, h) = size; + pub fn render_footage_frame( + filename: &str, + stream_index: i32, + time: Rational, + size: (i32, i32), + format: PixelFormat, + ) -> Result { + // Decoded frame LRU: playback asks for the same (or nearby) + // (file, stream, time) repeatedly (pre-render restarts, repeated + // scale-up at the same time, graph + montage interleaving); the + // decode itself is the expensive part and must not re-run per + // request. The target size is part of the key (see the cache + // type's doc). + let (w, h) = size; + let cache_size = if w > 0 && h > 0 { (w, h) } else { (-1, -1) }; + let cache_key = (filename.to_string(), stream_index, time_key(&time), cache_size.0, cache_size.1); + { + let mut cache = decoded_frames(); + let tick = DECODER_TICK.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let frame = cache.get(&cache_key).map(|(f, _)| f.clone()); + if let Some(frame) = frame { + cache.insert(cache_key.clone(), (frame.clone(), tick)); + return Ok(Texture::wrap_frame(frame)); + } + } + let decoder = open_decoder(filename, stream_index)?; let params = RetrieveVideoParams { stream: CodecStream::with_block(filename.to_string(), stream_index, None), time, @@ -943,6 +1024,24 @@ pub fn render_footage_frame( // Input node: source colorspace → the pipeline working space (ACEScg // by default; the legacy sRGB working space keeps the pass-through). convert_decoded_to_working(&mut dst, &decoded); + // Memoize the finished working-space frame (LRU-capped). + { + let mut cache = decoded_frames(); + let tick = DECODER_TICK.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if !cache.contains_key(&cache_key) { + if cache.len() >= MAX_CACHED_FRAMES { + if let Some(victim) = cache + .iter() + .filter(|(_, (_, t))| *t > 0) + .min_by_key(|(_, (_, t))| *t) + .map(|(k, _)| k.clone()) + { + cache.remove(&victim); + } + } + cache.insert(cache_key.clone(), (dst.clone(), tick)); + } + } Ok(Texture::wrap_frame(dst)) } @@ -1056,15 +1155,37 @@ fn composite_tracks_gpu( /// CPU composite of `frames` into one `size` frame — the fallback when no /// GPU device is available (or the pass fails): bottom (last) to top /// (first) via [`composite_over`]. +/// +/// GPU failures are remembered: a device whose wgpu pipeline fails +/// validation (e.g. an adapter that advertises ComputePipeline yet lacks +/// the required features) fails EVERY frame otherwise — each attempt +/// recompiles the shader, surfaces a validation error and stalls the +/// playback tick (the "picture barely updates on NVIDIA" report). After +/// one failure the composite stays on the CPU path for the process. +static GPU_COMPOSITE_FAILED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + fn composite_tracks(frames: Vec, size: (i32, i32)) -> Frame { let (w, h) = size; if w <= 0 || h <= 0 { return Frame::dummy(); } - if let Some(ctx) = crate::backend::GpuContext::shared() { - match composite_tracks_gpu(&ctx, &frames, (w, h)) { - Ok(frame) => return frame, - Err(err) => eprintln!("GPU track composite failed, using CPU: {err:#}"), + if !GPU_COMPOSITE_FAILED.load(std::sync::atomic::Ordering::Relaxed) { + // Single-frame composites (the common preview case) stay on the + // CPU path: the GPU route costs an upload+download round trip per + // frame for no benefit until two or more tracks overlap. + if frames.len() > 1 { + if let Some(ctx) = crate::backend::GpuContext::shared() { + match composite_tracks_gpu(&ctx, &frames, (w, h)) { + Ok(frame) => return frame, + Err(err) => { + eprintln!( + "GPU track composite failed, using CPU (and staying there): {err:#}" + ); + GPU_COMPOSITE_FAILED.store(true, std::sync::atomic::Ordering::Relaxed); + } + } + } } } let Ok(mut acc) = generate_frame(Rational::new(0, 1), (w, h), PixelFormat::F32) else { diff --git a/crates/oak-render/src/manager.rs b/crates/oak-render/src/manager.rs index 03e343b6a..7e1f50eca 100644 --- a/crates/oak-render/src/manager.rs +++ b/crates/oak-render/src/manager.rs @@ -83,7 +83,12 @@ pub struct RenderManager { /// for (M16 S1: dedup key — revisions alone collide across projects, /// since every fresh project shares small revision numbers). current_key: Mutex>, - /// Teardown in progress (M16 S1): set first thing in + /// The live project the current snapshot was written from — the + /// INLINE (test) backend's graph-mode render source. The process + /// backend loads the snapshot file in its workers instead; this slot + /// only exists so `render_produced_frame` can run the same + /// [`oak_node::traverser`] graph evaluation without forking a child. + inline_graph: Mutex>)>>, /// Teardown in progress (M16 S1): set first thing in /// [`RenderManager::shutdown`]; `set_graph_snapshot` / /// `clear_graph_snapshot` become no-ops afterwards so a stale push /// from a dying test/app cannot re-arm the worker pool mid-shutdown. @@ -112,6 +117,35 @@ impl RenderManager { } let backend = BackendKind::from_user_config(); let producer: crate::ticket::Producer = Arc::new(|time, params| { + // M16 S1 graph mode on the INLINE (test) backend: when the + // ticket names a viewer of the snapshotted project, evaluate + // the node graph (the same path the oak-worker pool runs in + // the process backend) instead of the montage shortcut — the + // preview must show the node graph, not the decoded-only + // montage. + if params.viewer != 0 && !params.project.is_empty() { + if let Some(m) = crate::manager::RenderManager::global() { + if let Some((uuid, project)) = m.inline_graph() { + if uuid == params.project { + if let Some(viewer_id) = + oak_node::id::NodeId::from_identity(params.viewer) + { + if let Ok(texture) = eval::render_graph_frame( + &project, + viewer_id, + time, + params.render_size(), + oak_core::PixelFormat::F32, + ) { + return Ok(crate::ticket::TicketPayload::Video( + texture, + )); + } + } + } + } + } + } eval::render_produced_frame(time, params) .map(crate::ticket::TicketPayload::Video) }); @@ -163,12 +197,22 @@ impl RenderManager { snapshots: GraphSnapshotStore::new(), current_snapshot: Mutex::new(None), current_key: Mutex::new(None), + inline_graph: Mutex::new(None), stopping: AtomicBool::new(false), process_pool, })); Ok(()) } + /// The inline backend's graph-mode render source: the live project the + /// current snapshot was written for, with its uuid. `None` when there + /// is no snapshot (or the process pool answers frames instead). + pub fn inline_graph( + &self, + ) -> Option<(String, std::sync::Arc>)> { + lock(&self.inline_graph).clone() + } + /// Global access; `None` before init. pub fn global() -> Option> { lock(&MANAGER).clone() @@ -226,6 +270,12 @@ impl RenderManager { return Ok(()); // teardown: no re-arm after the drain } let uuid = lock(project).uuid.clone(); + if std::env::var_os("OAK_DEBUG_SNAPSHOT").is_some() { + eprintln!( + "[snapshot] set_graph_snapshot uuid {uuid} revision {revision} key {:?}", + *lock(&self.current_key) + ); + } if *lock(&self.current_key) == Some((uuid.clone(), revision)) { return Ok(()); // unchanged state: no rewrite, no re-send } @@ -234,10 +284,30 @@ impl RenderManager { self.snapshots.release(&old); } self.dispatch.set_graph_snapshot(Some(path)); + // Keep a live graph handle for the inline backend's graph mode + // (the same snapshot content, in-process, no file required). + // `project` arrives by reference (the snapshot store's lifetime); + // oak-app installs the Arc via [`RenderManager::set_inline_project`] + // — the inline graph slot is only armed there, never from a + // borrowed reference. *lock(&self.current_key) = Some((uuid, revision)); Ok(()) } + /// Installs the live project the inline backend's graph mode renders + /// from (oak-app calls this on project adoption; it also arms the uuid + /// the snapshot dedups against). The process backend ignores it. + pub fn set_inline_project( + &self, + project: std::sync::Arc>, + ) { + let uuid = match project.lock() { + Ok(g) => g.uuid.clone(), + Err(p) => p.into_inner().uuid.clone(), + }; + *lock(&self.inline_graph) = Some((uuid, project)); + } + /// M16 S1 graph mode: force the workers to re-load the current project /// snapshot even when the undo-stack revision is unchanged. The /// color-settings dialog writes project settings directly (no undo @@ -277,6 +347,7 @@ impl RenderManager { self.snapshots.release(&old); } *lock(&self.current_key) = None; + *lock(&self.inline_graph) = None; self.dispatch.set_graph_snapshot(None); } diff --git a/crates/oak-render/src/procpool.rs b/crates/oak-render/src/procpool.rs index 414f1dab9..ee28f9d95 100644 --- a/crates/oak-render/src/procpool.rs +++ b/crates/oak-render/src/procpool.rs @@ -502,7 +502,16 @@ fn per_worker_gpu_budget(frame_size: (i32, i32), fps: u32) -> u64 { let pixels = (frame_size.0.max(0) as f64) * (frame_size.1.max(0) as f64); let pixel_ratio = pixels / (1920.0 * 1080.0); let fps_factor = (fps.max(1) as f64 / 24.0).sqrt().max(1.0); - let peak = 1u64 << 30; + // Peak-vs-quiet spread: a worker sustains a NVDEC session (surface + // pool + an in-flight frame upload) plus the wgpu pipeline and a + // couple of montage uploads. The 1 GiB 1080p / 4 GiB 4K estimates + // have been shown to still exhaust a 16 GB card when the pool pushes + // them simultaneously on several sources (the CUDA_ERROR_OUT_OF_MEMORY + // mid-playback flood), so the budget includes an extra headroom + // factor for undetected spikes (shared decoder surface growth during + // long GOP scans at a keyframe miss). + let headroom = 2; + let peak = (1u64 << 30) * headroom; let idle = 256u64 << 20; ((peak as f64 * pixel_ratio * fps_factor + idle as f64) as u64).max(1) } diff --git a/gpui b/gpui index 41dac8f33..209c593da 160000 --- a/gpui +++ b/gpui @@ -1 +1 @@ -Subproject commit 41dac8f33e12d9f657a451223511211241f485b1 +Subproject commit 209c593da605f480565ffb8144edec8edc574c37