diff --git a/Cargo.lock b/Cargo.lock index 1cc244d56..2ba380460 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4727,6 +4727,7 @@ dependencies = [ "gpui_elements", "gpui_platform", "gpui_widgets", + "half", "image", "oak-audio", "oak-codec", @@ -4741,6 +4742,7 @@ dependencies = [ "oak-undo", "serde_yaml", "smallvec", + "wgpu 29.0.4", ] [[package]] diff --git a/crates/oak-app/Cargo.toml b/crates/oak-app/Cargo.toml index 01ad584f2..85c8a913b 100644 --- a/crates/oak-app/Cargo.toml +++ b/crates/oak-app/Cargo.toml @@ -49,6 +49,11 @@ gpui_widgets = { path = "../../gpui/crates/gpui_widgets" } # `RenderImage`), matching the versions gpui itself uses. image = "0.25" smallvec = "1" +# The 10-bit display path: wgpu matches the version gpui_wgpu links (see the +# gpui workspace manifest), `half` packs the F32 pipeline output into +# Rgba16Float texture data (10-bit code values are distinguishable in f16). +wgpu = "29" +half = "2.7.1" # Editable-text widget (used by the file / export dialogs' path fields, the # same gpui-elements crate gpui_widgets builds on). gpui_elements = { path = "../../gpui/crates/gpui_elements" } diff --git a/crates/oak-app/src/app.rs b/crates/oak-app/src/app.rs index a32b85cf0..82874dd4d 100644 --- a/crates/oak-app/src/app.rs +++ b/crates/oak-app/src/app.rs @@ -3034,6 +3034,19 @@ fn run_with(args: AppArgs) { ..Default::default() }, |window, cx| { + // The 10-bit display path: hand the window's wgpu device + // to the engine so it can upload RGBA16F textures gpui's + // renderer samples straight into the swapchain (no 8-bit + // quantization). Linux/FreeBSD only — `gpu_context` does + // not exist on other platforms, where the BGRA8 CPU path + // stays in effect. + #[cfg(any(target_os = "linux", target_os = "freebsd"))] + if let Some(Ok(ctx)) = window + .gpu_context() + .map(|c| c.downcast::<(Arc, Arc)>()) + { + crate::oakui::gpu::register_context(ctx.0, ctx.1); + } // Compact pro-app text metrics: gpui's default rem is // 16px (desktop-app large); 14px matches the design's // density. All rem-based text scales; px spacing is diff --git a/crates/oak-app/src/oakui/frames.rs b/crates/oak-app/src/oakui/frames.rs index 8d00a48fe..4dafd6b81 100644 --- a/crates/oak-app/src/oakui/frames.rs +++ b/crates/oak-app/src/oakui/frames.rs @@ -135,6 +135,26 @@ pub(crate) fn bgra_bytes_to_render_image( ))) } +/// Packs F32 RGBA samples (the engine pipeline's pixel format) into the +/// half-float RGBA16F texture bytes of the 10-bit display path (the viewer +/// uploads these into a GPU texture and samples it straight to a 10-bit +/// swapchain — no 8-bit quantization in between). The 10-bit code step +/// `1/1023 ≈ 0.000977` is resolvable in f16 (whose ULP is ~0.000977 at +/// 1.0), so every displayable code survives the round trip. Samples must +/// hold exactly `width * height * 4` values (tightly packed rows); values +/// clamp to `0.0..=1.0` before packing. +pub(crate) fn f32_rgba_to_16f_bytes(width: u32, height: u32, samples: &[f32]) -> Option> { + if samples.len() != (width * height * 4) as usize { + return None; + } + let mut bytes = Vec::with_capacity(samples.len() * 2); + for &v in samples { + let h = half::f16::from_f32(v.clamp(0.0, 1.0)); + bytes.extend_from_slice(&h.to_bits().to_le_bytes()); + } + Some(bytes) +} + #[cfg(test)] mod tests { use super::*; @@ -148,4 +168,37 @@ mod tests { assert_eq!(&frame[0..4], &[127, 0, 255, 255]); assert_eq!(&frame[4..8], &[63, 0, 255, 255]); } + + #[test] + fn f32_round_trips_to_16f_without_8bit_quantization() { + // Two adjacent 10-bit codes near white: `v1 = 1022/1023` and + // `v2 = 1023/1023 = 1.0`. The 16f round trip keeps them on distinct + // codes (the f16 grid at 1.0 is 2^-10, ~the 10-bit step 1/1023, so + // the codes align), while the 8-bit path (×255) collapses both to the + // same code 255. + let (v1, v2) = (1022.0f32 / 1023.0, 1.0f32); + let samples = [v1, 0.0, 0.0, 1.0, v2, 0.0, 0.0, 1.0]; + let bytes = f32_rgba_to_16f_bytes(2, 1, &samples).expect("packed 16f bytes"); + assert_eq!(bytes.len(), 2 * 1 * 4 * 2, "two pixels, four f16 channels"); + let code_of = |value: f32| { + let h = half::f16::from_f32(value); + ((h.to_f32() * 1023.0).round()) as u32 + }; + assert_eq!( + code_of(v1), + 1022, + "v1 stays on 10-bit code 1022 after the 16f round trip" + ); + assert_eq!(code_of(v2), 1023, "white stays on code 1023"); + assert_ne!( + code_of(v1), + code_of(v2), + "adjacent 10-bit codes near white stay distinct through 16f" + ); + assert_eq!( + (v1 * 255.0).round() as u32, + (v2 * 255.0).round() as u32, + "the same two values collapse in 8-bit — proving the 16f path carries the resolution" + ); + } } diff --git a/crates/oak-app/src/oakui/gpu.rs b/crates/oak-app/src/oakui/gpu.rs new file mode 100644 index 000000000..7cebe6dc7 --- /dev/null +++ b/crates/oak-app/src/oakui/gpu.rs @@ -0,0 +1,125 @@ +// Oak Video Editor - Non-Linear Video Editor +// Copyright (C) 2026 Oak Team +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! The 10-bit display path's GPU uploads. +//! +//! The real and mock engines both produce their viewer picture as F32 RGBA +//! samples (the pipeline's internal format). Instead of downconverting to +//! BGRA8 and going through gpui's 8-bit sprite atlas, this module packs the +//! samples into a half-float RGBA16F texture and hands it to the viewer as a +//! [`SurfaceSource::Texture`](gpui::SurfaceSource) — the wgpu renderer +//! samples it straight into the (10-bit, `Rgb10a2Unorm`) swapchain, so no +//! 8-bit quantization happens between the render and the panel. +//! +//! The wgpu device/queue come from the window the app opens; the window +//! builder registers them once via [`register_context`]. Uploads are +//! best-effort: without a registered context (tests, headless runs) they +//! return `None` and the caller falls back to the BGRA8 CPU-frame path, +//! which keeps the pre-existing behavior. + +use std::sync::Mutex; + +/// The window's wgpu device/queue, registered by the app's window builder +/// (the same pair gpui's renderer draws with, so textures created here are +/// visible to it). `None` before the first window opens — uploads no-op. +static GPU_CONTEXT: Mutex, std::sync::Arc)>> = + Mutex::new(None); + +/// Register the window's wgpu device/queue for the 10-bit display path. +/// The app's window builder calls this once per window (last one wins; the +/// renderers all share the same device). +pub fn register_context(device: std::sync::Arc, queue: std::sync::Arc) { + if let Ok(mut ctx) = GPU_CONTEXT.lock() { + *ctx = Some((device, queue)); + } +} + +/// Whether a GPU context is registered (a window is open). The viewer uses +/// this to decide between the 10-bit surface path and the BGRA8 fallback. +pub fn context_ready() -> bool { + GPU_CONTEXT.lock().map(|ctx| ctx.is_some()).unwrap_or(false) +} + +/// Upload F32 RGBA samples (tightly packed, `width * height * 4` values) as +/// a half-float RGBA16F GPU texture for the 10-bit display path. Returns +/// `None` when no context is registered or the samples are malformed — the +/// caller then falls back to the BGRA8 CPU-frame path. +pub fn upload_rgba16f( + width: u32, + height: u32, + samples: &[f32], +) -> Option> { + if width == 0 || height == 0 || samples.len() != (width * height * 4) as usize { + return None; + } + let (device, queue) = { + let ctx = GPU_CONTEXT.lock().ok()?; + ctx.as_ref().map(|(d, q)| (d.clone(), q.clone()))? + }; + let bytes = super::frames::f32_rgba_to_16f_bytes(width, height, samples)?; + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("oak_display_rgba16f"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &bytes, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: None, + rows_per_image: None, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + Some(std::sync::Arc::new(texture)) +} + +/// Upload F32 display samples as an RGBA16F texture and register it for +/// `image_id` (the `RenderImage` it replaces) so the viewer switches to the +/// 10-bit surface path. Best-effort: without a registered context it's a +/// no-op and the viewer keeps the BGRA8 CPU-frame fallback. +pub fn register_display_frame(image_id: usize, width: u32, height: u32, samples: &[f32]) { + let Some(texture) = upload_rgba16f(width, height, samples) else { + return; + }; + gpui_widgets::viewer::register_gpu_frame( + image_id, + texture, + gpui::Size { + width: gpui::DevicePixels::from(width), + height: gpui::DevicePixels::from(height), + }, + ); +} diff --git a/crates/oak-app/src/oakui/mock.rs b/crates/oak-app/src/oakui/mock.rs index b3a432885..4c74a29a5 100644 --- a/crates/oak-app/src/oakui/mock.rs +++ b/crates/oak-app/src/oakui/mock.rs @@ -2756,6 +2756,10 @@ impl MockEngine { let image = Arc::new(crate::oakui::frames::f32_rgba_to_bgra_image( width, height, &samples, )); + // The 10-bit path: upload the F32 samples as an RGBA16F texture so + // the viewer samples them straight to the swapchain instead of the + // BGRA8 image (best-effort — no GPU, no registration, CPU path). + crate::oakui::gpu::register_display_frame(image.id.0, width, height, &samples); cache.insert(monitor, (frame.0, image.clone(), scope)); image } diff --git a/crates/oak-app/src/oakui/mod.rs b/crates/oak-app/src/oakui/mod.rs index 733bb2b1b..7f95c12fa 100644 --- a/crates/oak-app/src/oakui/mod.rs +++ b/crates/oak-app/src/oakui/mod.rs @@ -46,6 +46,7 @@ pub mod effectchain; pub mod engine; pub mod frames; pub mod graphops; +pub mod gpu; pub mod icons; pub mod mock; pub mod multicam; diff --git a/crates/oak-app/src/oakui/real.rs b/crates/oak-app/src/oakui/real.rs index 0cc3b5c02..006c20bcc 100644 --- a/crates/oak-app/src/oakui/real.rs +++ b/crates/oak-app/src/oakui/real.rs @@ -622,15 +622,51 @@ fn rendered_to_owned_image(rendered: &super::renderops::RenderedFrame) -> Option let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32); let pixels = f.shm.slot_to_vec(f.slot); let data = pixels.get(..meta.data_size.max(0) as usize)?; + if meta.format == super::renderops::PIXEL_FORMAT_F32 { + // M15 S3: the worker rendered F32 (the 10-bit display path) + // — repack the padded rows, apply the display transform and + // register the RGBA16F texture for the viewer; the BGRA8 + // image below is only the CPU fallback (scope / eyedropper / + // cached fills without a GPU). + let mut samples = repack_f32_row_bytes(meta.width, meta.height, meta.linesize, data)?; + super::displaycolor::apply_f32_rgba(&mut samples, (w * h) as i64); + let image = f32_rgba_to_bgra_image(w, h, &samples); + super::gpu::register_display_frame(image.id.0, w, h, &samples); + return Some(Arc::new(image)); + } bgra_bytes_to_render_image(w, h, data).map(Arc::new) } super::renderops::RenderedFrame::CpuF32 { .. } => { let (w, h, samples) = read_f32_frame(rendered)?; - Some(Arc::new(f32_rgba_to_bgra_image(w, h, &samples))) + let image = f32_rgba_to_bgra_image(w, h, &samples); + super::gpu::register_display_frame(image.id.0, w, h, &samples); + Some(Arc::new(image)) } } } +/// Repack one F32 RGBA shm slot (rows padded to `linesize`) into tightly +/// packed samples. The in-process variant is handled by [`read_f32_frame`]. +fn repack_f32_row_bytes(width: i32, height: i32, linesize: i32, data: &[u8]) -> Option> { + if width <= 0 || height <= 0 { + return None; + } + let row_bytes = (width * 4 * 4) as usize; + let linesize = (linesize as usize).max(row_bytes); + if data.len() < linesize * height as usize { + return None; + } + let mut samples = vec![0.0f32; (width * height * 4) as usize]; + for y in 0..height as usize { + let row = &data[y * linesize..y * linesize + row_bytes]; + for (i, px) in row.chunks_exact(4).enumerate() { + let v = f32::from_ne_bytes([px[0], px[1], px[2], px[3]]); + samples[y * (width as usize) * 4 + i] = v; + } + } + Some(samples) +} + // --------------------------------------------------------------------------- // Transport clock // --------------------------------------------------------------------------- @@ -1306,12 +1342,23 @@ impl RealEngine { return None; } let (width, height) = self.proxy_render_size()?; - match super::renderops::render_sequence_frame(&project, seq, frame.0, tb, width, height) { + match super::renderops::render_sequence_frame( + &project, + seq, + frame.0, + tb, + width, + height, + Some(oak_core::PixelFormat::F32), + ) { Ok(rendered) => { *slot = RendererSlot::Ready; - let out = rendered.to_display(); + let (image, scope, samples) = rendered.to_display()?; + if let Some(samples) = samples { + super::gpu::register_display_frame(image.id.0, width as u32, height as u32, &samples); + } release_rendered_frame(&rendered); - out + Some((image, scope)) } Err(error) => { println!("[real engine] render_frame failed: {error}"); @@ -1350,12 +1397,23 @@ impl RealEngine { return None; } let (width, height) = self.proxy_render_size()?; - match super::renderops::render_footage_frame(&project, node, frame.0, tb, width, height) { + match super::renderops::render_footage_frame( + &project, + node, + frame.0, + tb, + width, + height, + Some(oak_core::PixelFormat::F32), + ) { Ok(rendered) => { *slot = RendererSlot::Ready; - let out = rendered.to_display(); + let (image, scope, samples) = rendered.to_display()?; + if let Some(samples) = samples { + super::gpu::register_display_frame(image.id.0, width as u32, height as u32, &samples); + } release_rendered_frame(&rendered); - out + Some((image, scope)) } Err(error) => { println!("[real engine] source render_frame failed: {error}"); @@ -1414,12 +1472,24 @@ impl RealEngine { let mut event = None; if super::renderops::ensure_render_manager() { let rendered = match target { - FullResTarget::Sequence(seq) => { - super::renderops::render_sequence_frame(&project, seq, frame, tb, width, height) - } - FullResTarget::Footage(node) => { - super::renderops::render_footage_frame(&project, node, frame, tb, width, height) - } + FullResTarget::Sequence(seq) => super::renderops::render_sequence_frame( + &project, + seq, + frame, + tb, + width, + height, + Some(oak_core::PixelFormat::F32), + ), + FullResTarget::Footage(node) => super::renderops::render_footage_frame( + &project, + node, + frame, + tb, + width, + height, + Some(oak_core::PixelFormat::F32), + ), }; if let Ok(rendered) = rendered { // M15 S2: the process backend delivers a shm slot — copy the @@ -1785,11 +1855,11 @@ impl RealEngine { for frame in new_frames { let params = match monitor { Monitor::Program => super::renderops::sequence_frame_params( - &project, node, frame, tb, width, height, + &project, node, frame, tb, width, height, Some(oak_core::PixelFormat::F32), + ), + Monitor::Source => super::renderops::footage_frame_params( + &project, node, frame, tb, width, height, Some(oak_core::PixelFormat::F32), ), - Monitor::Source => { - super::renderops::footage_frame_params(&project, node, frame, tb, width, height) - } }; let Ok(params) = params else { continue }; let distance = frame.saturating_sub(playhead).abs(); @@ -1866,13 +1936,26 @@ impl RealEngine { let out = { let meta = &slot.meta; let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32); - let data = slot - .shm - .slot_bytes(slot.slot) - .get(..meta.data_size.max(0) as usize)?; - let image = bgra_bytes_to_render_image(w, h, data)?; - let scope = analyze_bgra8(w, h, data); - (Arc::new(image), scope) + if meta.format == super::renderops::PIXEL_FORMAT_F32 { + // M15 S3: the worker rendered F32 (the 10-bit display + // path) — repack/transform via `to_display` and register + // the RGBA16F texture for the viewer; the BGRA8 image it + // also builds is the CPU fallback only. + let (image, scope, samples) = + super::renderops::RenderedFrame::Shm(slot.clone()).to_display()?; + if let Some(samples) = samples { + super::gpu::register_display_frame(image.id.0, w, h, &samples); + } + (Arc::new(image), scope) + } else { + let data = slot + .shm + .slot_bytes(slot.slot) + .get(..meta.data_size.max(0) as usize)?; + let image = bgra_bytes_to_render_image(w, h, data)?; + let scope = analyze_bgra8(w, h, data); + (Arc::new(image), scope) + } }; if let Some(m) = RenderManager::global() { m.release_frame(&slot); @@ -2056,6 +2139,7 @@ impl RealEngine { (1, 1000), THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT, + None, ) .ok()?; let (width, height, bytes) = match &rendered { @@ -3501,6 +3585,9 @@ impl AppEngine for RealEngine { if gen != self.display_color_gen.get() { self.display_color_gen.set(gen); cache.clear(); + // The GPU textures hold the display-transformed pixels too — drop + // them with the CPU cache so the next frame re-uploads. + gpui_widgets::viewer::clear_gpu_frames(); } // The full-resolution fill replaces the proxy when its frame matches // the playhead; otherwise the proxy frame is displayed (rendered @@ -3550,10 +3637,9 @@ impl AppEngine for RealEngine { None => { let (width, height, samples) = synthetic_frame_samples(frame); let scope = analyze_f32_rgba(width, height, &samples); - ( - Arc::new(f32_rgba_to_bgra_image(width, height, &samples)), - scope, - ) + let image = f32_rgba_to_bgra_image(width, height, &samples); + super::gpu::register_display_frame(image.id.0, width, height, &samples); + (Arc::new(image), scope) } }; cache.entry(monitor).or_default().proxy = Some(ProxyEntry { @@ -6317,7 +6403,7 @@ mod tests { // The app's proxy size: sequence aspect (default 1920x1080) scaled // to a 480px long edge. - let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270) + let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270, None) .expect("render_frame must produce a frame"); assert_eq!((frame.width(), frame.height()), (480, 270)); assert!( @@ -6325,7 +6411,7 @@ mod tests { "the default process backend delivers shm slots (got format {})", frame.format() ); - let (image, _scope) = frame.to_display().expect("display image from the slot"); + let (image, _scope, _samples) = frame.to_display().expect("display image from the slot"); let bytes = image.as_bytes(0).expect("one frame"); assert_eq!(bytes.len(), 480 * 270 * 4, "BGRA8 proxy geometry"); assert!( @@ -6346,9 +6432,9 @@ mod tests { // Clip covering [0, 10) frames at the sequence's rate. graphops::place_footage_clip(&project, seq, footage, TrackType::Video, 0, 0, 10, 0) .expect("clip placement"); - let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270) + let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270, None) .expect("render_frame with a clip must produce a frame"); - let (image, _scope) = frame.to_display().expect("display image from the slot"); + let (image, _scope, _samples) = frame.to_display().expect("display image from the slot"); let bytes = image.as_bytes(0).expect("one frame"); let nonzero = bytes .chunks(4) @@ -6370,12 +6456,12 @@ mod tests { // A second frame at a later timestamp renders too. assert!( - crate::oakui::renderops::render_sequence_frame(&project, seq, 30, tb, 480, 270).is_ok() + crate::oakui::renderops::render_sequence_frame(&project, seq, 30, tb, 480, 270, None).is_ok() ); // Invalid geometry is rejected. assert!( - crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 0, 270).is_err() + crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 0, 270, None).is_err() ); oak_undo::global::clear().unwrap(); @@ -6418,13 +6504,13 @@ mod tests { let tb = graphops::sequence_time_base(&graphops::lock(&project).graph, seq).unwrap(); reset_main_heap_frame_copies(); - let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64) + let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64, None) .expect("render_frame must produce a frame"); let crate::oakui::renderops::RenderedFrame::Shm(slot) = &frame else { panic!("the process backend must deliver a shm slot"); }; assert_eq!(slot.meta.format, crate::oakui::renderops::SLOT_FORMAT_BGRA8); - let (image, _scope) = frame.to_display().expect("display image from the slot"); + let (image, _scope, _samples) = frame.to_display().expect("display image from the slot"); assert_eq!(image.as_bytes(0).expect("one frame").len(), 64 * 64 * 4); assert_eq!( main_heap_frame_copies(), @@ -6435,7 +6521,7 @@ mod tests { assert_eq!(main_heap_frame_copies(), 0); // The long-lived full-res path is the one counted copy. - let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64) + let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64, None) .expect("render_frame must produce a frame"); let image = rendered_to_owned_image(&frame).expect("owned full-res image"); assert_eq!(image.as_bytes(0).expect("one frame").len(), 64 * 64 * 4); diff --git a/crates/oak-app/src/oakui/renderops.rs b/crates/oak-app/src/oakui/renderops.rs index d9342c280..d8d3285a4 100644 --- a/crates/oak-app/src/oakui/renderops.rs +++ b/crates/oak-app/src/oakui/renderops.rs @@ -29,7 +29,7 @@ use std::sync::mpsc; use gpui::RenderImage; -use oak_core::{Rational, TimeRange}; +use oak_core::{PixelFormat, Rational, TimeRange}; use oak_node::id::NodeId; use oak_node::track::TrackType; use oak_render::manager::RenderManager; @@ -429,11 +429,13 @@ impl RenderedFrame { } } - /// The pixel format: the BGRA8 slot wire format for the shm variant, + /// The pixel format: the slot's wire format for the shm variant + /// (`PIXEL_FORMAT_F32` when the worker rendered F32 for the 10-bit + /// display path, `SLOT_FORMAT_BGRA8` for the 8-bit fallback), /// `PIXEL_FORMAT_F32` for the in-process variant. pub fn format(&self) -> i32 { match self { - RenderedFrame::Shm(_) => SLOT_FORMAT_BGRA8, + RenderedFrame::Shm(f) => f.meta.format, RenderedFrame::CpuF32 { .. } => PIXEL_FORMAT_F32, } } @@ -444,25 +446,43 @@ impl RenderedFrame { } /// Build the viewer display image plus the scope samples (M15 S2 - /// zero-copy onscreen path). For the shm variant the slot's BGRA8 - /// bytes are wrapped into the display buffer — the GPU-upload staging - /// copy, the single permitted main-process copy on the preview path - /// (design §3.5). The display color transform (display ICC) is applied - /// in place on that staging copy / on the F32 samples, so it costs no - /// extra copy. The caller releases the slot afterwards. - pub fn to_display(&self) -> Option<(RenderImage, ScopeData)> { + /// zero-copy onscreen path). For the shm variant the slot's bytes are + /// wrapped into the display buffer — the GPU-upload staging copy, the + /// single permitted main-process copy on the preview path (design §3.5). + /// The display color transform (display ICC) is applied in place on that + /// staging copy / on the F32 samples, so it costs no extra copy. + /// + /// The third return value is the display-transformed F32 RGBA samples + /// when the frame arrived in F32 (the 10-bit display path: the caller + /// uploads them as an RGBA16F texture and skips the BGRA8 image), `None` + /// for BGRA8 frames (whose 8-bit quantization is already baked in). The + /// caller releases the slot afterwards. + pub fn to_display(&self) -> Option<(RenderImage, ScopeData, Option>)> { match self { RenderedFrame::Shm(f) => { let meta = &f.meta; let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32); let pixels = f.shm.slot_bytes(f.slot); let data = pixels.get(..meta.data_size.max(0) as usize)?; - let scope = analyze_bgra8(w, h, data); - // The display transform edits the staging copy in place. - let mut owned = data.to_vec(); - super::displaycolor::apply_bgra8(&mut owned, (w * h) as i64); - let image = bgra_bytes_to_render_image(w, h, &owned)?; - Some((image, scope)) + if meta.format == PIXEL_FORMAT_F32 { + // M15 S3: the worker rendered F32 (the 10-bit display + // path) — repack the padded rows, transform and hand the + // samples back for the RGBA16F texture. + let mut samples = repack_f32_rows(meta.width, meta.height, meta.linesize, data)?; + let scope = analyze_f32_rgba(w, h, &samples); + super::displaycolor::apply_f32_rgba(&mut samples, (w * h) as i64); + let image = f32_rgba_to_bgra_image(w, h, &samples); + Some((image, scope, Some(samples))) + } else { + // BGRA8 slot: the worker already downconverted — wrap the + // bytes directly (no 10-bit path available for them). + let scope = analyze_bgra8(w, h, data); + // The display transform edits the staging copy in place. + let mut owned = data.to_vec(); + super::displaycolor::apply_bgra8(&mut owned, (w * h) as i64); + let image = bgra_bytes_to_render_image(w, h, &owned)?; + Some((image, scope, None)) + } } RenderedFrame::CpuF32 { width, @@ -477,6 +497,7 @@ impl RenderedFrame { Some(( f32_rgba_to_bgra_image(w, h, &samples), scope, + Some(samples), )) } } @@ -661,6 +682,9 @@ fn render_video(params: VideoTicketParams) -> Result { /// Build the video ticket params for one sequence-montage frame (M15 S2: /// shared by the synchronous render and the playback pre-render window). +/// `format` is the slot wire format the worker renders into: `None` keeps +/// the BGRA8 8-bit slot (the default), `Some(PixelFormat::F32)` renders F32 +/// for the 10-bit display path. pub fn sequence_frame_params( p: &ProjectRef, seq: NodeId, @@ -668,6 +692,7 @@ pub fn sequence_frame_params( tb: (i64, i64), width: i32, height: i32, + format: Option, ) -> Result { validate_geometry(width, height, tb)?; let time = Rational::new(frame_ts * tb.0, tb.1); @@ -684,7 +709,7 @@ pub fn sequence_frame_params( project, time, force_size: Some((width, height)), - force_format: None, + force_format: format, cache: None, cache_dir: None, cache_id: None, @@ -696,7 +721,8 @@ pub fn sequence_frame_params( /// Render one frame of the sequence's montage at `frame_ts` (a timestamp /// in the `(tb.1 / tb.0)`-per-frame timebase) into a `(width, height)` -/// frame. +/// frame. `format` is the slot wire format (see +/// [`sequence_frame_params`]). pub fn render_sequence_frame( p: &ProjectRef, seq: NodeId, @@ -704,13 +730,15 @@ pub fn render_sequence_frame( tb: (i64, i64), width: i32, height: i32, + format: Option, ) -> Result { - render_video(sequence_frame_params(p, seq, frame_ts, tb, width, height)?) + render_video(sequence_frame_params(p, seq, frame_ts, tb, width, height, format)?) } /// Build the video ticket params for one single-footage frame (M15 S2: /// shared by the synchronous render and the source-monitor pre-render -/// window). +/// window). `format` is the slot wire format (see +/// [`sequence_frame_params`]). pub fn footage_frame_params( p: &ProjectRef, footage: NodeId, @@ -718,6 +746,7 @@ pub fn footage_frame_params( tb: (i64, i64), width: i32, height: i32, + format: Option, ) -> Result { validate_geometry(width, height, tb)?; let (filename, stream_index, limits) = { @@ -746,7 +775,7 @@ pub fn footage_frame_params( project, time, force_size: Some((width, height)), - force_format: None, + force_format: format, cache: None, cache_dir: None, cache_id: None, @@ -757,7 +786,8 @@ pub fn footage_frame_params( } /// Render one frame of a single footage node (the source monitor) at -/// `frame_ts`, decoded straight from the media file. +/// `frame_ts`, decoded straight from the media file. `format` is the slot +/// wire format (see [`sequence_frame_params`]). pub fn render_footage_frame( p: &ProjectRef, footage: NodeId, @@ -765,8 +795,9 @@ pub fn render_footage_frame( tb: (i64, i64), width: i32, height: i32, + format: Option, ) -> Result { - render_video(footage_frame_params(p, footage, frame_ts, tb, width, height)?) + render_video(footage_frame_params(p, footage, frame_ts, tb, width, height, format)?) } /// Rendered interleaved f32 audio (the module audio ticket payload). @@ -1516,7 +1547,7 @@ mod tests { // …and the sequence frame ticket is clamped to its 32x32 size. let tb = graphops::sequence_time_base(&lock(&project).graph, seq).unwrap(); let params = - sequence_frame_params(&project, seq, 0, tb, 64, 64).expect("sequence frame params"); + sequence_frame_params(&project, seq, 0, tb, 64, 64, None).expect("sequence frame params"); assert_eq!(params.force_size, Some((32, 32)), "the proxy bounds the render size"); // The decoded pixels are the proxy's (solid blue), not the source's. diff --git a/gpui b/gpui index 4022d1db5..41b325833 160000 --- a/gpui +++ b/gpui @@ -1 +1 @@ -Subproject commit 4022d1db5e90765b2a46fb6604d773602321e5a4 +Subproject commit 41b32583399de476441680414a92adc6e865391a