app: true 10-bit presentation -- F32 frames reach the swapchain unquantized
CI / Build & test (Linux) (push) Failing after 17m38s
CI / Build & test (Windows) (push) Failing after 17m40s

The display path no longer passes through BGRA8: render workers are
asked for F32, rows are repacked, colour management runs on the F32
data, and the frame uploads to an Rgba16Float wgpu texture that the
viewer presents directly on the Rgb10a2Unorm swapchain. The BGRA8
image remains only for scopes, the eyedropper, thumbnails and the
no-GPU fallback.
This commit is contained in:
2026-08-27 19:55:05 +08:00
parent 5d21f83e1e
commit 246de6352f
10 changed files with 382 additions and 62 deletions
Generated
+2
View File
@@ -4727,6 +4727,7 @@ dependencies = [
"gpui_elements",
"gpui_platform",
"gpui_widgets",
"half",
"image",
"oak-audio",
"oak-codec",
@@ -4741,6 +4742,7 @@ dependencies = [
"oak-undo",
"serde_yaml",
"smallvec",
"wgpu 29.0.4",
]
[[package]]
+5
View File
@@ -49,6 +49,11 @@ gpui_widgets = { path = "../../gpui/crates/gpui_widgets" }
# `RenderImage`), matching the versions gpui itself uses.
image = "0.25"
smallvec = "1"
# The 10-bit display path: wgpu matches the version gpui_wgpu links (see the
# gpui workspace manifest), `half` packs the F32 pipeline output into
# Rgba16Float texture data (10-bit code values are distinguishable in f16).
wgpu = "29"
half = "2.7.1"
# Editable-text widget (used by the file / export dialogs' path fields, the
# same gpui-elements crate gpui_widgets builds on).
gpui_elements = { path = "../../gpui/crates/gpui_elements" }
+13
View File
@@ -3034,6 +3034,19 @@ fn run_with<E: AppEngine>(args: AppArgs) {
..Default::default()
},
|window, cx| {
// The 10-bit display path: hand the window's wgpu device
// to the engine so it can upload RGBA16F textures gpui's
// renderer samples straight into the swapchain (no 8-bit
// quantization). Linux/FreeBSD only — `gpu_context` does
// not exist on other platforms, where the BGRA8 CPU path
// stays in effect.
#[cfg(any(target_os = "linux", target_os = "freebsd"))]
if let Some(Ok(ctx)) = window
.gpu_context()
.map(|c| c.downcast::<(Arc<wgpu::Device>, Arc<wgpu::Queue>)>())
{
crate::oakui::gpu::register_context(ctx.0, ctx.1);
}
// Compact pro-app text metrics: gpui's default rem is
// 16px (desktop-app large); 14px matches the design's
// density. All rem-based text scales; px spacing is
+53
View File
@@ -135,6 +135,26 @@ pub(crate) fn bgra_bytes_to_render_image(
)))
}
/// Packs F32 RGBA samples (the engine pipeline's pixel format) into the
/// half-float RGBA16F texture bytes of the 10-bit display path (the viewer
/// uploads these into a GPU texture and samples it straight to a 10-bit
/// swapchain — no 8-bit quantization in between). The 10-bit code step
/// `1/1023 ≈ 0.000977` is resolvable in f16 (whose ULP is ~0.000977 at
/// 1.0), so every displayable code survives the round trip. Samples must
/// hold exactly `width * height * 4` values (tightly packed rows); values
/// clamp to `0.0..=1.0` before packing.
pub(crate) fn f32_rgba_to_16f_bytes(width: u32, height: u32, samples: &[f32]) -> Option<Vec<u8>> {
if samples.len() != (width * height * 4) as usize {
return None;
}
let mut bytes = Vec::with_capacity(samples.len() * 2);
for &v in samples {
let h = half::f16::from_f32(v.clamp(0.0, 1.0));
bytes.extend_from_slice(&h.to_bits().to_le_bytes());
}
Some(bytes)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -148,4 +168,37 @@ mod tests {
assert_eq!(&frame[0..4], &[127, 0, 255, 255]);
assert_eq!(&frame[4..8], &[63, 0, 255, 255]);
}
#[test]
fn f32_round_trips_to_16f_without_8bit_quantization() {
// Two adjacent 10-bit codes near white: `v1 = 1022/1023` and
// `v2 = 1023/1023 = 1.0`. The 16f round trip keeps them on distinct
// codes (the f16 grid at 1.0 is 2^-10, ~the 10-bit step 1/1023, so
// the codes align), while the 8-bit path (×255) collapses both to the
// same code 255.
let (v1, v2) = (1022.0f32 / 1023.0, 1.0f32);
let samples = [v1, 0.0, 0.0, 1.0, v2, 0.0, 0.0, 1.0];
let bytes = f32_rgba_to_16f_bytes(2, 1, &samples).expect("packed 16f bytes");
assert_eq!(bytes.len(), 2 * 1 * 4 * 2, "two pixels, four f16 channels");
let code_of = |value: f32| {
let h = half::f16::from_f32(value);
((h.to_f32() * 1023.0).round()) as u32
};
assert_eq!(
code_of(v1),
1022,
"v1 stays on 10-bit code 1022 after the 16f round trip"
);
assert_eq!(code_of(v2), 1023, "white stays on code 1023");
assert_ne!(
code_of(v1),
code_of(v2),
"adjacent 10-bit codes near white stay distinct through 16f"
);
assert_eq!(
(v1 * 255.0).round() as u32,
(v2 * 255.0).round() as u32,
"the same two values collapse in 8-bit — proving the 16f path carries the resolution"
);
}
}
+125
View File
@@ -0,0 +1,125 @@
// Oak Video Editor - Non-Linear Video Editor
// Copyright (C) 2026 Oak Team
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! The 10-bit display path's GPU uploads.
//!
//! The real and mock engines both produce their viewer picture as F32 RGBA
//! samples (the pipeline's internal format). Instead of downconverting to
//! BGRA8 and going through gpui's 8-bit sprite atlas, this module packs the
//! samples into a half-float RGBA16F texture and hands it to the viewer as a
//! [`SurfaceSource::Texture`](gpui::SurfaceSource) — the wgpu renderer
//! samples it straight into the (10-bit, `Rgb10a2Unorm`) swapchain, so no
//! 8-bit quantization happens between the render and the panel.
//!
//! The wgpu device/queue come from the window the app opens; the window
//! builder registers them once via [`register_context`]. Uploads are
//! best-effort: without a registered context (tests, headless runs) they
//! return `None` and the caller falls back to the BGRA8 CPU-frame path,
//! which keeps the pre-existing behavior.
use std::sync::Mutex;
/// The window's wgpu device/queue, registered by the app's window builder
/// (the same pair gpui's renderer draws with, so textures created here are
/// visible to it). `None` before the first window opens — uploads no-op.
static GPU_CONTEXT: Mutex<Option<(std::sync::Arc<wgpu::Device>, std::sync::Arc<wgpu::Queue>)>> =
Mutex::new(None);
/// Register the window's wgpu device/queue for the 10-bit display path.
/// The app's window builder calls this once per window (last one wins; the
/// renderers all share the same device).
pub fn register_context(device: std::sync::Arc<wgpu::Device>, queue: std::sync::Arc<wgpu::Queue>) {
if let Ok(mut ctx) = GPU_CONTEXT.lock() {
*ctx = Some((device, queue));
}
}
/// Whether a GPU context is registered (a window is open). The viewer uses
/// this to decide between the 10-bit surface path and the BGRA8 fallback.
pub fn context_ready() -> bool {
GPU_CONTEXT.lock().map(|ctx| ctx.is_some()).unwrap_or(false)
}
/// Upload F32 RGBA samples (tightly packed, `width * height * 4` values) as
/// a half-float RGBA16F GPU texture for the 10-bit display path. Returns
/// `None` when no context is registered or the samples are malformed — the
/// caller then falls back to the BGRA8 CPU-frame path.
pub fn upload_rgba16f(
width: u32,
height: u32,
samples: &[f32],
) -> Option<std::sync::Arc<wgpu::Texture>> {
if width == 0 || height == 0 || samples.len() != (width * height * 4) as usize {
return None;
}
let (device, queue) = {
let ctx = GPU_CONTEXT.lock().ok()?;
ctx.as_ref().map(|(d, q)| (d.clone(), q.clone()))?
};
let bytes = super::frames::f32_rgba_to_16f_bytes(width, height, samples)?;
let texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some("oak_display_rgba16f"),
size: wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba16Float,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
&bytes,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: None,
rows_per_image: None,
},
wgpu::Extent3d {
width,
height,
depth_or_array_layers: 1,
},
);
Some(std::sync::Arc::new(texture))
}
/// Upload F32 display samples as an RGBA16F texture and register it for
/// `image_id` (the `RenderImage` it replaces) so the viewer switches to the
/// 10-bit surface path. Best-effort: without a registered context it's a
/// no-op and the viewer keeps the BGRA8 CPU-frame fallback.
pub fn register_display_frame(image_id: usize, width: u32, height: u32, samples: &[f32]) {
let Some(texture) = upload_rgba16f(width, height, samples) else {
return;
};
gpui_widgets::viewer::register_gpu_frame(
image_id,
texture,
gpui::Size {
width: gpui::DevicePixels::from(width),
height: gpui::DevicePixels::from(height),
},
);
}
+4
View File
@@ -2756,6 +2756,10 @@ impl MockEngine {
let image = Arc::new(crate::oakui::frames::f32_rgba_to_bgra_image(
width, height, &samples,
));
// The 10-bit path: upload the F32 samples as an RGBA16F texture so
// the viewer samples them straight to the swapchain instead of the
// BGRA8 image (best-effort — no GPU, no registration, CPU path).
crate::oakui::gpu::register_display_frame(image.id.0, width, height, &samples);
cache.insert(monitor, (frame.0, image.clone(), scope));
image
}
+1
View File
@@ -46,6 +46,7 @@ pub mod effectchain;
pub mod engine;
pub mod frames;
pub mod graphops;
pub mod gpu;
pub mod icons;
pub mod mock;
pub mod multicam;
+123 -37
View File
@@ -622,15 +622,51 @@ fn rendered_to_owned_image(rendered: &super::renderops::RenderedFrame) -> Option
let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32);
let pixels = f.shm.slot_to_vec(f.slot);
let data = pixels.get(..meta.data_size.max(0) as usize)?;
if meta.format == super::renderops::PIXEL_FORMAT_F32 {
// M15 S3: the worker rendered F32 (the 10-bit display path)
// — repack the padded rows, apply the display transform and
// register the RGBA16F texture for the viewer; the BGRA8
// image below is only the CPU fallback (scope / eyedropper /
// cached fills without a GPU).
let mut samples = repack_f32_row_bytes(meta.width, meta.height, meta.linesize, data)?;
super::displaycolor::apply_f32_rgba(&mut samples, (w * h) as i64);
let image = f32_rgba_to_bgra_image(w, h, &samples);
super::gpu::register_display_frame(image.id.0, w, h, &samples);
return Some(Arc::new(image));
}
bgra_bytes_to_render_image(w, h, data).map(Arc::new)
}
super::renderops::RenderedFrame::CpuF32 { .. } => {
let (w, h, samples) = read_f32_frame(rendered)?;
Some(Arc::new(f32_rgba_to_bgra_image(w, h, &samples)))
let image = f32_rgba_to_bgra_image(w, h, &samples);
super::gpu::register_display_frame(image.id.0, w, h, &samples);
Some(Arc::new(image))
}
}
}
/// Repack one F32 RGBA shm slot (rows padded to `linesize`) into tightly
/// packed samples. The in-process variant is handled by [`read_f32_frame`].
fn repack_f32_row_bytes(width: i32, height: i32, linesize: i32, data: &[u8]) -> Option<Vec<f32>> {
if width <= 0 || height <= 0 {
return None;
}
let row_bytes = (width * 4 * 4) as usize;
let linesize = (linesize as usize).max(row_bytes);
if data.len() < linesize * height as usize {
return None;
}
let mut samples = vec![0.0f32; (width * height * 4) as usize];
for y in 0..height as usize {
let row = &data[y * linesize..y * linesize + row_bytes];
for (i, px) in row.chunks_exact(4).enumerate() {
let v = f32::from_ne_bytes([px[0], px[1], px[2], px[3]]);
samples[y * (width as usize) * 4 + i] = v;
}
}
Some(samples)
}
// ---------------------------------------------------------------------------
// Transport clock
// ---------------------------------------------------------------------------
@@ -1306,12 +1342,23 @@ impl RealEngine {
return None;
}
let (width, height) = self.proxy_render_size()?;
match super::renderops::render_sequence_frame(&project, seq, frame.0, tb, width, height) {
match super::renderops::render_sequence_frame(
&project,
seq,
frame.0,
tb,
width,
height,
Some(oak_core::PixelFormat::F32),
) {
Ok(rendered) => {
*slot = RendererSlot::Ready;
let out = rendered.to_display();
let (image, scope, samples) = rendered.to_display()?;
if let Some(samples) = samples {
super::gpu::register_display_frame(image.id.0, width as u32, height as u32, &samples);
}
release_rendered_frame(&rendered);
out
Some((image, scope))
}
Err(error) => {
println!("[real engine] render_frame failed: {error}");
@@ -1350,12 +1397,23 @@ impl RealEngine {
return None;
}
let (width, height) = self.proxy_render_size()?;
match super::renderops::render_footage_frame(&project, node, frame.0, tb, width, height) {
match super::renderops::render_footage_frame(
&project,
node,
frame.0,
tb,
width,
height,
Some(oak_core::PixelFormat::F32),
) {
Ok(rendered) => {
*slot = RendererSlot::Ready;
let out = rendered.to_display();
let (image, scope, samples) = rendered.to_display()?;
if let Some(samples) = samples {
super::gpu::register_display_frame(image.id.0, width as u32, height as u32, &samples);
}
release_rendered_frame(&rendered);
out
Some((image, scope))
}
Err(error) => {
println!("[real engine] source render_frame failed: {error}");
@@ -1414,12 +1472,24 @@ impl RealEngine {
let mut event = None;
if super::renderops::ensure_render_manager() {
let rendered = match target {
FullResTarget::Sequence(seq) => {
super::renderops::render_sequence_frame(&project, seq, frame, tb, width, height)
}
FullResTarget::Footage(node) => {
super::renderops::render_footage_frame(&project, node, frame, tb, width, height)
}
FullResTarget::Sequence(seq) => super::renderops::render_sequence_frame(
&project,
seq,
frame,
tb,
width,
height,
Some(oak_core::PixelFormat::F32),
),
FullResTarget::Footage(node) => super::renderops::render_footage_frame(
&project,
node,
frame,
tb,
width,
height,
Some(oak_core::PixelFormat::F32),
),
};
if let Ok(rendered) = rendered {
// M15 S2: the process backend delivers a shm slot — copy the
@@ -1785,11 +1855,11 @@ impl RealEngine {
for frame in new_frames {
let params = match monitor {
Monitor::Program => super::renderops::sequence_frame_params(
&project, node, frame, tb, width, height,
&project, node, frame, tb, width, height, Some(oak_core::PixelFormat::F32),
),
Monitor::Source => super::renderops::footage_frame_params(
&project, node, frame, tb, width, height, Some(oak_core::PixelFormat::F32),
),
Monitor::Source => {
super::renderops::footage_frame_params(&project, node, frame, tb, width, height)
}
};
let Ok(params) = params else { continue };
let distance = frame.saturating_sub(playhead).abs();
@@ -1866,13 +1936,26 @@ impl RealEngine {
let out = {
let meta = &slot.meta;
let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32);
let data = slot
.shm
.slot_bytes(slot.slot)
.get(..meta.data_size.max(0) as usize)?;
let image = bgra_bytes_to_render_image(w, h, data)?;
let scope = analyze_bgra8(w, h, data);
(Arc::new(image), scope)
if meta.format == super::renderops::PIXEL_FORMAT_F32 {
// M15 S3: the worker rendered F32 (the 10-bit display
// path) — repack/transform via `to_display` and register
// the RGBA16F texture for the viewer; the BGRA8 image it
// also builds is the CPU fallback only.
let (image, scope, samples) =
super::renderops::RenderedFrame::Shm(slot.clone()).to_display()?;
if let Some(samples) = samples {
super::gpu::register_display_frame(image.id.0, w, h, &samples);
}
(Arc::new(image), scope)
} else {
let data = slot
.shm
.slot_bytes(slot.slot)
.get(..meta.data_size.max(0) as usize)?;
let image = bgra_bytes_to_render_image(w, h, data)?;
let scope = analyze_bgra8(w, h, data);
(Arc::new(image), scope)
}
};
if let Some(m) = RenderManager::global() {
m.release_frame(&slot);
@@ -2056,6 +2139,7 @@ impl RealEngine {
(1, 1000),
THUMBNAIL_WIDTH,
THUMBNAIL_HEIGHT,
None,
)
.ok()?;
let (width, height, bytes) = match &rendered {
@@ -3501,6 +3585,9 @@ impl AppEngine for RealEngine {
if gen != self.display_color_gen.get() {
self.display_color_gen.set(gen);
cache.clear();
// The GPU textures hold the display-transformed pixels too — drop
// them with the CPU cache so the next frame re-uploads.
gpui_widgets::viewer::clear_gpu_frames();
}
// The full-resolution fill replaces the proxy when its frame matches
// the playhead; otherwise the proxy frame is displayed (rendered
@@ -3550,10 +3637,9 @@ impl AppEngine for RealEngine {
None => {
let (width, height, samples) = synthetic_frame_samples(frame);
let scope = analyze_f32_rgba(width, height, &samples);
(
Arc::new(f32_rgba_to_bgra_image(width, height, &samples)),
scope,
)
let image = f32_rgba_to_bgra_image(width, height, &samples);
super::gpu::register_display_frame(image.id.0, width, height, &samples);
(Arc::new(image), scope)
}
};
cache.entry(monitor).or_default().proxy = Some(ProxyEntry {
@@ -6317,7 +6403,7 @@ mod tests {
// The app's proxy size: sequence aspect (default 1920x1080) scaled
// to a 480px long edge.
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270)
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270, None)
.expect("render_frame must produce a frame");
assert_eq!((frame.width(), frame.height()), (480, 270));
assert!(
@@ -6325,7 +6411,7 @@ mod tests {
"the default process backend delivers shm slots (got format {})",
frame.format()
);
let (image, _scope) = frame.to_display().expect("display image from the slot");
let (image, _scope, _samples) = frame.to_display().expect("display image from the slot");
let bytes = image.as_bytes(0).expect("one frame");
assert_eq!(bytes.len(), 480 * 270 * 4, "BGRA8 proxy geometry");
assert!(
@@ -6346,9 +6432,9 @@ mod tests {
// Clip covering [0, 10) frames at the sequence's rate.
graphops::place_footage_clip(&project, seq, footage, TrackType::Video, 0, 0, 10, 0)
.expect("clip placement");
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270)
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 480, 270, None)
.expect("render_frame with a clip must produce a frame");
let (image, _scope) = frame.to_display().expect("display image from the slot");
let (image, _scope, _samples) = frame.to_display().expect("display image from the slot");
let bytes = image.as_bytes(0).expect("one frame");
let nonzero = bytes
.chunks(4)
@@ -6370,12 +6456,12 @@ mod tests {
// A second frame at a later timestamp renders too.
assert!(
crate::oakui::renderops::render_sequence_frame(&project, seq, 30, tb, 480, 270).is_ok()
crate::oakui::renderops::render_sequence_frame(&project, seq, 30, tb, 480, 270, None).is_ok()
);
// Invalid geometry is rejected.
assert!(
crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 0, 270).is_err()
crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 0, 270, None).is_err()
);
oak_undo::global::clear().unwrap();
@@ -6418,13 +6504,13 @@ mod tests {
let tb = graphops::sequence_time_base(&graphops::lock(&project).graph, seq).unwrap();
reset_main_heap_frame_copies();
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64)
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64, None)
.expect("render_frame must produce a frame");
let crate::oakui::renderops::RenderedFrame::Shm(slot) = &frame else {
panic!("the process backend must deliver a shm slot");
};
assert_eq!(slot.meta.format, crate::oakui::renderops::SLOT_FORMAT_BGRA8);
let (image, _scope) = frame.to_display().expect("display image from the slot");
let (image, _scope, _samples) = frame.to_display().expect("display image from the slot");
assert_eq!(image.as_bytes(0).expect("one frame").len(), 64 * 64 * 4);
assert_eq!(
main_heap_frame_copies(),
@@ -6435,7 +6521,7 @@ mod tests {
assert_eq!(main_heap_frame_copies(), 0);
// The long-lived full-res path is the one counted copy.
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64)
let frame = crate::oakui::renderops::render_sequence_frame(&project, seq, 0, tb, 64, 64, None)
.expect("render_frame must produce a frame");
let image = rendered_to_owned_image(&frame).expect("owned full-res image");
assert_eq!(image.as_bytes(0).expect("one frame").len(), 64 * 64 * 4);
+55 -24
View File
@@ -29,7 +29,7 @@
use std::sync::mpsc;
use gpui::RenderImage;
use oak_core::{Rational, TimeRange};
use oak_core::{PixelFormat, Rational, TimeRange};
use oak_node::id::NodeId;
use oak_node::track::TrackType;
use oak_render::manager::RenderManager;
@@ -429,11 +429,13 @@ impl RenderedFrame {
}
}
/// The pixel format: the BGRA8 slot wire format for the shm variant,
/// The pixel format: the slot's wire format for the shm variant
/// (`PIXEL_FORMAT_F32` when the worker rendered F32 for the 10-bit
/// display path, `SLOT_FORMAT_BGRA8` for the 8-bit fallback),
/// `PIXEL_FORMAT_F32` for the in-process variant.
pub fn format(&self) -> i32 {
match self {
RenderedFrame::Shm(_) => SLOT_FORMAT_BGRA8,
RenderedFrame::Shm(f) => f.meta.format,
RenderedFrame::CpuF32 { .. } => PIXEL_FORMAT_F32,
}
}
@@ -444,25 +446,43 @@ impl RenderedFrame {
}
/// Build the viewer display image plus the scope samples (M15 S2
/// zero-copy onscreen path). For the shm variant the slot's BGRA8
/// bytes are wrapped into the display buffer — the GPU-upload staging
/// copy, the single permitted main-process copy on the preview path
/// (design §3.5). The display color transform (display ICC) is applied
/// in place on that staging copy / on the F32 samples, so it costs no
/// extra copy. The caller releases the slot afterwards.
pub fn to_display(&self) -> Option<(RenderImage, ScopeData)> {
/// zero-copy onscreen path). For the shm variant the slot's bytes are
/// wrapped into the display buffer — the GPU-upload staging copy, the
/// single permitted main-process copy on the preview path (design §3.5).
/// The display color transform (display ICC) is applied in place on that
/// staging copy / on the F32 samples, so it costs no extra copy.
///
/// The third return value is the display-transformed F32 RGBA samples
/// when the frame arrived in F32 (the 10-bit display path: the caller
/// uploads them as an RGBA16F texture and skips the BGRA8 image), `None`
/// for BGRA8 frames (whose 8-bit quantization is already baked in). The
/// caller releases the slot afterwards.
pub fn to_display(&self) -> Option<(RenderImage, ScopeData, Option<Vec<f32>>)> {
match self {
RenderedFrame::Shm(f) => {
let meta = &f.meta;
let (w, h) = (meta.width.max(0) as u32, meta.height.max(0) as u32);
let pixels = f.shm.slot_bytes(f.slot);
let data = pixels.get(..meta.data_size.max(0) as usize)?;
let scope = analyze_bgra8(w, h, data);
// The display transform edits the staging copy in place.
let mut owned = data.to_vec();
super::displaycolor::apply_bgra8(&mut owned, (w * h) as i64);
let image = bgra_bytes_to_render_image(w, h, &owned)?;
Some((image, scope))
if meta.format == PIXEL_FORMAT_F32 {
// M15 S3: the worker rendered F32 (the 10-bit display
// path) — repack the padded rows, transform and hand the
// samples back for the RGBA16F texture.
let mut samples = repack_f32_rows(meta.width, meta.height, meta.linesize, data)?;
let scope = analyze_f32_rgba(w, h, &samples);
super::displaycolor::apply_f32_rgba(&mut samples, (w * h) as i64);
let image = f32_rgba_to_bgra_image(w, h, &samples);
Some((image, scope, Some(samples)))
} else {
// BGRA8 slot: the worker already downconverted — wrap the
// bytes directly (no 10-bit path available for them).
let scope = analyze_bgra8(w, h, data);
// The display transform edits the staging copy in place.
let mut owned = data.to_vec();
super::displaycolor::apply_bgra8(&mut owned, (w * h) as i64);
let image = bgra_bytes_to_render_image(w, h, &owned)?;
Some((image, scope, None))
}
}
RenderedFrame::CpuF32 {
width,
@@ -477,6 +497,7 @@ impl RenderedFrame {
Some((
f32_rgba_to_bgra_image(w, h, &samples),
scope,
Some(samples),
))
}
}
@@ -661,6 +682,9 @@ fn render_video(params: VideoTicketParams) -> Result<RenderedFrame, String> {
/// Build the video ticket params for one sequence-montage frame (M15 S2:
/// shared by the synchronous render and the playback pre-render window).
/// `format` is the slot wire format the worker renders into: `None` keeps
/// the BGRA8 8-bit slot (the default), `Some(PixelFormat::F32)` renders F32
/// for the 10-bit display path.
pub fn sequence_frame_params(
p: &ProjectRef,
seq: NodeId,
@@ -668,6 +692,7 @@ pub fn sequence_frame_params(
tb: (i64, i64),
width: i32,
height: i32,
format: Option<oak_core::PixelFormat>,
) -> Result<VideoTicketParams, String> {
validate_geometry(width, height, tb)?;
let time = Rational::new(frame_ts * tb.0, tb.1);
@@ -684,7 +709,7 @@ pub fn sequence_frame_params(
project,
time,
force_size: Some((width, height)),
force_format: None,
force_format: format,
cache: None,
cache_dir: None,
cache_id: None,
@@ -696,7 +721,8 @@ pub fn sequence_frame_params(
/// Render one frame of the sequence's montage at `frame_ts` (a timestamp
/// in the `(tb.1 / tb.0)`-per-frame timebase) into a `(width, height)`
/// frame.
/// frame. `format` is the slot wire format (see
/// [`sequence_frame_params`]).
pub fn render_sequence_frame(
p: &ProjectRef,
seq: NodeId,
@@ -704,13 +730,15 @@ pub fn render_sequence_frame(
tb: (i64, i64),
width: i32,
height: i32,
format: Option<oak_core::PixelFormat>,
) -> Result<RenderedFrame, String> {
render_video(sequence_frame_params(p, seq, frame_ts, tb, width, height)?)
render_video(sequence_frame_params(p, seq, frame_ts, tb, width, height, format)?)
}
/// Build the video ticket params for one single-footage frame (M15 S2:
/// shared by the synchronous render and the source-monitor pre-render
/// window).
/// window). `format` is the slot wire format (see
/// [`sequence_frame_params`]).
pub fn footage_frame_params(
p: &ProjectRef,
footage: NodeId,
@@ -718,6 +746,7 @@ pub fn footage_frame_params(
tb: (i64, i64),
width: i32,
height: i32,
format: Option<oak_core::PixelFormat>,
) -> Result<VideoTicketParams, String> {
validate_geometry(width, height, tb)?;
let (filename, stream_index, limits) = {
@@ -746,7 +775,7 @@ pub fn footage_frame_params(
project,
time,
force_size: Some((width, height)),
force_format: None,
force_format: format,
cache: None,
cache_dir: None,
cache_id: None,
@@ -757,7 +786,8 @@ pub fn footage_frame_params(
}
/// Render one frame of a single footage node (the source monitor) at
/// `frame_ts`, decoded straight from the media file.
/// `frame_ts`, decoded straight from the media file. `format` is the slot
/// wire format (see [`sequence_frame_params`]).
pub fn render_footage_frame(
p: &ProjectRef,
footage: NodeId,
@@ -765,8 +795,9 @@ pub fn render_footage_frame(
tb: (i64, i64),
width: i32,
height: i32,
format: Option<oak_core::PixelFormat>,
) -> Result<RenderedFrame, String> {
render_video(footage_frame_params(p, footage, frame_ts, tb, width, height)?)
render_video(footage_frame_params(p, footage, frame_ts, tb, width, height, format)?)
}
/// Rendered interleaved f32 audio (the module audio ticket payload).
@@ -1516,7 +1547,7 @@ mod tests {
// …and the sequence frame ticket is clamped to its 32x32 size.
let tb = graphops::sequence_time_base(&lock(&project).graph, seq).unwrap();
let params =
sequence_frame_params(&project, seq, 0, tb, 64, 64).expect("sequence frame params");
sequence_frame_params(&project, seq, 0, tb, 64, 64, None).expect("sequence frame params");
assert_eq!(params.force_size, Some((32, 32)), "the proxy bounds the render size");
// The decoded pixels are the proxy's (solid blue), not the source's.
+1 -1
Submodule gpui updated: 4022d1db5e...41b3258339