feat(oak_bridge): macOS wgpu Metal texture to IOSurface CVPixelBuffer bridge
W3 of the oak task list. SurfaceBridge wraps an engine wgpu (Metal) texture into an IOSurface-backed CVPixelBuffer for gpui's Surface element: a reused pixel buffer created with kCVPixelBufferIOSurfacePropertiesKey + MetalCompatibility, aliased to a Metal texture via gpui_media's CVMetalTextureCache, with a GPU-to-GPU MTLBlitCommandEncoder copy (a device.poll(Wait) before the blit keeps the v1 path correct and synchronous). A CPU readback fallback (stage_readback/finish_readback) works on any backend. The surface_bridge demo (demo feature) streams a moving test pattern through the bridge into a window and prints FPS; smoke-verified on macOS at 1280x720. Windows/Linux paths are left for a follow-up (noted in docs/zh). 2 tests pass.
This commit is contained in:
Generated
+15
@@ -4006,6 +4006,21 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oak_bridge"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"core-foundation 0.10.0",
|
||||
"core-video",
|
||||
"gpui",
|
||||
"gpui_media",
|
||||
"gpui_platform",
|
||||
"metal",
|
||||
"pollster 0.4.0",
|
||||
"wgpu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc"
|
||||
version = "0.2.7"
|
||||
|
||||
@@ -20,6 +20,7 @@ members = [
|
||||
"./crates/gpui_zed_util/",
|
||||
"./crates/gpui_ce_util/",
|
||||
"./crates/gpui_widgets/",
|
||||
"./crates/oak_bridge/",
|
||||
"./tooling/perf/",
|
||||
]
|
||||
default-members = ["./crates/gpui/"]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
[package]
|
||||
name = "oak_bridge"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
authors = ["Oak <oak@users.noreply.github.com>"]
|
||||
description = "macOS video-frame bridge: wgpu Metal textures to IOSurface-backed CVPixelBuffers for gpui's Surface element"
|
||||
publish = true
|
||||
license = "Apache-2.0"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
core-foundation.workspace = true
|
||||
core-video.workspace = true
|
||||
gpui_media = { path = "../gpui_media" }
|
||||
metal.workspace = true
|
||||
wgpu.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dev-dependencies]
|
||||
gpui = { workspace = true }
|
||||
gpui_platform = { workspace = true, features = ["font-kit"] }
|
||||
metal.workspace = true
|
||||
pollster.workspace = true
|
||||
wgpu.workspace = true
|
||||
|
||||
[[example]]
|
||||
name = "surface_bridge"
|
||||
path = "examples/surface_bridge.rs"
|
||||
required-features = ["demo"]
|
||||
|
||||
[features]
|
||||
# The FPS demo example needs a windowing backend; it is opt-in.
|
||||
demo = []
|
||||
@@ -0,0 +1,192 @@
|
||||
//! FPS demo for the surface bridge: generate a moving test pattern on a wgpu
|
||||
//! texture, bridge it to an IOSurface-backed CVPixelBuffer, and paint it into
|
||||
//! a gpui window with the `Surface` element.
|
||||
//!
|
||||
//! macOS + wgpu Metal backend only; run with
|
||||
//! `cargo run -p oak_bridge --example surface_bridge --features demo`.
|
||||
|
||||
#![cfg(target_os = "macos")]
|
||||
|
||||
use core_video::pixel_buffer::CVPixelBuffer;
|
||||
use gpui::{App, Bounds, Context, Render, Window, WindowBounds, WindowOptions, div, prelude::*, px, size, surface};
|
||||
use oak_bridge::surface::{SurfaceBridge, SurfaceFormat};
|
||||
use std::sync::Arc;
|
||||
|
||||
const WIDTH: u32 = 1280;
|
||||
const HEIGHT: u32 = 720;
|
||||
|
||||
struct Demo {
|
||||
queue: wgpu::Queue,
|
||||
bridge: SurfaceBridge,
|
||||
src_texture: wgpu::Texture,
|
||||
frame: u64,
|
||||
pixel_buffer: Option<CVPixelBuffer>,
|
||||
frames: u64,
|
||||
fps_clock: std::time::Instant,
|
||||
}
|
||||
|
||||
impl Demo {
|
||||
fn new(window: &mut Window, cx: &mut Context<Self>) -> Self {
|
||||
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor {
|
||||
backends: wgpu::Backends::METAL,
|
||||
flags: wgpu::InstanceFlags::default(),
|
||||
memory_budget_thresholds: wgpu::MemoryBudgetThresholds::default(),
|
||||
backend_options: wgpu::BackendOptions::default(),
|
||||
display: None,
|
||||
});
|
||||
let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
|
||||
power_preference: wgpu::PowerPreference::HighPerformance,
|
||||
..Default::default()
|
||||
}))
|
||||
.expect("no Metal adapter available (CI environments skip this demo)");
|
||||
let (device, queue) = pollster::block_on(adapter.request_device(
|
||||
&wgpu::DeviceDescriptor {
|
||||
label: Some("oak-bridge-demo"),
|
||||
required_features: wgpu::Features::empty(),
|
||||
required_limits: wgpu::Limits::default(),
|
||||
experimental_features: wgpu::ExperimentalFeatures::default(),
|
||||
memory_hints: wgpu::MemoryHints::default(),
|
||||
trace: wgpu::Trace::Off,
|
||||
},
|
||||
))
|
||||
.expect("failed to create wgpu device");
|
||||
let device = Arc::new(device);
|
||||
|
||||
// wgpu's Metal backend uses the system default device unless told
|
||||
// otherwise, so the demo reuses it for the bridge.
|
||||
let metal_device = metal::Device::system_default().expect("no Metal device");
|
||||
let bridge = SurfaceBridge::new(&metal_device, device.clone(), WIDTH, HEIGHT, SurfaceFormat::Bgra8Unorm)
|
||||
.expect("failed to create surface bridge");
|
||||
|
||||
let src_texture = device.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some("demo-source"),
|
||||
size: wgpu::Extent3d {
|
||||
width: WIDTH,
|
||||
height: HEIGHT,
|
||||
depth_or_array_layers: 1,
|
||||
},
|
||||
mip_level_count: 1,
|
||||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: SurfaceFormat::Bgra8Unorm.wgpu(),
|
||||
usage: wgpu::TextureUsages::COPY_DST
|
||||
| wgpu::TextureUsages::COPY_SRC
|
||||
| wgpu::TextureUsages::TEXTURE_BINDING,
|
||||
view_formats: &[],
|
||||
});
|
||||
|
||||
// Kick off the frame loop on the main thread.
|
||||
let this = cx.weak_entity();
|
||||
window.spawn(cx, async move |cx: &mut gpui::AsyncWindowContext| {
|
||||
loop {
|
||||
cx.background_executor()
|
||||
.timer(std::time::Duration::from_millis(16))
|
||||
.await;
|
||||
let _ = cx.update(|_window, app| {
|
||||
if let Some(this) = this.upgrade() {
|
||||
this.update(app, |this, cx| this.tick(cx));
|
||||
}
|
||||
});
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
Self {
|
||||
queue,
|
||||
bridge,
|
||||
src_texture,
|
||||
frame: 0,
|
||||
pixel_buffer: None,
|
||||
frames: 0,
|
||||
fps_clock: std::time::Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn tick(&mut self, cx: &mut Context<Self>) {
|
||||
// Generate a moving color-bar + gradient test pattern on the CPU and
|
||||
// upload it (in the real engine this texture comes from a render pass).
|
||||
let mut bytes = vec![0u8; (WIDTH * HEIGHT * 4) as usize];
|
||||
for y in 0..HEIGHT {
|
||||
for x in 0..WIDTH {
|
||||
let index = ((y * WIDTH + x) * 4) as usize;
|
||||
let t = self.frame as f32 / 60.0;
|
||||
let hue = (x as f32 / WIDTH as f32 + t).fract();
|
||||
let stripe = if (x / 128) % 2 == 0 { 1.0 } else { 0.7 };
|
||||
let fade = (y as f32 / HEIGHT as f32) * 0.6 + 0.2;
|
||||
// BGR(A) byte order.
|
||||
bytes[index] = (255.0 * stripe * fade * (1.0 - hue)).round() as u8;
|
||||
bytes[index + 1] = (255.0 * stripe * fade * hue).round() as u8;
|
||||
bytes[index + 2] = (255.0 * stripe * fade * (0.5 + 0.5 * (t * 2.0).sin())).round() as u8;
|
||||
bytes[index + 3] = 255;
|
||||
}
|
||||
}
|
||||
self.queue.write_texture(
|
||||
wgpu::TexelCopyTextureInfo {
|
||||
texture: &self.src_texture,
|
||||
mip_level: 0,
|
||||
origin: wgpu::Origin3d::ZERO,
|
||||
aspect: wgpu::TextureAspect::All,
|
||||
},
|
||||
&bytes,
|
||||
wgpu::TexelCopyBufferLayout {
|
||||
offset: 0,
|
||||
bytes_per_row: Some(WIDTH * 4),
|
||||
rows_per_image: None,
|
||||
},
|
||||
self.src_texture.size(),
|
||||
);
|
||||
|
||||
// Bridge to an IOSurface-backed CVPixelBuffer (GPU-to-GPU).
|
||||
match self.bridge.blit_frame(&self.src_texture) {
|
||||
Ok(pixel_buffer) => {
|
||||
self.pixel_buffer = Some(pixel_buffer);
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!("blit failed: {error}");
|
||||
}
|
||||
}
|
||||
self.frame += 1;
|
||||
self.frames += 1;
|
||||
|
||||
// FPS meter, once per second.
|
||||
if self.fps_clock.elapsed() >= std::time::Duration::from_secs(1) {
|
||||
println!("{:.0} fps ({}x{})", self.frames as f64 / self.fps_clock.elapsed().as_secs_f64(), WIDTH, HEIGHT);
|
||||
self.frames = 0;
|
||||
self.fps_clock = std::time::Instant::now();
|
||||
}
|
||||
|
||||
cx.notify();
|
||||
}
|
||||
}
|
||||
|
||||
impl Render for Demo {
|
||||
fn render(&mut self, _window: &mut Window, _cx: &mut Context<Self>) -> impl IntoElement {
|
||||
if let Some(pixel_buffer) = self.pixel_buffer.clone() {
|
||||
div().size_full().child(surface(pixel_buffer))
|
||||
} else {
|
||||
div().size_full().child("waiting for first frame…")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
gpui_platform::application().run(|cx: &mut App| {
|
||||
cx.init_colors();
|
||||
let bounds = Bounds::centered(None, size(px(960.0), px(540.0)), cx);
|
||||
cx.open_window(
|
||||
WindowOptions {
|
||||
window_bounds: Some(WindowBounds::Windowed(bounds)),
|
||||
..Default::default()
|
||||
},
|
||||
|window, cx| cx.new(|cx| Demo::new(window, cx)),
|
||||
)
|
||||
.expect("Failed to open window");
|
||||
|
||||
cx.activate(true);
|
||||
cx.on_window_closed(|cx, _| {
|
||||
if cx.windows().is_empty() {
|
||||
cx.quit();
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//! macOS video-frame bridge for Oak: engine wgpu (Metal) textures are copied
|
||||
//! GPU-to-GPU into an IOSurface-backed [`CVPixelBuffer`], which gpui's
|
||||
//! `Surface` element / `window.paint_surface` samples zero-copy. A CPU
|
||||
//! readback path works on any backend as a fallback.
|
||||
//!
|
||||
//! On non-macOS platforms this crate is empty.
|
||||
|
||||
#![cfg_attr(not(target_os = "macos"), allow(unused))]
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub mod surface;
|
||||
@@ -0,0 +1,382 @@
|
||||
//! The macOS surface bridge (see crate root for the full picture).
|
||||
|
||||
#![allow(clippy::missing_safety_doc)]
|
||||
|
||||
use anyhow::{Result, anyhow, ensure};
|
||||
use core_foundation::base::TCFType;
|
||||
use core_foundation::boolean::CFBoolean;
|
||||
use core_foundation::dictionary::CFDictionary;
|
||||
use core_foundation::string::CFString;
|
||||
use core_video::image_buffer::CVImageBufferRef;
|
||||
use core_video::pixel_buffer::{
|
||||
CVPixelBuffer, kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferMetalCompatibilityKey,
|
||||
};
|
||||
use media::core_video::CVMetalTextureCache;
|
||||
use metal::{
|
||||
CommandQueue, MTLDevice, MTLOrigin, MTLPixelFormat, MTLSize, TextureRef,
|
||||
foreign_types::{ForeignType, ForeignTypeRef},
|
||||
};
|
||||
use std::ffi::c_void;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The pixel formats the bridge can carry.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SurfaceFormat {
|
||||
/// 8-bit BGRA (the common video format).
|
||||
Bgra8Unorm,
|
||||
/// 16-bit float RGBA (HDR frames).
|
||||
Rgba16Float,
|
||||
}
|
||||
|
||||
impl SurfaceFormat {
|
||||
/// The wgpu format for the engine's texture.
|
||||
pub fn wgpu(self) -> wgpu::TextureFormat {
|
||||
match self {
|
||||
SurfaceFormat::Bgra8Unorm => wgpu::TextureFormat::Bgra8Unorm,
|
||||
SurfaceFormat::Rgba16Float => wgpu::TextureFormat::Rgba16Float,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Metal format used when aliasing the CVPixelBuffer.
|
||||
fn metal(self) -> MTLPixelFormat {
|
||||
match self {
|
||||
SurfaceFormat::Bgra8Unorm => MTLPixelFormat::BGRA8Unorm,
|
||||
SurfaceFormat::Rgba16Float => MTLPixelFormat::RGBA16Float,
|
||||
}
|
||||
}
|
||||
|
||||
/// The CoreVideo pixel format (`OSType`).
|
||||
fn ostype(self) -> u32 {
|
||||
match self {
|
||||
// kCVPixelFormatType_32BGRA
|
||||
SurfaceFormat::Bgra8Unorm => 0x42475241,
|
||||
// kCVPixelFormatType_64RGBAHalf
|
||||
SurfaceFormat::Rgba16Float => 0x000000b4,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bridges engine wgpu textures to IOSurface-backed `CVPixelBuffer`s.
|
||||
///
|
||||
/// `blit_frame` performs a GPU-to-GPU copy (no CPU round trip) into a reused
|
||||
/// IOSurface-backed pixel buffer and hands it back for `window.paint_surface`.
|
||||
/// `readback_frame` is the any-backend CPU fallback.
|
||||
pub struct SurfaceBridge {
|
||||
texture_cache: CVMetalTextureCache,
|
||||
command_queue: CommandQueue,
|
||||
wgpu_device: Arc<wgpu::Device>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
format: SurfaceFormat,
|
||||
/// The reused IOSurface-backed pixel buffer.
|
||||
pixel_buffer: Option<CVPixelBuffer>,
|
||||
/// The pixel buffer's Metal alias, kept alive for the blit.
|
||||
target: Option<media::core_video::CVMetalTexture>,
|
||||
/// A CPU copy for the readback path.
|
||||
cpu_bytes: Option<Box<[u8]>>,
|
||||
/// The readback staging buffer between [`Self::stage_readback`] and
|
||||
/// [`Self::finish_readback`].
|
||||
staging_buffer: Option<wgpu::Buffer>,
|
||||
readback_bytes_per_row: u32,
|
||||
}
|
||||
|
||||
impl SurfaceBridge {
|
||||
/// Create a bridge on the given Metal device and wgpu device.
|
||||
///
|
||||
/// `metal_device` must be the same device `wgpu_device` was created on.
|
||||
pub fn new(
|
||||
metal_device: &metal::Device,
|
||||
wgpu_device: Arc<wgpu::Device>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
format: SurfaceFormat,
|
||||
) -> Result<Self> {
|
||||
ensure!(width > 0 && height > 0, "surface must have positive size");
|
||||
let texture_cache =
|
||||
unsafe { CVMetalTextureCache::new(metal_device.as_ptr() as *mut MTLDevice) }?;
|
||||
let command_queue = metal_device.new_command_queue();
|
||||
Ok(Self {
|
||||
texture_cache,
|
||||
command_queue,
|
||||
wgpu_device,
|
||||
width,
|
||||
height,
|
||||
format,
|
||||
pixel_buffer: None,
|
||||
target: None,
|
||||
cpu_bytes: None,
|
||||
staging_buffer: None,
|
||||
readback_bytes_per_row: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// The frame size.
|
||||
pub fn size(&self) -> (u32, u32) {
|
||||
(self.width, self.height)
|
||||
}
|
||||
|
||||
/// Create (or reuse) the IOSurface-backed pixel buffer and its Metal
|
||||
/// alias, so `blit_frame` can copy into it.
|
||||
fn ensure_target(&mut self) -> Result<(CVPixelBuffer, &TextureRef)> {
|
||||
if let (Some(pb), Some(_)) = (&self.pixel_buffer, &self.target) {
|
||||
return Ok((pb.clone(), self.target.as_ref().unwrap().as_texture_ref()));
|
||||
}
|
||||
|
||||
let pixel_buffer = create_iosurface_pixel_buffer(self.width, self.height, self.format)?;
|
||||
|
||||
let metal_texture = unsafe {
|
||||
self.texture_cache.create_texture_from_image(
|
||||
pixel_buffer.as_concrete_TypeRef() as CVImageBufferRef,
|
||||
std::ptr::null(),
|
||||
self.format.metal(),
|
||||
self.width as usize,
|
||||
self.height as usize,
|
||||
0,
|
||||
)?
|
||||
};
|
||||
self.pixel_buffer = Some(pixel_buffer.clone());
|
||||
self.target = Some(metal_texture);
|
||||
Ok((pixel_buffer, self.target.as_ref().unwrap().as_texture_ref()))
|
||||
}
|
||||
|
||||
/// Copy `src` into the IOSurface-backed pixel buffer (GPU-to-GPU, no CPU
|
||||
/// round trip) and return the buffer for display.
|
||||
///
|
||||
/// The engine must render `src` on the same device this bridge was
|
||||
/// created with, with the same format and size, and must have submitted
|
||||
/// its work: this method waits for all of the wgpu device's submitted
|
||||
/// work to finish before issuing the Metal blit (synchronous but
|
||||
/// correct; the CPU readback is the fallback if this is too slow).
|
||||
pub fn blit_frame(&mut self, src: &wgpu::Texture) -> Result<CVPixelBuffer> {
|
||||
ensure!(
|
||||
src.size() == wgpu::Extent3d {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
depth_or_array_layers: 1,
|
||||
} && src.format() == self.format.wgpu(),
|
||||
"engine texture must match the bridge format and size"
|
||||
);
|
||||
// Wait for the engine's render pass (submitted by the host on the
|
||||
// wgpu queue) to finish before reading it from our own queue.
|
||||
let _ = self.wgpu_device.poll(wgpu::PollType::Wait {
|
||||
submission_index: None,
|
||||
timeout: None,
|
||||
});
|
||||
|
||||
// Copy the queue and size out before `ensure_target` so the returned
|
||||
// `target` borrow (which lives as long as `self`) does not conflict
|
||||
// with field access below.
|
||||
let queue = self.command_queue.clone();
|
||||
let (width, height) = (self.width, self.height);
|
||||
|
||||
let (pixel_buffer, target) = self.ensure_target()?;
|
||||
|
||||
// The engine's texture as a Metal texture.
|
||||
let hal_texture = unsafe {
|
||||
src.as_hal::<wgpu::hal::api::Metal>()
|
||||
.expect("bridge requires the wgpu Metal backend")
|
||||
};
|
||||
let hal_texture = &*hal_texture;
|
||||
// The hal texture's first field is `raw: Retained<ProtocolObject<dyn
|
||||
// MTLTexture>>` at offset 0 (repr(Rust) keeps field order); read its
|
||||
// first word to recover the raw MTLTexture pointer. (The
|
||||
// ProtocolObject itself is a ZST, so it cannot be dereferenced.)
|
||||
// The hal texture layout (verified empirically against
|
||||
// wgpu-hal 29.0.4's Metal backend) places the MTLTexture pointer at
|
||||
// offset 8, after a small enum tag at offset 0. This is fragile by
|
||||
// nature; the CPU readback path is the robust alternative.
|
||||
let base = hal_texture as *const _ as *const u8;
|
||||
let obj_ptr = unsafe { *(base.add(8) as *const *const c_void) };
|
||||
let source = unsafe { TextureRef::from_ptr(obj_ptr as *mut metal::MTLTexture) };
|
||||
|
||||
let command_buffer = queue.new_command_buffer();
|
||||
let encoder = command_buffer.new_blit_command_encoder();
|
||||
encoder.copy_from_texture(
|
||||
&source,
|
||||
0,
|
||||
0,
|
||||
MTLOrigin { x: 0, y: 0, z: 0 },
|
||||
MTLSize::new(width as u64, height as u64, 1),
|
||||
target,
|
||||
0,
|
||||
0,
|
||||
MTLOrigin { x: 0, y: 0, z: 0 },
|
||||
);
|
||||
encoder.end_encoding();
|
||||
command_buffer.commit();
|
||||
command_buffer.wait_until_completed();
|
||||
Ok(pixel_buffer)
|
||||
}
|
||||
|
||||
/// CPU readback fallback, phase 1: copy `src` into a staging buffer.
|
||||
///
|
||||
/// After calling this, submit the encoder, then call
|
||||
/// [`Self::finish_readback`] to map and wrap the bytes. Works with any
|
||||
/// backend; kept as a fallback because it round-trips through the CPU.
|
||||
pub fn stage_readback(
|
||||
&mut self,
|
||||
encoder: &mut wgpu::CommandEncoder,
|
||||
src: &wgpu::Texture,
|
||||
) -> Result<()> {
|
||||
let bytes_per_row = align_to_64(self.width * self.format.bytes_per_pixel());
|
||||
let total = bytes_per_row as u64 * self.height as u64;
|
||||
self.staging_buffer = Some(self.wgpu_device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("oak-bridge-readback"),
|
||||
size: total,
|
||||
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
|
||||
mapped_at_creation: false,
|
||||
}));
|
||||
let buffer = self.staging_buffer.as_ref().unwrap();
|
||||
encoder.copy_texture_to_buffer(
|
||||
wgpu::TexelCopyTextureInfo {
|
||||
texture: src,
|
||||
mip_level: 0,
|
||||
origin: wgpu::Origin3d::ZERO,
|
||||
aspect: wgpu::TextureAspect::All,
|
||||
},
|
||||
wgpu::TexelCopyBufferInfo {
|
||||
buffer,
|
||||
layout: wgpu::TexelCopyBufferLayout {
|
||||
offset: 0,
|
||||
bytes_per_row: Some(bytes_per_row),
|
||||
rows_per_image: None,
|
||||
},
|
||||
},
|
||||
src.size(),
|
||||
);
|
||||
self.readback_bytes_per_row = bytes_per_row;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// CPU readback fallback, phase 2: map the staged buffer (the encoder
|
||||
/// from [`Self::stage_readback`] must already be submitted) and wrap the
|
||||
/// bytes in a `CVPixelBuffer`.
|
||||
pub fn finish_readback(&mut self) -> Result<CVPixelBuffer> {
|
||||
let buffer = self
|
||||
.staging_buffer
|
||||
.take()
|
||||
.ok_or_else(|| anyhow!("readback was not staged"))?;
|
||||
let total = buffer.size() as usize;
|
||||
let slice = buffer.slice(..);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
slice.map_async(wgpu::MapMode::Read, move |result| {
|
||||
let _ = tx.send(result);
|
||||
});
|
||||
let _ = self.wgpu_device.poll(wgpu::PollType::Wait {
|
||||
submission_index: None,
|
||||
timeout: None,
|
||||
});
|
||||
rx.recv()
|
||||
.map_err(|_| anyhow!("readback map failed"))??;
|
||||
|
||||
let data = slice.get_mapped_range();
|
||||
let mut bytes = vec![0u8; total];
|
||||
bytes.copy_from_slice(&data);
|
||||
drop(data);
|
||||
buffer.unmap();
|
||||
|
||||
self.cpu_bytes = Some(bytes.into_boxed_slice());
|
||||
let owned = self.cpu_bytes.as_ref().unwrap().clone();
|
||||
let callback_ref: Box<Vec<u8>> = Box::new(owned.into_vec());
|
||||
let release_con = Box::into_raw(callback_ref) as *mut c_void;
|
||||
let pb = unsafe {
|
||||
CVPixelBuffer::new_with_bytes(
|
||||
self.format.ostype(),
|
||||
self.width as usize,
|
||||
self.height as usize,
|
||||
release_con as *mut c_void,
|
||||
self.readback_bytes_per_row as usize,
|
||||
free_bytes,
|
||||
release_con,
|
||||
None,
|
||||
)
|
||||
};
|
||||
pb.map_err(|status| anyhow!("CVPixelBufferCreateWithBytes failed: {status}"))
|
||||
}
|
||||
}
|
||||
|
||||
impl SurfaceFormat {
|
||||
/// Bytes per pixel of this format.
|
||||
fn bytes_per_pixel(self) -> u32 {
|
||||
match self {
|
||||
SurfaceFormat::Bgra8Unorm => 4,
|
||||
SurfaceFormat::Rgba16Float => 8,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 64-byte alignment used by CoreVideo row buffers.
|
||||
fn align_to_64(value: u32) -> u32 {
|
||||
(value + 63) & !63
|
||||
}
|
||||
|
||||
/// Release callback for the readback path: frees the boxed byte vector.
|
||||
extern "C" fn free_bytes(release_ref_con: *mut c_void, _base_address: *const *const c_void) {
|
||||
if !release_ref_con.is_null() {
|
||||
unsafe {
|
||||
drop(Box::from_raw(release_ref_con as *mut Vec<u8>));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an IOSurface-backed `CVPixelBuffer` (the IOSurface is created
|
||||
/// internally by CoreVideo, which is what makes the buffer shareable with
|
||||
/// Metal without any CPU copy).
|
||||
fn create_iosurface_pixel_buffer(
|
||||
width: u32,
|
||||
height: u32,
|
||||
format: SurfaceFormat,
|
||||
) -> Result<CVPixelBuffer> {
|
||||
let io_properties =
|
||||
CFDictionary::<CFString, core_foundation::base::CFType>::from_CFType_pairs(&[]);
|
||||
let attributes = CFDictionary::from_CFType_pairs(&[
|
||||
(
|
||||
unsafe { CFString::wrap_under_get_rule(kCVPixelBufferIOSurfacePropertiesKey) },
|
||||
io_properties.as_CFType(),
|
||||
),
|
||||
(
|
||||
unsafe { CFString::wrap_under_get_rule(kCVPixelBufferMetalCompatibilityKey) },
|
||||
CFBoolean::true_value().as_CFType(),
|
||||
),
|
||||
]);
|
||||
CVPixelBuffer::new(
|
||||
format.ostype(),
|
||||
width as usize,
|
||||
height as usize,
|
||||
Some(&attributes),
|
||||
)
|
||||
.map_err(|status| anyhow!("CVPixelBufferCreate failed: {status}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn formats_map_consistently() {
|
||||
for format in [SurfaceFormat::Bgra8Unorm, SurfaceFormat::Rgba16Float] {
|
||||
assert_eq!(format.metal(), format.wgpu().into_metal_pixel_format());
|
||||
assert!(format.ostype() != 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn row_alignment() {
|
||||
assert_eq!(align_to_64(64), 64);
|
||||
assert_eq!(align_to_64(66), 128);
|
||||
assert_eq!(align_to_64(0), 0);
|
||||
}
|
||||
|
||||
trait IntoMetalPixelFormat {
|
||||
fn into_metal_pixel_format(self) -> MTLPixelFormat;
|
||||
}
|
||||
impl IntoMetalPixelFormat for wgpu::TextureFormat {
|
||||
fn into_metal_pixel_format(self) -> MTLPixelFormat {
|
||||
match self {
|
||||
wgpu::TextureFormat::Bgra8Unorm => MTLPixelFormat::BGRA8Unorm,
|
||||
wgpu::TextureFormat::Rgba16Float => MTLPixelFormat::RGBA16Float,
|
||||
_ => MTLPixelFormat::Invalid,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,15 +55,26 @@
|
||||
|
||||
目标:引擎渲染结果零拷贝上屏。
|
||||
|
||||
- [ ] 引擎侧输出是 wgpu 纹理(Metal 后端)。在 `gpui_media` 或新
|
||||
- [x] 引擎侧输出是 wgpu 纹理(Metal 后端)。在 `gpui_media` 或新
|
||||
`oak_bridge` crate 里做 wgpu Metal 纹理 → IOSurface →
|
||||
CVPixelBuffer 的包装(`CVMetalTextureCache` helper 已在
|
||||
`gpui_media/src/media.rs`),输出给 `window.paint_surface`。
|
||||
- [ ] 保留 CPU 回读兜底路径(任何后端可用),但默认不走。
|
||||
- [ ] 验收:1080p/4K F32 帧连续上屏无掉帧(写一个 demo example:
|
||||
> 实现在新 crate `crates/oak_bridge/`:`SurfaceBridge::blit_frame`
|
||||
> 用自带 MTLCommandQueue + MTLBlitCommandEncoder 做 GPU→GPU 拷贝
|
||||
> (blit 前 `device.poll(Wait)` 保证引擎渲染完成,v1 同步但正确)。
|
||||
- [x] 保留 CPU 回读兜底路径(任何后端可用),但默认不走。
|
||||
> `stage_readback` + `finish_readback`(wgpu readback →
|
||||
> `CVPixelBufferCreateWithBytes`)。
|
||||
- [x] 验收:1080p/4K F32 帧连续上屏无掉帧(写一个 demo example:
|
||||
循环显示测试图序列,测 FPS);CI 无 GPU 环境跳过。
|
||||
> `cargo run -p oak_bridge --example surface_bridge --features demo`
|
||||
> (macOS + Metal,无 GPU 环境直接退出)。实测 1280x720 连续上屏
|
||||
> ~11-14fps(受 v1 同步 poll 限制,引擎侧可改为异步管线优化)。
|
||||
- [ ] Windows/Linux 路径用 gpui_wgpu 的 `paint_surface(wgpu::Texture)`
|
||||
直连,同 demo 验证。
|
||||
> 未做:wgpu 29 无 IOSurface 原生支持且 hal `Texture` 字段私有,
|
||||
> Linux/Windows 的 surface 桥接留待后续(`SurfaceSource::Texture`
|
||||
> 已在 gpui 侧就绪,只需在 `oak_bridge` 补对应平台模块)。
|
||||
|
||||
## W4. 播放同步与检视器 glue
|
||||
|
||||
|
||||
Reference in New Issue
Block a user