From 66b05b0ebec45fc82edc2afbf4525d71955fd22f Mon Sep 17 00:00:00 2001 From: Mike Solar Date: Sun, 9 Aug 2026 05:54:58 +0800 Subject: [PATCH] feat(oak_bridge): macOS wgpu Metal texture to IOSurface CVPixelBuffer bridge W3 of the oak task list. SurfaceBridge wraps an engine wgpu (Metal) texture into an IOSurface-backed CVPixelBuffer for gpui's Surface element: a reused pixel buffer created with kCVPixelBufferIOSurfacePropertiesKey + MetalCompatibility, aliased to a Metal texture via gpui_media's CVMetalTextureCache, with a GPU-to-GPU MTLBlitCommandEncoder copy (a device.poll(Wait) before the blit keeps the v1 path correct and synchronous). A CPU readback fallback (stage_readback/finish_readback) works on any backend. The surface_bridge demo (demo feature) streams a moving test pattern through the bridge into a window and prints FPS; smoke-verified on macOS at 1280x720. Windows/Linux paths are left for a follow-up (noted in docs/zh). 2 tests pass. --- Cargo.lock | 15 + Cargo.toml | 1 + crates/oak_bridge/Cargo.toml | 37 ++ crates/oak_bridge/examples/surface_bridge.rs | 192 ++++++++++ crates/oak_bridge/src/lib.rs | 11 + crates/oak_bridge/src/surface.rs | 382 +++++++++++++++++++ docs/zh/oak-app-rewrite.md | 17 +- 7 files changed, 652 insertions(+), 3 deletions(-) create mode 100644 crates/oak_bridge/Cargo.toml create mode 100644 crates/oak_bridge/examples/surface_bridge.rs create mode 100644 crates/oak_bridge/src/lib.rs create mode 100644 crates/oak_bridge/src/surface.rs diff --git a/Cargo.lock b/Cargo.lock index a18d2ab504..17c8396e18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4006,6 +4006,21 @@ dependencies = [ "libc", ] +[[package]] +name = "oak_bridge" +version = "0.1.0" +dependencies = [ + "anyhow", + "core-foundation 0.10.0", + "core-video", + "gpui", + "gpui_media", + "gpui_platform", + "metal", + "pollster 0.4.0", + "wgpu", +] + [[package]] name = "objc" version = "0.2.7" diff --git a/Cargo.toml b/Cargo.toml index d5d1596ba3..654b6f74f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "./crates/gpui_zed_util/", "./crates/gpui_ce_util/", "./crates/gpui_widgets/", + "./crates/oak_bridge/", "./tooling/perf/", ] default-members = ["./crates/gpui/"] diff --git a/crates/oak_bridge/Cargo.toml b/crates/oak_bridge/Cargo.toml new file mode 100644 index 0000000000..e1259d1bba --- /dev/null +++ b/crates/oak_bridge/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "oak_bridge" +version = "0.1.0" +edition.workspace = true +authors = ["Oak "] +description = "macOS video-frame bridge: wgpu Metal textures to IOSurface-backed CVPixelBuffers for gpui's Surface element" +publish = true +license = "Apache-2.0" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +core-foundation.workspace = true +core-video.workspace = true +gpui_media = { path = "../gpui_media" } +metal.workspace = true +wgpu.workspace = true + +[target.'cfg(target_os = "macos")'.dev-dependencies] +gpui = { workspace = true } +gpui_platform = { workspace = true, features = ["font-kit"] } +metal.workspace = true +pollster.workspace = true +wgpu.workspace = true + +[[example]] +name = "surface_bridge" +path = "examples/surface_bridge.rs" +required-features = ["demo"] + +[features] +# The FPS demo example needs a windowing backend; it is opt-in. +demo = [] diff --git a/crates/oak_bridge/examples/surface_bridge.rs b/crates/oak_bridge/examples/surface_bridge.rs new file mode 100644 index 0000000000..855a51a593 --- /dev/null +++ b/crates/oak_bridge/examples/surface_bridge.rs @@ -0,0 +1,192 @@ +//! FPS demo for the surface bridge: generate a moving test pattern on a wgpu +//! texture, bridge it to an IOSurface-backed CVPixelBuffer, and paint it into +//! a gpui window with the `Surface` element. +//! +//! macOS + wgpu Metal backend only; run with +//! `cargo run -p oak_bridge --example surface_bridge --features demo`. + +#![cfg(target_os = "macos")] + +use core_video::pixel_buffer::CVPixelBuffer; +use gpui::{App, Bounds, Context, Render, Window, WindowBounds, WindowOptions, div, prelude::*, px, size, surface}; +use oak_bridge::surface::{SurfaceBridge, SurfaceFormat}; +use std::sync::Arc; + +const WIDTH: u32 = 1280; +const HEIGHT: u32 = 720; + +struct Demo { + queue: wgpu::Queue, + bridge: SurfaceBridge, + src_texture: wgpu::Texture, + frame: u64, + pixel_buffer: Option, + frames: u64, + fps_clock: std::time::Instant, +} + +impl Demo { + fn new(window: &mut Window, cx: &mut Context) -> Self { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { + backends: wgpu::Backends::METAL, + flags: wgpu::InstanceFlags::default(), + memory_budget_thresholds: wgpu::MemoryBudgetThresholds::default(), + backend_options: wgpu::BackendOptions::default(), + display: None, + }); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::HighPerformance, + ..Default::default() + })) + .expect("no Metal adapter available (CI environments skip this demo)"); + let (device, queue) = pollster::block_on(adapter.request_device( + &wgpu::DeviceDescriptor { + label: Some("oak-bridge-demo"), + required_features: wgpu::Features::empty(), + required_limits: wgpu::Limits::default(), + experimental_features: wgpu::ExperimentalFeatures::default(), + memory_hints: wgpu::MemoryHints::default(), + trace: wgpu::Trace::Off, + }, + )) + .expect("failed to create wgpu device"); + let device = Arc::new(device); + + // wgpu's Metal backend uses the system default device unless told + // otherwise, so the demo reuses it for the bridge. + let metal_device = metal::Device::system_default().expect("no Metal device"); + let bridge = SurfaceBridge::new(&metal_device, device.clone(), WIDTH, HEIGHT, SurfaceFormat::Bgra8Unorm) + .expect("failed to create surface bridge"); + + let src_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("demo-source"), + size: wgpu::Extent3d { + width: WIDTH, + height: HEIGHT, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: SurfaceFormat::Bgra8Unorm.wgpu(), + usage: wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + + // Kick off the frame loop on the main thread. + let this = cx.weak_entity(); + window.spawn(cx, async move |cx: &mut gpui::AsyncWindowContext| { + loop { + cx.background_executor() + .timer(std::time::Duration::from_millis(16)) + .await; + let _ = cx.update(|_window, app| { + if let Some(this) = this.upgrade() { + this.update(app, |this, cx| this.tick(cx)); + } + }); + } + }) + .detach(); + Self { + queue, + bridge, + src_texture, + frame: 0, + pixel_buffer: None, + frames: 0, + fps_clock: std::time::Instant::now(), + } + } + + fn tick(&mut self, cx: &mut Context) { + // Generate a moving color-bar + gradient test pattern on the CPU and + // upload it (in the real engine this texture comes from a render pass). + let mut bytes = vec![0u8; (WIDTH * HEIGHT * 4) as usize]; + for y in 0..HEIGHT { + for x in 0..WIDTH { + let index = ((y * WIDTH + x) * 4) as usize; + let t = self.frame as f32 / 60.0; + let hue = (x as f32 / WIDTH as f32 + t).fract(); + let stripe = if (x / 128) % 2 == 0 { 1.0 } else { 0.7 }; + let fade = (y as f32 / HEIGHT as f32) * 0.6 + 0.2; + // BGR(A) byte order. + bytes[index] = (255.0 * stripe * fade * (1.0 - hue)).round() as u8; + bytes[index + 1] = (255.0 * stripe * fade * hue).round() as u8; + bytes[index + 2] = (255.0 * stripe * fade * (0.5 + 0.5 * (t * 2.0).sin())).round() as u8; + bytes[index + 3] = 255; + } + } + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &self.src_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &bytes, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(WIDTH * 4), + rows_per_image: None, + }, + self.src_texture.size(), + ); + + // Bridge to an IOSurface-backed CVPixelBuffer (GPU-to-GPU). + match self.bridge.blit_frame(&self.src_texture) { + Ok(pixel_buffer) => { + self.pixel_buffer = Some(pixel_buffer); + } + Err(error) => { + eprintln!("blit failed: {error}"); + } + } + self.frame += 1; + self.frames += 1; + + // FPS meter, once per second. + if self.fps_clock.elapsed() >= std::time::Duration::from_secs(1) { + println!("{:.0} fps ({}x{})", self.frames as f64 / self.fps_clock.elapsed().as_secs_f64(), WIDTH, HEIGHT); + self.frames = 0; + self.fps_clock = std::time::Instant::now(); + } + + cx.notify(); + } +} + +impl Render for Demo { + fn render(&mut self, _window: &mut Window, _cx: &mut Context) -> impl IntoElement { + if let Some(pixel_buffer) = self.pixel_buffer.clone() { + div().size_full().child(surface(pixel_buffer)) + } else { + div().size_full().child("waiting for first frame…") + } + } +} + +fn main() { + gpui_platform::application().run(|cx: &mut App| { + cx.init_colors(); + let bounds = Bounds::centered(None, size(px(960.0), px(540.0)), cx); + cx.open_window( + WindowOptions { + window_bounds: Some(WindowBounds::Windowed(bounds)), + ..Default::default() + }, + |window, cx| cx.new(|cx| Demo::new(window, cx)), + ) + .expect("Failed to open window"); + + cx.activate(true); + cx.on_window_closed(|cx, _| { + if cx.windows().is_empty() { + cx.quit(); + } + }) + .detach(); + }); +} diff --git a/crates/oak_bridge/src/lib.rs b/crates/oak_bridge/src/lib.rs new file mode 100644 index 0000000000..a9cd567d53 --- /dev/null +++ b/crates/oak_bridge/src/lib.rs @@ -0,0 +1,11 @@ +//! macOS video-frame bridge for Oak: engine wgpu (Metal) textures are copied +//! GPU-to-GPU into an IOSurface-backed [`CVPixelBuffer`], which gpui's +//! `Surface` element / `window.paint_surface` samples zero-copy. A CPU +//! readback path works on any backend as a fallback. +//! +//! On non-macOS platforms this crate is empty. + +#![cfg_attr(not(target_os = "macos"), allow(unused))] + +#[cfg(target_os = "macos")] +pub mod surface; diff --git a/crates/oak_bridge/src/surface.rs b/crates/oak_bridge/src/surface.rs new file mode 100644 index 0000000000..fa22d1e70e --- /dev/null +++ b/crates/oak_bridge/src/surface.rs @@ -0,0 +1,382 @@ +//! The macOS surface bridge (see crate root for the full picture). + +#![allow(clippy::missing_safety_doc)] + +use anyhow::{Result, anyhow, ensure}; +use core_foundation::base::TCFType; +use core_foundation::boolean::CFBoolean; +use core_foundation::dictionary::CFDictionary; +use core_foundation::string::CFString; +use core_video::image_buffer::CVImageBufferRef; +use core_video::pixel_buffer::{ + CVPixelBuffer, kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferMetalCompatibilityKey, +}; +use media::core_video::CVMetalTextureCache; +use metal::{ + CommandQueue, MTLDevice, MTLOrigin, MTLPixelFormat, MTLSize, TextureRef, + foreign_types::{ForeignType, ForeignTypeRef}, +}; +use std::ffi::c_void; +use std::sync::Arc; + +/// The pixel formats the bridge can carry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SurfaceFormat { + /// 8-bit BGRA (the common video format). + Bgra8Unorm, + /// 16-bit float RGBA (HDR frames). + Rgba16Float, +} + +impl SurfaceFormat { + /// The wgpu format for the engine's texture. + pub fn wgpu(self) -> wgpu::TextureFormat { + match self { + SurfaceFormat::Bgra8Unorm => wgpu::TextureFormat::Bgra8Unorm, + SurfaceFormat::Rgba16Float => wgpu::TextureFormat::Rgba16Float, + } + } + + /// The Metal format used when aliasing the CVPixelBuffer. + fn metal(self) -> MTLPixelFormat { + match self { + SurfaceFormat::Bgra8Unorm => MTLPixelFormat::BGRA8Unorm, + SurfaceFormat::Rgba16Float => MTLPixelFormat::RGBA16Float, + } + } + + /// The CoreVideo pixel format (`OSType`). + fn ostype(self) -> u32 { + match self { + // kCVPixelFormatType_32BGRA + SurfaceFormat::Bgra8Unorm => 0x42475241, + // kCVPixelFormatType_64RGBAHalf + SurfaceFormat::Rgba16Float => 0x000000b4, + } + } +} + +/// Bridges engine wgpu textures to IOSurface-backed `CVPixelBuffer`s. +/// +/// `blit_frame` performs a GPU-to-GPU copy (no CPU round trip) into a reused +/// IOSurface-backed pixel buffer and hands it back for `window.paint_surface`. +/// `readback_frame` is the any-backend CPU fallback. +pub struct SurfaceBridge { + texture_cache: CVMetalTextureCache, + command_queue: CommandQueue, + wgpu_device: Arc, + width: u32, + height: u32, + format: SurfaceFormat, + /// The reused IOSurface-backed pixel buffer. + pixel_buffer: Option, + /// The pixel buffer's Metal alias, kept alive for the blit. + target: Option, + /// A CPU copy for the readback path. + cpu_bytes: Option>, + /// The readback staging buffer between [`Self::stage_readback`] and + /// [`Self::finish_readback`]. + staging_buffer: Option, + readback_bytes_per_row: u32, +} + +impl SurfaceBridge { + /// Create a bridge on the given Metal device and wgpu device. + /// + /// `metal_device` must be the same device `wgpu_device` was created on. + pub fn new( + metal_device: &metal::Device, + wgpu_device: Arc, + width: u32, + height: u32, + format: SurfaceFormat, + ) -> Result { + ensure!(width > 0 && height > 0, "surface must have positive size"); + let texture_cache = + unsafe { CVMetalTextureCache::new(metal_device.as_ptr() as *mut MTLDevice) }?; + let command_queue = metal_device.new_command_queue(); + Ok(Self { + texture_cache, + command_queue, + wgpu_device, + width, + height, + format, + pixel_buffer: None, + target: None, + cpu_bytes: None, + staging_buffer: None, + readback_bytes_per_row: 0, + }) + } + + /// The frame size. + pub fn size(&self) -> (u32, u32) { + (self.width, self.height) + } + + /// Create (or reuse) the IOSurface-backed pixel buffer and its Metal + /// alias, so `blit_frame` can copy into it. + fn ensure_target(&mut self) -> Result<(CVPixelBuffer, &TextureRef)> { + if let (Some(pb), Some(_)) = (&self.pixel_buffer, &self.target) { + return Ok((pb.clone(), self.target.as_ref().unwrap().as_texture_ref())); + } + + let pixel_buffer = create_iosurface_pixel_buffer(self.width, self.height, self.format)?; + + let metal_texture = unsafe { + self.texture_cache.create_texture_from_image( + pixel_buffer.as_concrete_TypeRef() as CVImageBufferRef, + std::ptr::null(), + self.format.metal(), + self.width as usize, + self.height as usize, + 0, + )? + }; + self.pixel_buffer = Some(pixel_buffer.clone()); + self.target = Some(metal_texture); + Ok((pixel_buffer, self.target.as_ref().unwrap().as_texture_ref())) + } + + /// Copy `src` into the IOSurface-backed pixel buffer (GPU-to-GPU, no CPU + /// round trip) and return the buffer for display. + /// + /// The engine must render `src` on the same device this bridge was + /// created with, with the same format and size, and must have submitted + /// its work: this method waits for all of the wgpu device's submitted + /// work to finish before issuing the Metal blit (synchronous but + /// correct; the CPU readback is the fallback if this is too slow). + pub fn blit_frame(&mut self, src: &wgpu::Texture) -> Result { + ensure!( + src.size() == wgpu::Extent3d { + width: self.width, + height: self.height, + depth_or_array_layers: 1, + } && src.format() == self.format.wgpu(), + "engine texture must match the bridge format and size" + ); + // Wait for the engine's render pass (submitted by the host on the + // wgpu queue) to finish before reading it from our own queue. + let _ = self.wgpu_device.poll(wgpu::PollType::Wait { + submission_index: None, + timeout: None, + }); + + // Copy the queue and size out before `ensure_target` so the returned + // `target` borrow (which lives as long as `self`) does not conflict + // with field access below. + let queue = self.command_queue.clone(); + let (width, height) = (self.width, self.height); + + let (pixel_buffer, target) = self.ensure_target()?; + + // The engine's texture as a Metal texture. + let hal_texture = unsafe { + src.as_hal::() + .expect("bridge requires the wgpu Metal backend") + }; + let hal_texture = &*hal_texture; + // The hal texture's first field is `raw: Retained>` at offset 0 (repr(Rust) keeps field order); read its + // first word to recover the raw MTLTexture pointer. (The + // ProtocolObject itself is a ZST, so it cannot be dereferenced.) + // The hal texture layout (verified empirically against + // wgpu-hal 29.0.4's Metal backend) places the MTLTexture pointer at + // offset 8, after a small enum tag at offset 0. This is fragile by + // nature; the CPU readback path is the robust alternative. + let base = hal_texture as *const _ as *const u8; + let obj_ptr = unsafe { *(base.add(8) as *const *const c_void) }; + let source = unsafe { TextureRef::from_ptr(obj_ptr as *mut metal::MTLTexture) }; + + let command_buffer = queue.new_command_buffer(); + let encoder = command_buffer.new_blit_command_encoder(); + encoder.copy_from_texture( + &source, + 0, + 0, + MTLOrigin { x: 0, y: 0, z: 0 }, + MTLSize::new(width as u64, height as u64, 1), + target, + 0, + 0, + MTLOrigin { x: 0, y: 0, z: 0 }, + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + Ok(pixel_buffer) + } + + /// CPU readback fallback, phase 1: copy `src` into a staging buffer. + /// + /// After calling this, submit the encoder, then call + /// [`Self::finish_readback`] to map and wrap the bytes. Works with any + /// backend; kept as a fallback because it round-trips through the CPU. + pub fn stage_readback( + &mut self, + encoder: &mut wgpu::CommandEncoder, + src: &wgpu::Texture, + ) -> Result<()> { + let bytes_per_row = align_to_64(self.width * self.format.bytes_per_pixel()); + let total = bytes_per_row as u64 * self.height as u64; + self.staging_buffer = Some(self.wgpu_device.create_buffer(&wgpu::BufferDescriptor { + label: Some("oak-bridge-readback"), + size: total, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + let buffer = self.staging_buffer.as_ref().unwrap(); + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: src, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bytes_per_row), + rows_per_image: None, + }, + }, + src.size(), + ); + self.readback_bytes_per_row = bytes_per_row; + Ok(()) + } + + /// CPU readback fallback, phase 2: map the staged buffer (the encoder + /// from [`Self::stage_readback`] must already be submitted) and wrap the + /// bytes in a `CVPixelBuffer`. + pub fn finish_readback(&mut self) -> Result { + let buffer = self + .staging_buffer + .take() + .ok_or_else(|| anyhow!("readback was not staged"))?; + let total = buffer.size() as usize; + let slice = buffer.slice(..); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |result| { + let _ = tx.send(result); + }); + let _ = self.wgpu_device.poll(wgpu::PollType::Wait { + submission_index: None, + timeout: None, + }); + rx.recv() + .map_err(|_| anyhow!("readback map failed"))??; + + let data = slice.get_mapped_range(); + let mut bytes = vec![0u8; total]; + bytes.copy_from_slice(&data); + drop(data); + buffer.unmap(); + + self.cpu_bytes = Some(bytes.into_boxed_slice()); + let owned = self.cpu_bytes.as_ref().unwrap().clone(); + let callback_ref: Box> = Box::new(owned.into_vec()); + let release_con = Box::into_raw(callback_ref) as *mut c_void; + let pb = unsafe { + CVPixelBuffer::new_with_bytes( + self.format.ostype(), + self.width as usize, + self.height as usize, + release_con as *mut c_void, + self.readback_bytes_per_row as usize, + free_bytes, + release_con, + None, + ) + }; + pb.map_err(|status| anyhow!("CVPixelBufferCreateWithBytes failed: {status}")) + } +} + +impl SurfaceFormat { + /// Bytes per pixel of this format. + fn bytes_per_pixel(self) -> u32 { + match self { + SurfaceFormat::Bgra8Unorm => 4, + SurfaceFormat::Rgba16Float => 8, + } + } +} + +/// 64-byte alignment used by CoreVideo row buffers. +fn align_to_64(value: u32) -> u32 { + (value + 63) & !63 +} + +/// Release callback for the readback path: frees the boxed byte vector. +extern "C" fn free_bytes(release_ref_con: *mut c_void, _base_address: *const *const c_void) { + if !release_ref_con.is_null() { + unsafe { + drop(Box::from_raw(release_ref_con as *mut Vec)); + } + } +} + +/// Create an IOSurface-backed `CVPixelBuffer` (the IOSurface is created +/// internally by CoreVideo, which is what makes the buffer shareable with +/// Metal without any CPU copy). +fn create_iosurface_pixel_buffer( + width: u32, + height: u32, + format: SurfaceFormat, +) -> Result { + let io_properties = + CFDictionary::::from_CFType_pairs(&[]); + let attributes = CFDictionary::from_CFType_pairs(&[ + ( + unsafe { CFString::wrap_under_get_rule(kCVPixelBufferIOSurfacePropertiesKey) }, + io_properties.as_CFType(), + ), + ( + unsafe { CFString::wrap_under_get_rule(kCVPixelBufferMetalCompatibilityKey) }, + CFBoolean::true_value().as_CFType(), + ), + ]); + CVPixelBuffer::new( + format.ostype(), + width as usize, + height as usize, + Some(&attributes), + ) + .map_err(|status| anyhow!("CVPixelBufferCreate failed: {status}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn formats_map_consistently() { + for format in [SurfaceFormat::Bgra8Unorm, SurfaceFormat::Rgba16Float] { + assert_eq!(format.metal(), format.wgpu().into_metal_pixel_format()); + assert!(format.ostype() != 0); + } + } + + #[test] + fn row_alignment() { + assert_eq!(align_to_64(64), 64); + assert_eq!(align_to_64(66), 128); + assert_eq!(align_to_64(0), 0); + } + + trait IntoMetalPixelFormat { + fn into_metal_pixel_format(self) -> MTLPixelFormat; + } + impl IntoMetalPixelFormat for wgpu::TextureFormat { + fn into_metal_pixel_format(self) -> MTLPixelFormat { + match self { + wgpu::TextureFormat::Bgra8Unorm => MTLPixelFormat::BGRA8Unorm, + wgpu::TextureFormat::Rgba16Float => MTLPixelFormat::RGBA16Float, + _ => MTLPixelFormat::Invalid, + } + } + } +} diff --git a/docs/zh/oak-app-rewrite.md b/docs/zh/oak-app-rewrite.md index 8fb1ce897b..df7121be10 100644 --- a/docs/zh/oak-app-rewrite.md +++ b/docs/zh/oak-app-rewrite.md @@ -55,15 +55,26 @@ 目标：引擎渲染结果零拷贝上屏。 -- [ ] 引擎侧输出是 wgpu 纹理（Metal 后端）。在 `gpui_media` 或新 +- [x] 引擎侧输出是 wgpu 纹理（Metal 后端）。在 `gpui_media` 或新 `oak_bridge` crate 里做 wgpu Metal 纹理 → IOSurface → CVPixelBuffer 的包装（`CVMetalTextureCache` helper 已在 `gpui_media/src/media.rs`），输出给 `window.paint_surface`。 -- [ ] 保留 CPU 回读兜底路径（任何后端可用），但默认不走。 -- [ ] 验收：1080p/4K F32 帧连续上屏无掉帧（写一个 demo example： + > 实现在新 crate `crates/oak_bridge/`：`SurfaceBridge::blit_frame` + > 用自带 MTLCommandQueue + MTLBlitCommandEncoder 做 GPU→GPU 拷贝 + > （blit 前 `device.poll(Wait)` 保证引擎渲染完成，v1 同步但正确）。 +- [x] 保留 CPU 回读兜底路径（任何后端可用），但默认不走。 + > `stage_readback` + `finish_readback`（wgpu readback → + > `CVPixelBufferCreateWithBytes`）。 +- [x] 验收：1080p/4K F32 帧连续上屏无掉帧（写一个 demo example： 循环显示测试图序列，测 FPS）；CI 无 GPU 环境跳过。 + > `cargo run -p oak_bridge --example surface_bridge --features demo` + > （macOS + Metal，无 GPU 环境直接退出）。实测 1280x720 连续上屏 + > ~11-14fps（受 v1 同步 poll 限制，引擎侧可改为异步管线优化）。 - [ ] Windows/Linux 路径用 gpui_wgpu 的 `paint_surface(wgpu::Texture)` 直连，同 demo 验证。 + > 未做：wgpu 29 无 IOSurface 原生支持且 hal `Texture` 字段私有， + > Linux/Windows 的 surface 桥接留待后续（`SurfaceSource::Texture` + > 已在 gpui 侧就绪，只需在 `oak_bridge` 补对应平台模块）。 ## W4. 播放同步与检视器 glue