Introduce surface rendering

Co-Authored-By: Julia <julia@zed.dev>
2023-11-28 16:47:31 +01:00
parent eac4b2d076
commit 600b564bbf
7 changed files with 332 additions and 43 deletions
@@ -1,7 +1,7 @@
 use crate::{
    point, size, AtlasTextureId, AtlasTextureKind, AtlasTile, Bounds, ContentMask, DevicePixels,
    Hsla, MetalAtlas, MonochromeSprite, Path, PathId, PathVertex, PolychromeSprite, PrimitiveBatch,
-    Quad, ScaledPixels, Scene, Shadow, Size, Underline,
+    Quad, ScaledPixels, Scene, Shadow, Size, Surface, Underline,
 };
 use cocoa::{
    base::{NO, YES},
@@ -9,6 +9,9 @@ use cocoa::{
    quartzcore::AutoresizingMask,
 };
 use collections::HashMap;
+use core_foundation::base::TCFType;
+use foreign_types::ForeignType;
+use media::core_video::CVMetalTextureCache;
 use metal::{CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange};
 use objc::{self, msg_send, sel, sel_impl};
 use smallvec::SmallVec;
@@ -27,9 +30,11 @@ pub(crate) struct MetalRenderer {
    underlines_pipeline_state: metal::RenderPipelineState,
    monochrome_sprites_pipeline_state: metal::RenderPipelineState,
    polychrome_sprites_pipeline_state: metal::RenderPipelineState,
+    surfaces_pipeline_state: metal::RenderPipelineState,
    unit_vertices: metal::Buffer,
    instances: metal::Buffer,
    sprite_atlas: Arc<MetalAtlas>,
+    core_video_texture_cache: CVMetalTextureCache,
 }

 impl MetalRenderer {
@@ -143,6 +148,14 @@ impl MetalRenderer {
            "polychrome_sprite_fragment",
            MTLPixelFormat::BGRA8Unorm,
        );
+        let surfaces_pipeline_state = build_pipeline_state(
+            &device,
+            &library,
+            "surfaces",
+            "surface_vertex",
+            "surface_fragment",
+            MTLPixelFormat::BGRA8Unorm,
+        );

        let command_queue = device.new_command_queue();
        let sprite_atlas = Arc::new(MetalAtlas::new(device.clone()));
@@ -157,9 +170,11 @@ impl MetalRenderer {
            underlines_pipeline_state,
            monochrome_sprites_pipeline_state,
            polychrome_sprites_pipeline_state,
+            surfaces_pipeline_state,
            unit_vertices,
            instances,
            sprite_atlas,
+            core_video_texture_cache: CVMetalTextureCache::new(device.as_ptr()).unwrap(),
        }
    }

@@ -268,6 +283,14 @@ impl MetalRenderer {
                        command_encoder,
                    );
                }
+                PrimitiveBatch::Surfaces(surfaces) => {
+                    self.draw_surfaces(
+                        surfaces,
+                        &mut instance_offset,
+                        viewport_size,
+                        command_encoder,
+                    );
+                }
            }
        }

@@ -793,6 +816,102 @@ impl MetalRenderer {
        );
        *offset = next_offset;
    }
+
+    fn draw_surfaces(
+        &mut self,
+        surfaces: &[Surface],
+        offset: &mut usize,
+        viewport_size: Size<DevicePixels>,
+        command_encoder: &metal::RenderCommandEncoderRef,
+    ) {
+        command_encoder.set_render_pipeline_state(&self.surfaces_pipeline_state);
+        command_encoder.set_vertex_buffer(
+            SurfaceInputIndex::Vertices as u64,
+            Some(&self.unit_vertices),
+            0,
+        );
+        command_encoder.set_vertex_bytes(
+            SurfaceInputIndex::ViewportSize as u64,
+            mem::size_of_val(&viewport_size) as u64,
+            &viewport_size as *const Size<DevicePixels> as *const _,
+        );
+
+        for surface in surfaces {
+            let texture_size = size(
+                DevicePixels::from(surface.image_buffer.width() as i32),
+                DevicePixels::from(surface.image_buffer.height() as i32),
+            );
+
+            assert_eq!(
+                surface.image_buffer.pixel_format_type(),
+                media::core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
+            );
+
+            let y_texture = self
+                .core_video_texture_cache
+                .create_texture_from_image(
+                    surface.image_buffer.as_concrete_TypeRef(),
+                    ptr::null(),
+                    MTLPixelFormat::R8Unorm,
+                    surface.image_buffer.plane_width(0),
+                    surface.image_buffer.plane_height(0),
+                    0,
+                )
+                .unwrap();
+            let cb_cr_texture = self
+                .core_video_texture_cache
+                .create_texture_from_image(
+                    surface.image_buffer.as_concrete_TypeRef(),
+                    ptr::null(),
+                    MTLPixelFormat::RG8Unorm,
+                    surface.image_buffer.plane_width(1),
+                    surface.image_buffer.plane_height(1),
+                    1,
+                )
+                .unwrap();
+
+            align_offset(offset);
+            let next_offset = *offset + mem::size_of::<Surface>();
+            assert!(
+                next_offset <= INSTANCE_BUFFER_SIZE,
+                "instance buffer exhausted"
+            );
+
+            command_encoder.set_vertex_buffer(
+                SurfaceInputIndex::Surfaces as u64,
+                Some(&self.instances),
+                *offset as u64,
+            );
+            command_encoder.set_vertex_bytes(
+                SurfaceInputIndex::TextureSize as u64,
+                mem::size_of_val(&texture_size) as u64,
+                &texture_size as *const Size<DevicePixels> as *const _,
+            );
+            command_encoder.set_fragment_texture(
+                SurfaceInputIndex::YTexture as u64,
+                Some(y_texture.as_texture_ref()),
+            );
+            command_encoder.set_fragment_texture(
+                SurfaceInputIndex::CbCrTexture as u64,
+                Some(cb_cr_texture.as_texture_ref()),
+            );
+
+            unsafe {
+                let buffer_contents =
+                    (self.instances.contents() as *mut u8).add(*offset) as *mut SurfaceBounds;
+                ptr::write(
+                    buffer_contents,
+                    SurfaceBounds {
+                        bounds: surface.bounds,
+                        content_mask: surface.content_mask.clone(),
+                    },
+                );
+            }
+
+            command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6);
+            *offset = next_offset;
+        }
+    }
 }

 fn build_pipeline_state(
@@ -898,6 +1017,16 @@ enum SpriteInputIndex {
    AtlasTexture = 4,
 }

+#[repr(C)]
+enum SurfaceInputIndex {
+    Vertices = 0,
+    Surfaces = 1,
+    ViewportSize = 2,
+    TextureSize = 3,
+    YTexture = 4,
+    CbCrTexture = 5,
+}
+
 #[repr(C)]
 enum PathRasterizationInputIndex {
    Vertices = 0,
@@ -911,3 +1040,10 @@ pub struct PathSprite {
    pub color: Hsla,
    pub tile: AtlasTile,
 }
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[repr(C)]
+pub struct SurfaceBounds {
+    pub bounds: Bounds<ScaledPixels>,
+    pub content_mask: ContentMask<ScaledPixels>,
+}
@@ -469,6 +469,58 @@ fragment float4 path_sprite_fragment(
  return color;
 }

+struct SurfaceVertexOutput {
+  float4 position [[position]];
+  float2 texture_position;
+  float clip_distance [[clip_distance]][4];
+};
+
+struct SurfaceFragmentInput {
+  float4 position [[position]];
+  float2 texture_position;
+};
+
+vertex SurfaceVertexOutput surface_vertex(
+    uint unit_vertex_id [[vertex_id]], uint surface_id [[instance_id]],
+    constant float2 *unit_vertices [[buffer(SurfaceInputIndex_Vertices)]],
+    constant SurfaceBounds *surfaces [[buffer(SurfaceInputIndex_Surfaces)]],
+    constant Size_DevicePixels *viewport_size
+    [[buffer(SurfaceInputIndex_ViewportSize)]],
+    constant Size_DevicePixels *texture_size
+    [[buffer(SurfaceInputIndex_TextureSize)]]) {
+  float2 unit_vertex = unit_vertices[unit_vertex_id];
+  SurfaceBounds surface = surfaces[surface_id];
+  float4 device_position =
+      to_device_position(unit_vertex, surface.bounds, viewport_size);
+  float4 clip_distance = distance_from_clip_rect(unit_vertex, surface.bounds,
+                                                 surface.content_mask.bounds);
+  // We are going to copy the whole texture, so the texture position corresponds
+  // to the current vertex of the unit triangle.
+  float2 texture_position = unit_vertex;
+  return SurfaceVertexOutput{
+      device_position,
+      texture_position,
+      {clip_distance.x, clip_distance.y, clip_distance.z, clip_distance.w}};
+}
+
+fragment float4 surface_fragment(SurfaceFragmentInput input [[stage_in]],
+                                 texture2d<float> y_texture
+                                 [[texture(SurfaceInputIndex_YTexture)]],
+                                 texture2d<float> cb_cr_texture
+                                 [[texture(SurfaceInputIndex_CbCrTexture)]]) {
+  constexpr sampler texture_sampler(mag_filter::linear, min_filter::linear);
+  const float4x4 ycbcrToRGBTransform =
+      float4x4(float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f),
+               float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f),
+               float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f),
+               float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f));
+  float4 ycbcr = float4(
+      y_texture.sample(texture_sampler, input.texture_position).r,
+      cb_cr_texture.sample(texture_sampler, input.texture_position).rg, 1.0);
+
+  return ycbcrToRGBTransform * ycbcr;
+}
+
 float4 hsla_to_rgba(Hsla hsla) {
  float h = hsla.h * 6.0; // Now, it's an angle but scaled in [0, 6) range
  float s = hsla.s;