Use LiveKit's Rust SDK on Linux while continue using Swift SDK on Mac (#21550)

Similar to #20826 but keeps the Swift implementation. There were quite a
few changes in the `call` crate, and so that code now has two variants.

Closes #13714

Release Notes:

- Added preliminary Linux support for voice chat and viewing
screenshares.

---------

Co-authored-by: Kirill Bulatov <mail4score@gmail.com>
Co-authored-by: Kirill Bulatov <kirill@zed.dev>
Co-authored-by: Mikayla <mikayla@zed.dev>

crates/gpui/src/platform/linux.rs

@@ -20,3 +20,5 @@ pub(crate) use text_system::*;
 pub(crate) use wayland::*;
 #[cfg(feature = "x11")]
 pub(crate) use x11::*;
+
+pub(crate) type PlatformScreenCaptureFrame = ();

crates/gpui/src/platform/linux/platform.rs

@@ -35,8 +35,8 @@ use crate::{
     px, Action, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DisplayId,
     ForegroundExecutor, Keymap, Keystroke, LinuxDispatcher, Menu, MenuItem, Modifiers, OwnedMenu,
     PathPromptOptions, Pixels, Platform, PlatformDisplay, PlatformInputHandler, PlatformTextSystem,
-    PlatformWindow, Point, PromptLevel, Result, SemanticVersion, SharedString, Size, Task,
-    WindowAppearance, WindowOptions, WindowParams,
+    PlatformWindow, Point, PromptLevel, Result, ScreenCaptureSource, SemanticVersion, SharedString,
+    Size, Task, WindowAppearance, WindowOptions, WindowParams,
 };
 
 pub(crate) const SCROLL_LINES: f32 = 3.0;
@@ -242,6 +242,14 @@ impl<P: LinuxClient + 'static> Platform for P {
         self.displays()
     }
 
+    fn screen_capture_sources(
+        &self,
+    ) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
+        let (mut tx, rx) = oneshot::channel();
+        tx.send(Err(anyhow!("screen capture not implemented"))).ok();
+        rx
+    }
+
     fn active_window(&self) -> Option<AnyWindowHandle> {
         self.active_window()
     }

crates/gpui/src/platform/mac.rs

@@ -4,12 +4,14 @@ mod dispatcher;
 mod display;
 mod display_link;
 mod events;
+mod screen_capture;
 
 #[cfg(not(feature = "macos-blade"))]
 mod metal_atlas;
 #[cfg(not(feature = "macos-blade"))]
 pub mod metal_renderer;
 
+use media::core_video::CVImageBuffer;
 #[cfg(not(feature = "macos-blade"))]
 use metal_renderer as renderer;
 
@@ -49,6 +51,9 @@ pub(crate) use window::*;
 #[cfg(feature = "font-kit")]
 pub(crate) use text_system::*;
 
+/// A frame of video captured from a screen.
+pub(crate) type PlatformScreenCaptureFrame = CVImageBuffer;
+
 trait BoolExt {
     fn to_objc(self) -> BOOL;
 }

crates/gpui/src/platform/mac/platform.rs

@@ -1,14 +1,14 @@
 use super::{
     attributed_string::{NSAttributedString, NSMutableAttributedString},
     events::key_to_native,
-    BoolExt,
+    renderer, screen_capture, BoolExt,
 };
 use crate::{
     hash, Action, AnyWindowHandle, BackgroundExecutor, ClipboardEntry, ClipboardItem,
     ClipboardString, CursorStyle, ForegroundExecutor, Image, ImageFormat, Keymap, MacDispatcher,
     MacDisplay, MacWindow, Menu, MenuItem, PathPromptOptions, Platform, PlatformDisplay,
-    PlatformTextSystem, PlatformWindow, Result, SemanticVersion, Task, WindowAppearance,
-    WindowParams,
+    PlatformTextSystem, PlatformWindow, Result, ScreenCaptureSource, SemanticVersion, Task,
+    WindowAppearance, WindowParams,
 };
 use anyhow::anyhow;
 use block::ConcreteBlock;
@@ -58,8 +58,6 @@ use std::{
 };
 use strum::IntoEnumIterator;
 
-use super::renderer;
-
 #[allow(non_upper_case_globals)]
 const NSUTF8StringEncoding: NSUInteger = 4;
 
@@ -552,6 +550,12 @@ impl Platform for MacPlatform {
             .collect()
     }
 
+    fn screen_capture_sources(
+        &self,
+    ) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
+        screen_capture::get_sources()
+    }
+
     fn active_window(&self) -> Option<AnyWindowHandle> {
         MacWindow::active_window()
     }

crates/gpui/src/platform/mac/screen_capture.rs

@@ -0,0 +1,239 @@
+use crate::{
+    platform::{ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream},
+    px, size, Pixels, Size,
+};
+use anyhow::{anyhow, Result};
+use block::ConcreteBlock;
+use cocoa::{
+    base::{id, nil, YES},
+    foundation::NSArray,
+};
+use core_foundation::base::TCFType;
+use ctor::ctor;
+use futures::channel::oneshot;
+use media::core_media::{CMSampleBuffer, CMSampleBufferRef};
+use metal::NSInteger;
+use objc::{
+    class,
+    declare::ClassDecl,
+    msg_send,
+    runtime::{Class, Object, Sel},
+    sel, sel_impl,
+};
+use std::{cell::RefCell, ffi::c_void, mem, ptr, rc::Rc};
+
+#[derive(Clone)]
+pub struct MacScreenCaptureSource {
+    sc_display: id,
+}
+
+pub struct MacScreenCaptureStream {
+    sc_stream: id,
+    sc_stream_output: id,
+}
+
+#[link(name = "ScreenCaptureKit", kind = "framework")]
+extern "C" {}
+
+static mut DELEGATE_CLASS: *const Class = ptr::null();
+static mut OUTPUT_CLASS: *const Class = ptr::null();
+const FRAME_CALLBACK_IVAR: &str = "frame_callback";
+
+#[allow(non_upper_case_globals)]
+const SCStreamOutputTypeScreen: NSInteger = 0;
+
+impl ScreenCaptureSource for MacScreenCaptureSource {
+    fn resolution(&self) -> Result<Size<Pixels>> {
+        unsafe {
+            let width: i64 = msg_send![self.sc_display, width];
+            let height: i64 = msg_send![self.sc_display, height];
+            Ok(size(px(width as f32), px(height as f32)))
+        }
+    }
+
+    fn stream(
+        &self,
+        frame_callback: Box<dyn Fn(ScreenCaptureFrame)>,
+    ) -> oneshot::Receiver<Result<Box<dyn ScreenCaptureStream>>> {
+        unsafe {
+            let stream: id = msg_send![class!(SCStream), alloc];
+            let filter: id = msg_send![class!(SCContentFilter), alloc];
+            let configuration: id = msg_send![class!(SCStreamConfiguration), alloc];
+            let delegate: id = msg_send![DELEGATE_CLASS, alloc];
+            let output: id = msg_send![OUTPUT_CLASS, alloc];
+
+            let excluded_windows = NSArray::array(nil);
+            let filter: id = msg_send![filter, initWithDisplay:self.sc_display excludingWindows:excluded_windows];
+            let configuration: id = msg_send![configuration, init];
+            let delegate: id = msg_send![delegate, init];
+            let output: id = msg_send![output, init];
+
+            output.as_mut().unwrap().set_ivar(
+                FRAME_CALLBACK_IVAR,
+                Box::into_raw(Box::new(frame_callback)) as *mut c_void,
+            );
+
+            let stream: id = msg_send![stream, initWithFilter:filter configuration:configuration delegate:delegate];
+
+            let (mut tx, rx) = oneshot::channel();
+
+            let mut error: id = nil;
+            let _: () = msg_send![stream, addStreamOutput:output type:SCStreamOutputTypeScreen sampleHandlerQueue:0 error:&mut error as *mut id];
+            if error != nil {
+                let message: id = msg_send![error, localizedDescription];
+                tx.send(Err(anyhow!("failed to add stream  output {message:?}")))
+                    .ok();
+                return rx;
+            }
+
+            let tx = Rc::new(RefCell::new(Some(tx)));
+            let handler = ConcreteBlock::new({
+                move |error: id| {
+                    let result = if error == nil {
+                        let stream = MacScreenCaptureStream {
+                            sc_stream: stream,
+                            sc_stream_output: output,
+                        };
+                        Ok(Box::new(stream) as Box<dyn ScreenCaptureStream>)
+                    } else {
+                        let message: id = msg_send![error, localizedDescription];
+                        Err(anyhow!("failed to stop screen capture stream {message:?}"))
+                    };
+                    if let Some(tx) = tx.borrow_mut().take() {
+                        tx.send(result).ok();
+                    }
+                }
+            });
+            let handler = handler.copy();
+            let _: () = msg_send![stream, startCaptureWithCompletionHandler:handler];
+            rx
+        }
+    }
+}
+
+impl Drop for MacScreenCaptureSource {
+    fn drop(&mut self) {
+        unsafe {
+            let _: () = msg_send![self.sc_display, release];
+        }
+    }
+}
+
+impl ScreenCaptureStream for MacScreenCaptureStream {}
+
+impl Drop for MacScreenCaptureStream {
+    fn drop(&mut self) {
+        unsafe {
+            let mut error: id = nil;
+            let _: () = msg_send![self.sc_stream, removeStreamOutput:self.sc_stream_output type:SCStreamOutputTypeScreen error:&mut error as *mut _];
+            if error != nil {
+                let message: id = msg_send![error, localizedDescription];
+                log::error!("failed to add stream  output {message:?}");
+            }
+
+            let handler = ConcreteBlock::new(move |error: id| {
+                if error != nil {
+                    let message: id = msg_send![error, localizedDescription];
+                    log::error!("failed to stop screen capture stream {message:?}");
+                }
+            });
+            let block = handler.copy();
+            let _: () = msg_send![self.sc_stream, stopCaptureWithCompletionHandler:block];
+            let _: () = msg_send![self.sc_stream, release];
+            let _: () = msg_send![self.sc_stream_output, release];
+        }
+    }
+}
+
+pub(crate) fn get_sources() -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
+    unsafe {
+        let (mut tx, rx) = oneshot::channel();
+        let tx = Rc::new(RefCell::new(Some(tx)));
+
+        let block = ConcreteBlock::new(move |shareable_content: id, error: id| {
+            let Some(mut tx) = tx.borrow_mut().take() else {
+                return;
+            };
+            let result = if error == nil {
+                let displays: id = msg_send![shareable_content, displays];
+                let mut result = Vec::new();
+                for i in 0..displays.count() {
+                    let display = displays.objectAtIndex(i);
+                    let source = MacScreenCaptureSource {
+                        sc_display: msg_send![display, retain],
+                    };
+                    result.push(Box::new(source) as Box<dyn ScreenCaptureSource>);
+                }
+                Ok(result)
+            } else {
+                let msg: id = msg_send![error, localizedDescription];
+                Err(anyhow!("Failed to register: {:?}", msg))
+            };
+            tx.send(result).ok();
+        });
+        let block = block.copy();
+
+        let _: () = msg_send![
+            class!(SCShareableContent),
+            getShareableContentExcludingDesktopWindows:YES
+                                   onScreenWindowsOnly:YES
+                                     completionHandler:block];
+        rx
+    }
+}
+
+#[ctor]
+unsafe fn build_classes() {
+    let mut decl = ClassDecl::new("GPUIStreamDelegate", class!(NSObject)).unwrap();
+    decl.add_method(
+        sel!(outputVideoEffectDidStartForStream:),
+        output_video_effect_did_start_for_stream as extern "C" fn(&Object, Sel, id),
+    );
+    decl.add_method(
+        sel!(outputVideoEffectDidStopForStream:),
+        output_video_effect_did_stop_for_stream as extern "C" fn(&Object, Sel, id),
+    );
+    decl.add_method(
+        sel!(stream:didStopWithError:),
+        stream_did_stop_with_error as extern "C" fn(&Object, Sel, id, id),
+    );
+    DELEGATE_CLASS = decl.register();
+
+    let mut decl = ClassDecl::new("GPUIStreamOutput", class!(NSObject)).unwrap();
+    decl.add_method(
+        sel!(stream:didOutputSampleBuffer:ofType:),
+        stream_did_output_sample_buffer_of_type as extern "C" fn(&Object, Sel, id, id, NSInteger),
+    );
+    decl.add_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR);
+
+    OUTPUT_CLASS = decl.register();
+}
+
+extern "C" fn output_video_effect_did_start_for_stream(_this: &Object, _: Sel, _stream: id) {}
+
+extern "C" fn output_video_effect_did_stop_for_stream(_this: &Object, _: Sel, _stream: id) {}
+
+extern "C" fn stream_did_stop_with_error(_this: &Object, _: Sel, _stream: id, _error: id) {}
+
+extern "C" fn stream_did_output_sample_buffer_of_type(
+    this: &Object,
+    _: Sel,
+    _stream: id,
+    sample_buffer: id,
+    buffer_type: NSInteger,
+) {
+    if buffer_type != SCStreamOutputTypeScreen {
+        return;
+    }
+
+    unsafe {
+        let sample_buffer = sample_buffer as CMSampleBufferRef;
+        let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer);
+        if let Some(buffer) = sample_buffer.image_buffer() {
+            let callback: Box<Box<dyn Fn(ScreenCaptureFrame)>> =
+                Box::from_raw(*this.get_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR) as *mut _);
+            callback(ScreenCaptureFrame(buffer));
+            mem::forget(callback);
+        }
+    }
+}

crates/gpui/src/platform/test.rs

@@ -7,3 +7,5 @@ pub(crate) use dispatcher::*;
 pub(crate) use display::*;
 pub(crate) use platform::*;
 pub(crate) use window::*;
+
+pub use platform::TestScreenCaptureSource;

crates/gpui/src/platform/test/platform.rs

@@ -1,7 +1,7 @@
 use crate::{
-    AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, Keymap,
-    Platform, PlatformDisplay, PlatformTextSystem, Task, TestDisplay, TestWindow, WindowAppearance,
-    WindowParams,
+    px, size, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor,
+    Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, ScreenCaptureSource,
+    ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance, WindowParams,
 };
 use anyhow::Result;
 use collections::VecDeque;
@@ -31,6 +31,7 @@ pub(crate) struct TestPlatform {
     #[cfg(any(target_os = "linux", target_os = "freebsd"))]
     current_primary_item: Mutex<Option<ClipboardItem>>,
     pub(crate) prompts: RefCell<TestPrompts>,
+    screen_capture_sources: RefCell<Vec<TestScreenCaptureSource>>,
     pub opened_url: RefCell<Option<String>>,
     pub text_system: Arc<dyn PlatformTextSystem>,
     #[cfg(target_os = "windows")]
@@ -38,6 +39,31 @@ pub(crate) struct TestPlatform {
     weak: Weak<Self>,
 }
 
+#[derive(Clone)]
+/// A fake screen capture source, used for testing.
+pub struct TestScreenCaptureSource {}
+
+pub struct TestScreenCaptureStream {}
+
+impl ScreenCaptureSource for TestScreenCaptureSource {
+    fn resolution(&self) -> Result<crate::Size<crate::Pixels>> {
+        Ok(size(px(1.), px(1.)))
+    }
+
+    fn stream(
+        &self,
+        _frame_callback: Box<dyn Fn(ScreenCaptureFrame)>,
+    ) -> oneshot::Receiver<Result<Box<dyn ScreenCaptureStream>>> {
+        let (mut tx, rx) = oneshot::channel();
+        let stream = TestScreenCaptureStream {};
+        tx.send(Ok(Box::new(stream) as Box<dyn ScreenCaptureStream>))
+            .ok();
+        rx
+    }
+}
+
+impl ScreenCaptureStream for TestScreenCaptureStream {}
+
 #[derive(Default)]
 pub(crate) struct TestPrompts {
     multiple_choice: VecDeque<oneshot::Sender<usize>>,
@@ -72,6 +98,7 @@ impl TestPlatform {
             background_executor: executor,
             foreground_executor,
             prompts: Default::default(),
+            screen_capture_sources: Default::default(),
             active_cursor: Default::default(),
             active_display: Rc::new(TestDisplay::new()),
             active_window: Default::default(),
@@ -114,6 +141,10 @@ impl TestPlatform {
         !self.prompts.borrow().multiple_choice.is_empty()
     }
 
+    pub(crate) fn set_screen_capture_sources(&self, sources: Vec<TestScreenCaptureSource>) {
+        *self.screen_capture_sources.borrow_mut() = sources;
+    }
+
     pub(crate) fn prompt(&self, msg: &str, detail: Option<&str>) -> oneshot::Receiver<usize> {
         let (tx, rx) = oneshot::channel();
         self.background_executor()
@@ -202,6 +233,20 @@ impl Platform for TestPlatform {
         Some(self.active_display.clone())
     }
 
+    fn screen_capture_sources(
+        &self,
+    ) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
+        let (mut tx, rx) = oneshot::channel();
+        tx.send(Ok(self
+            .screen_capture_sources
+            .borrow()
+            .iter()
+            .map(|source| Box::new(source.clone()) as Box<dyn ScreenCaptureSource>)
+            .collect()))
+            .ok();
+        rx
+    }
+
     fn active_window(&self) -> Option<crate::AnyWindowHandle> {
         self.active_window
             .borrow()
@@ -330,6 +375,13 @@ impl Platform for TestPlatform {
     }
 }
 
+impl TestScreenCaptureSource {
+    /// Create a fake screen capture source, for testing.
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
 #[cfg(target_os = "windows")]
 impl Drop for TestPlatform {
     fn drop(&mut self) {

crates/gpui/src/platform/windows.rs

@@ -21,3 +21,5 @@ pub(crate) use window::*;
 pub(crate) use wrapper::*;
 
 pub(crate) use windows::Win32::Foundation::HWND;
+
+pub(crate) type PlatformScreenCaptureFrame = ();

crates/gpui/src/platform/windows/platform.rs

@@ -325,6 +325,14 @@ impl Platform for WindowsPlatform {
         WindowsDisplay::primary_monitor().map(|display| Rc::new(display) as Rc<dyn PlatformDisplay>)
     }
 
+    fn screen_capture_sources(
+        &self,
+    ) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
+        let (mut tx, rx) = oneshot::channel();
+        tx.send(Err(anyhow!("screen capture not implemented"))).ok();
+        rx
+    }
+
     fn active_window(&self) -> Option<AnyWindowHandle> {
         let active_window_hwnd = unsafe { GetActiveWindow() };
         self.try_get_windows_inner_from_hwnd(active_window_hwnd)