From ad0f75f3869f81ec60edb77769d5ca0be8eec85b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 25 Aug 2022 13:48:10 -0600 Subject: [PATCH 01/89] Add capture example to GPUI Added a linker arg to the GPUI build script. Not sure if we'll want to bake this into GPUI or do it via another crate, but this is convenient for exploration for now. --- crates/gpui/Cargo.toml | 6 +++ crates/gpui/build.rs | 3 ++ crates/gpui/examples/capture.rs | 78 +++++++++++++++++++++++++++++++++ crates/gpui/script/capture | 4 ++ 4 files changed, 91 insertions(+) create mode 100644 crates/gpui/examples/capture.rs create mode 100755 crates/gpui/script/capture diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 72c65a4c33..255e85a527 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -3,6 +3,7 @@ authors = ["Nathan Sobo "] edition = "2021" name = "gpui" version = "0.1.0" +description = "A GPU-accelerated UI framework" [lib] path = "src/gpui.rs" @@ -11,6 +12,11 @@ doctest = false [features] test-support = ["backtrace", "dhat", "env_logger", "collections/test-support"] +[package.metadata.bundle.example.capture] +name = "Capture" +identifier = "rs.gpui.examples.Capture" +description = "An example of screen capture" + [dependencies] collections = { path = "../collections" } gpui_macros = { path = "../gpui_macros" } diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 836d586c26..1c6cc11de0 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -9,6 +9,9 @@ fn main() { compile_context_predicate_parser(); compile_metal_shaders(); generate_shader_bindings(); + + // Support screen capture + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); } fn generate_dispatch_bindings() { diff --git a/crates/gpui/examples/capture.rs b/crates/gpui/examples/capture.rs new file mode 100644 index 0000000000..27bb843ab9 --- /dev/null +++ b/crates/gpui/examples/capture.rs @@ -0,0 +1,78 @@ +use std::{slice, str}; + +use block::ConcreteBlock; +use cocoa::{ + base::id, + foundation::{NSString, NSUInteger}, +}; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; +use log::LevelFilter; +use objc::{class, msg_send, sel, sel_impl}; +use simplelog::SimpleLogger; + +#[allow(non_upper_case_globals)] +const NSUTF8StringEncoding: NSUInteger = 4; + +actions!(capture, [Quit]); + +fn main() { + SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); + + gpui::App::new(()).unwrap().run(|cx| { + cx.platform().activate(true); + cx.add_global_action(quit); + + cx.add_bindings([Binding::new("cmd-q", Quit, None)]); + cx.set_menus(vec![Menu { + name: "Zed", + items: vec![MenuItem::Action { + name: "Quit", + action: Box::new(Quit), + }], + }]); + + unsafe { + let block = ConcreteBlock::new(move |content: id, error: id| { + println!("got response with shareable content"); + dbg!(content); + dbg!(error); + dbg!(string_from_objc(msg_send![error, localizedDescription])); + }); + + let _: id = msg_send![ + class!(SCShareableContent), + getShareableContentWithCompletionHandler: block + ]; + } + + // cx.add_window(Default::default(), |_| ScreenCaptureView); + }); +} + +struct ScreenCaptureView; + +impl gpui::Entity for ScreenCaptureView { + type Event = (); +} + +impl gpui::View for ScreenCaptureView { + fn ui_name() -> &'static str { + "View" + } + + fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { + Empty::new().boxed() + } +} + +pub unsafe fn string_from_objc(string: id) -> String { + let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; + let bytes = string.UTF8String() as *const u8; + str::from_utf8(slice::from_raw_parts(bytes, len)) + .unwrap() + .to_string() +} + +fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { + cx.platform().quit(); +} diff --git a/crates/gpui/script/capture b/crates/gpui/script/capture new file mode 100755 index 0000000000..753e5108e7 --- /dev/null +++ b/crates/gpui/script/capture @@ -0,0 +1,4 @@ +#!/bin/bash + +cargo bundle --example capture +open ../../target/debug/examples/bundle/osx/Capture.app From 7918bf39f5f0efc4163a564872a4ae52c2b29434 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 26 Aug 2022 15:10:26 -0600 Subject: [PATCH 02/89] Make capture example its own crate --- Cargo.lock | 15 +++++++++++++ crates/capture/Cargo.toml | 21 +++++++++++++++++++ crates/capture/build.rs | 3 +++ crates/capture/script/capture | 5 +++++ .../capture.rs => capture/src/main.rs} | 10 +++++---- crates/gpui/Cargo.toml | 5 ----- crates/gpui/build.rs | 3 --- crates/gpui/script/capture | 4 ---- 8 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 crates/capture/Cargo.toml create mode 100644 crates/capture/build.rs create mode 100755 crates/capture/script/capture rename crates/{gpui/examples/capture.rs => capture/src/main.rs} (88%) delete mode 100755 crates/gpui/script/capture diff --git a/Cargo.lock b/Cargo.lock index 634ef452a3..e12e49184d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -750,6 +750,21 @@ dependencies = [ "winx", ] +[[package]] +name = "capture" +version = "0.1.0" +dependencies = [ + "block", + "cocoa", + "core-foundation", + "core-graphics", + "foreign-types", + "gpui", + "log", + "objc", + "simplelog", +] + [[package]] name = "castaway" version = "0.1.2" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml new file mode 100644 index 0000000000..4dc975353f --- /dev/null +++ b/crates/capture/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "capture" +version = "0.1.0" +edition = "2021" +description = "An example of screen capture" + +[package.metadata.bundle] +name = "Capture" +identifier = "dev.zed.Capture" + +[dependencies] +gpui = { path = "../gpui" } + +block = "0.1" +cocoa = "0.24" +core-foundation = "0.9.3" +core-graphics = "0.22.3" +foreign-types = "0.3" +log = { version = "0.4.16", features = ["kv_unstable_serde"] } +objc = "0.2" +simplelog = "0.9" diff --git a/crates/capture/build.rs b/crates/capture/build.rs new file mode 100644 index 0000000000..5aa9032376 --- /dev/null +++ b/crates/capture/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); +} diff --git a/crates/capture/script/capture b/crates/capture/script/capture new file mode 100755 index 0000000000..0e6cf1f409 --- /dev/null +++ b/crates/capture/script/capture @@ -0,0 +1,5 @@ +#!/bin/bash + +cargo bundle +TTY=`tty` +open ../../target/debug/bundle/osx/Capture.app --stdout $TTY --stderr $TTY diff --git a/crates/gpui/examples/capture.rs b/crates/capture/src/main.rs similarity index 88% rename from crates/gpui/examples/capture.rs rename to crates/capture/src/main.rs index 27bb843ab9..6bad5bb7f3 100644 --- a/crates/gpui/examples/capture.rs +++ b/crates/capture/src/main.rs @@ -33,10 +33,12 @@ fn main() { unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { - println!("got response with shareable content"); - dbg!(content); - dbg!(error); - dbg!(string_from_objc(msg_send![error, localizedDescription])); + println!( + "got response with shareable content {:?} {:?} {:?}", + content, + error, + string_from_objc(msg_send![error, localizedDescription]), + ) }); let _: id = msg_send![ diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 255e85a527..08c32135fb 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -12,11 +12,6 @@ doctest = false [features] test-support = ["backtrace", "dhat", "env_logger", "collections/test-support"] -[package.metadata.bundle.example.capture] -name = "Capture" -identifier = "rs.gpui.examples.Capture" -description = "An example of screen capture" - [dependencies] collections = { path = "../collections" } gpui_macros = { path = "../gpui_macros" } diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 1c6cc11de0..836d586c26 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -9,9 +9,6 @@ fn main() { compile_context_predicate_parser(); compile_metal_shaders(); generate_shader_bindings(); - - // Support screen capture - println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); } fn generate_dispatch_bindings() { diff --git a/crates/gpui/script/capture b/crates/gpui/script/capture deleted file mode 100755 index 753e5108e7..0000000000 --- a/crates/gpui/script/capture +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -cargo bundle --example capture -open ../../target/debug/examples/bundle/osx/Capture.app From 45519cdd271b783193d0fe3449894d6ae5fe901b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 29 Aug 2022 18:00:51 +0200 Subject: [PATCH 03/89] WIP --- Cargo.lock | 1 + crates/capture/Cargo.toml | 3 ++ crates/capture/build.rs | 7 +++ crates/capture/src/dummy.m | 12 ++++++ crates/capture/src/main.rs | 75 ++++++++++++++++++++++++++++++--- crates/gpui/src/platform/mac.rs | 2 +- styles/package-lock.json | 1 + 7 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 crates/capture/src/dummy.m diff --git a/Cargo.lock b/Cargo.lock index e12e49184d..f001ab934a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -755,6 +755,7 @@ name = "capture" version = "0.1.0" dependencies = [ "block", + "cc", "cocoa", "core-foundation", "core-graphics", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 4dc975353f..2b483a25ca 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -19,3 +19,6 @@ foreign-types = "0.3" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" simplelog = "0.9" + +[build-dependencies] +cc = "1.0" \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 5aa9032376..33baf392bb 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,3 +1,10 @@ fn main() { println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); + println!("cargo:rustc-link-lib=framework=CoreMedia"); + println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); + + cc::Build::new() + .file("src/dummy.m") + .define("MACOSX_DEPLOYMENT_TARGET", "12.3") + .compile("dummy"); } diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m new file mode 100644 index 0000000000..8edb379692 --- /dev/null +++ b/crates/capture/src/dummy.m @@ -0,0 +1,12 @@ +#import + +@interface MyClass : NSObject +@end + +@implementation MyClass +- (void)stream:(SCStream *)stream + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + ofType:(SCStreamOutputType)type { +} + +@end \ No newline at end of file diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 6bad5bb7f3..e790d622ee 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,13 +1,14 @@ -use std::{slice, str}; +use std::{ffi::CStr, slice, str}; use block::ConcreteBlock; use cocoa::{ - base::id, - foundation::{NSString, NSUInteger}, + base::{id, nil}, + foundation::{NSArray, NSString, NSUInteger, NSInteger}, }; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; +use core_graphics::display::CGDirectDisplayID; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::dispatch_get_main_queue}; use log::LevelFilter; -use objc::{class, msg_send, sel, sel_impl}; +use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; #[allow(non_upper_case_globals)] @@ -32,13 +33,57 @@ fn main() { }]); unsafe { + + let block = ConcreteBlock::new(move |content: id, error: id| { println!( "got response with shareable content {:?} {:?} {:?}", content, error, string_from_objc(msg_send![error, localizedDescription]), - ) + ); + + let displays: id = msg_send![content, displays]; + if let Some(display) = (0..displays.count()) + .map(|ix| displays.objectAtIndex(ix)) + .next() + { + let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); + decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); + let capture_output_class = decl.register(); + + let output: id = msg_send![capture_output_class, alloc]; + let output: id = msg_send![output, init]; + + let excluded_windows: id = msg_send![class!(NSArray), array]; + let filter: id = msg_send![class!(SCContentFilter), alloc]; + let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; + let config: id = msg_send![class!(SCStreamConfiguration), alloc]; + // Configure the display content width and height. + let _: () = msg_send![config, setWidth: 800]; + let _: () = msg_send![config, setHeight: 600]; + let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; + let _: () = msg_send![config, setQueueDepth: 5]; + + let stream: id = msg_send![class!(SCStream), alloc]; + let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; + let error: id = nil; + let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; + println!("Added stream output... error? {}", string_from_objc(msg_send![error, localizedDescription])); + + + let start_capture_completion = ConcreteBlock::new(move |error: id| { + println!("Started capturing... error? {}", string_from_objc(msg_send![error, localizedDescription])); + println!("recovery suggestion {}", string_from_objc(msg_send![error, localizedRecoverySuggestion])); + println!("failure reason {}", string_from_objc(msg_send![error, localizedFailureReason])); + + + }); + + assert!(!stream.is_null()); + let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; + } }); let _: id = msg_send![ @@ -75,6 +120,24 @@ pub unsafe fn string_from_objc(string: id) -> String { .to_string() } +extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: NSInteger) { + println!("sample_output"); +} + + +extern "C" { + fn CMTimeMake(value: u64, timescale: i32) -> CMTime; +} + +#[repr(C)] +struct CMTime { + value: i64, + timescale: i32, + flags: u32, + epoch: i64, +} + + fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 8cf3f62874..1d2f5fc8ae 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -1,5 +1,5 @@ mod atlas; -mod dispatcher; +pub mod dispatcher; mod event; mod fonts; mod geometry; diff --git a/styles/package-lock.json b/styles/package-lock.json index 5499f1852c..582f1c8496 100644 --- a/styles/package-lock.json +++ b/styles/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "styles", "version": "1.0.0", "license": "ISC", "dependencies": { From d91f26d0163dcf7d5a8d40ba5709a94960822977 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 11:15:49 -0600 Subject: [PATCH 04/89] Make SCStreamOutput protocol accessible in Rust --- crates/capture/build.rs | 2 ++ crates/capture/src/main.rs | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 33baf392bb..abbc019943 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -2,9 +2,11 @@ fn main() { println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-link-lib=framework=CoreMedia"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); + println!("cargo:rustc-link-arg=-ObjC"); cc::Build::new() .file("src/dummy.m") .define("MACOSX_DEPLOYMENT_TARGET", "12.3") + .flag("-ObjC") .compile("dummy"); } diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index e790d622ee..7398134729 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -33,16 +33,7 @@ fn main() { }]); unsafe { - - let block = ConcreteBlock::new(move |content: id, error: id| { - println!( - "got response with shareable content {:?} {:?} {:?}", - content, - error, - string_from_objc(msg_send![error, localizedDescription]), - ); - let displays: id = msg_send![content, displays]; if let Some(display) = (0..displays.count()) .map(|ix| displays.objectAtIndex(ix)) From 497232ee59376aff79e961975b5fdbfc7c2f60a7 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 11:32:11 -0600 Subject: [PATCH 05/89] Get capture starting without error --- crates/capture/src/main.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 7398134729..bfa0d03602 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -34,11 +34,21 @@ fn main() { unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { + if !error.is_null() { + println!("ERROR {}", string_from_objc(msg_send![error, localizedDescription])); + return; + } + let displays: id = msg_send![content, displays]; + if let Some(display) = (0..displays.count()) .map(|ix| displays.objectAtIndex(ix)) .next() { + + let display_id: u32 = msg_send![display, displayID]; + println!("display id {:?}", display_id); + let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); @@ -51,6 +61,7 @@ fn main() { let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; let config: id = msg_send![class!(SCStreamConfiguration), alloc]; + let config: id = msg_send![config, init]; // Configure the display content width and height. let _: () = msg_send![config, setWidth: 800]; let _: () = msg_send![config, setHeight: 600]; @@ -65,9 +76,10 @@ fn main() { let start_capture_completion = ConcreteBlock::new(move |error: id| { - println!("Started capturing... error? {}", string_from_objc(msg_send![error, localizedDescription])); - println!("recovery suggestion {}", string_from_objc(msg_send![error, localizedRecoverySuggestion])); - println!("failure reason {}", string_from_objc(msg_send![error, localizedFailureReason])); + if !error.is_null() { + println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); + return; + } }); From 0df97dce028ac440adeb4748172698634a5dca53 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 11:40:30 -0600 Subject: [PATCH 06/89] WIP --- crates/capture/src/main.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index bfa0d03602..59d87b5225 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -72,8 +72,6 @@ fn main() { let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; let error: id = nil; let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; - println!("Added stream output... error? {}", string_from_objc(msg_send![error, localizedDescription])); - let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { @@ -81,7 +79,7 @@ fn main() { return; } - + println!("starting capture"); }); assert!(!stream.is_null()); From 30a3c0fb46e05d544c0f725dbf30dfa3b8bb12a4 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 29 Aug 2022 19:59:58 +0200 Subject: [PATCH 07/89] WIP --- crates/capture/build.rs | 3 +-- crates/capture/src/main.rs | 7 ++++--- crates/gpui/build.rs | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index abbc019943..f45dc922c6 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -6,7 +6,6 @@ fn main() { cc::Build::new() .file("src/dummy.m") - .define("MACOSX_DEPLOYMENT_TARGET", "12.3") - .flag("-ObjC") + .flag("-mmacosx-version-min=12.3") .compile("dummy"); } diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 59d87b5225..a77ac61342 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,4 +1,4 @@ -use std::{ffi::CStr, slice, str}; +use std::{ffi::CStr, slice, str, ptr}; use block::ConcreteBlock; use cocoa::{ @@ -6,7 +6,7 @@ use cocoa::{ foundation::{NSArray, NSString, NSUInteger, NSInteger}, }; use core_graphics::display::CGDirectDisplayID; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::dispatch_get_main_queue}; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::{dispatch_get_main_queue, dispatch_queue_create}}; use log::LevelFilter; use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; @@ -71,7 +71,8 @@ fn main() { let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; let error: id = nil; - let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; + let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); + let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: queue error: &error]; let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 836d586c26..967423d3f7 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -19,6 +19,7 @@ fn generate_dispatch_bindings() { .header("src/platform/mac/dispatch.h") .allowlist_var("_dispatch_main_q") .allowlist_function("dispatch_async_f") + .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() From 4440c9b18edada3afbf3a1dd353e8c339cba1cfd Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 16:07:33 -0600 Subject: [PATCH 08/89] Bind to capture's macOS C frameworks with bindgen --- Cargo.lock | 1 + crates/capture/Cargo.toml | 1 + crates/capture/build.rs | 23 ++++++++++++++++++++++- crates/capture/src/bindings.h | 2 ++ crates/capture/src/bindings.rs | 5 +++++ crates/capture/src/main.rs | 34 ++++++++++++++-------------------- 6 files changed, 45 insertions(+), 21 deletions(-) create mode 100644 crates/capture/src/bindings.h create mode 100644 crates/capture/src/bindings.rs diff --git a/Cargo.lock b/Cargo.lock index f001ab934a..7dff1a8bb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -754,6 +754,7 @@ dependencies = [ name = "capture" version = "0.1.0" dependencies = [ + "bindgen", "block", "cc", "cocoa", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 2b483a25ca..aade52f2fb 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -21,4 +21,5 @@ objc = "0.2" simplelog = "0.9" [build-dependencies] +bindgen = "0.59.2" cc = "1.0" \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index f45dc922c6..4f8ada8670 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,9 +1,30 @@ +use std::{env, path::PathBuf}; + fn main() { - println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-link-lib=framework=CoreMedia"); + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); + println!("cargo:rustc-link-lib=framework=System"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); println!("cargo:rustc-link-arg=-ObjC"); + let bindings = bindgen::Builder::default() + .header("src/bindings.h") + .clang_arg("-isysroot/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk") + .allowlist_function("CMTimeMake") + .allowlist_type("CMSampleBufferRef") + .allowlist_var("_dispatch_main_q") + .allowlist_function("dispatch_async_f") + .allowlist_function("dispatch_queue_create") + .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .layout_tests(false) + .generate() + .expect("unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("couldn't write dispatch bindings"); + cc::Build::new() .file("src/dummy.m") .flag("-mmacosx-version-min=12.3") diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h new file mode 100644 index 0000000000..0df70f8e11 --- /dev/null +++ b/crates/capture/src/bindings.h @@ -0,0 +1,2 @@ +#import +#import diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs new file mode 100644 index 0000000000..24de9f5855 --- /dev/null +++ b/crates/capture/src/bindings.rs @@ -0,0 +1,5 @@ +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + +pub fn dispatch_get_main_queue() -> dispatch_queue_t { + unsafe { &_dispatch_main_q as *const _ as dispatch_queue_t } +} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index a77ac61342..b41a623092 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,16 +1,19 @@ -use std::{ffi::CStr, slice, str, ptr}; +mod bindings; + +use std::{slice, str}; use block::ConcreteBlock; use cocoa::{ base::{id, nil}, foundation::{NSArray, NSString, NSUInteger, NSInteger}, }; -use core_graphics::display::CGDirectDisplayID; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem, mac::dispatcher::{dispatch_get_main_queue, dispatch_queue_create}}; +use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; +use crate::bindings::dispatch_get_main_queue; + #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -57,6 +60,10 @@ fn main() { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + let conforms: bool = msg_send![output, conformsToProtocol: Protocol::get("SCStreamOutput").unwrap()]; + dbg!(conforms); + assert!(conforms, "expect CaptureOutput instance to conform to SCStreamOutput protocol"); + let excluded_windows: id = msg_send![class!(NSArray), array]; let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; @@ -65,14 +72,15 @@ fn main() { // Configure the display content width and height. let _: () = msg_send![config, setWidth: 800]; let _: () = msg_send![config, setHeight: 600]; - let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; + let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 5]; let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; let error: id = nil; - let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); - let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: queue error: &error]; + // let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); + + let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { @@ -126,20 +134,6 @@ extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: println!("sample_output"); } - -extern "C" { - fn CMTimeMake(value: u64, timescale: i32) -> CMTime; -} - -#[repr(C)] -struct CMTime { - value: i64, - timescale: i32, - flags: u32, - epoch: i64, -} - - fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } From 99cb66dfb033275e9ccabd93e8d6764ed8dd5f0e Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 16:25:37 -0600 Subject: [PATCH 09/89] Programmatically locate the system SDK path for bindgen --- crates/capture/build.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 4f8ada8670..d75b110b9b 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,4 +1,4 @@ -use std::{env, path::PathBuf}; +use std::{env, path::PathBuf, process::Command}; fn main() { println!("cargo:rustc-link-lib=framework=CoreMedia"); @@ -7,9 +7,19 @@ fn main() { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); println!("cargo:rustc-link-arg=-ObjC"); + let sdk_path = String::from_utf8( + Command::new("xcrun") + .args(&["--sdk", "macosx", "--show-sdk-path"]) + .output() + .unwrap() + .stdout, + ) + .unwrap(); + let sdk_path = sdk_path.trim_end(); + let bindings = bindgen::Builder::default() .header("src/bindings.h") - .clang_arg("-isysroot/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk") + .clang_arg(format!("-isysroot{}", sdk_path)) .allowlist_function("CMTimeMake") .allowlist_type("CMSampleBufferRef") .allowlist_var("_dispatch_main_q") From 82ec2dc7ca8d668f832b076c3617eed1ce21a701 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 29 Aug 2022 19:08:07 -0600 Subject: [PATCH 10/89] Try to use the dummy capture handler, still not working --- crates/capture/build.rs | 3 +++ crates/capture/src/bindings.h | 1 + crates/capture/src/bindings.rs | 4 +++- crates/capture/src/dummy.m | 16 +++++++++++--- crates/capture/src/main.rs | 39 ++++++++++++++++++++++++---------- 5 files changed, 48 insertions(+), 15 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index d75b110b9b..6ab5a40e0f 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -20,8 +20,10 @@ fn main() { let bindings = bindgen::Builder::default() .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) + .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") .allowlist_type("CMSampleBufferRef") + .allowlist_type("SCStreamOutputType") .allowlist_var("_dispatch_main_q") .allowlist_function("dispatch_async_f") .allowlist_function("dispatch_queue_create") @@ -35,6 +37,7 @@ fn main() { .write_to_file(out_path.join("bindings.rs")) .expect("couldn't write dispatch bindings"); + println!("cargo:rerun-if-changed=src/dummy.m"); cc::Build::new() .file("src/dummy.m") .flag("-mmacosx-version-min=12.3") diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h index 0df70f8e11..45ac30446b 100644 --- a/crates/capture/src/bindings.h +++ b/crates/capture/src/bindings.h @@ -1,2 +1,3 @@ #import +#import #import diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index 24de9f5855..e845910603 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -1,5 +1,7 @@ +use objc::*; + include!(concat!(env!("OUT_DIR"), "/bindings.rs")); pub fn dispatch_get_main_queue() -> dispatch_queue_t { - unsafe { &_dispatch_main_q as *const _ as dispatch_queue_t } + unsafe { std::mem::transmute(&_dispatch_main_q) } } diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m index 8edb379692..9e3a343ce5 100644 --- a/crates/capture/src/dummy.m +++ b/crates/capture/src/dummy.m @@ -1,12 +1,22 @@ #import -@interface MyClass : NSObject +@interface MyClass : NSObject @end @implementation MyClass + - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - ofType:(SCStreamOutputType)type { + ofType:(SCStreamOutputType)type { + printf("dummy capture handler called"); } -@end \ No newline at end of file +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + printf("dummy did stop with error called"); +} + +int main() { + [[MyClass alloc] init]; +} + +@end diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index b41a623092..70a8ca62b2 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,6 +1,6 @@ mod bindings; -use std::{slice, str}; +use std::{slice, str, ptr::{self}}; use block::ConcreteBlock; use cocoa::{ @@ -12,7 +12,7 @@ use log::LevelFilter; use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; use simplelog::SimpleLogger; -use crate::bindings::dispatch_get_main_queue; +use crate::bindings::{dispatch_get_main_queue, dispatch_queue_create, NSObject, CMSampleBufferRef}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -52,18 +52,27 @@ fn main() { let display_id: u32 = msg_send![display, displayID]; println!("display id {:?}", display_id); - let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); - decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); - let capture_output_class = decl.register(); + // let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + // decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); + // decl.add_protocol(Protocol::get("SCStreamDelegate").unwrap()); + // decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); + // decl.add_method(sel!(stream:didStopWithError:), did_stop_with_error as extern "C" fn(&Object, Sel, id, id)); + // let capture_output_class = decl.register(); - let output: id = msg_send![capture_output_class, alloc]; + // let output: id = msg_send![capture_output_class, alloc]; + // let output: id = msg_send![output, init]; + + let output: id = msg_send![class!(MyClass), alloc]; let output: id = msg_send![output, init]; + // Do we conform to the protocol? let conforms: bool = msg_send![output, conformsToProtocol: Protocol::get("SCStreamOutput").unwrap()]; dbg!(conforms); assert!(conforms, "expect CaptureOutput instance to conform to SCStreamOutput protocol"); + // Confirm we can send the protocol message to the object + let _: () = msg_send![output, stream:NSObject(ptr::null_mut()) didOutputSampleBuffer:NSObject(ptr::null_mut()) ofType:0]; + let excluded_windows: id = msg_send![class!(NSArray), array]; let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; @@ -76,11 +85,15 @@ fn main() { let _: () = msg_send![config, setQueueDepth: 5]; let stream: id = msg_send![class!(SCStream), alloc]; - let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: nil]; + let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; let error: id = nil; - // let queue = dispatch_queue_create(ptr::null(), ptr::null_mut()); + let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); - let _: () = msg_send![stream, addStreamOutput: output type: 0 sampleHandlerQueue: dispatch_get_main_queue() error: &error]; + let _: () = msg_send![stream, + addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen + sampleHandlerQueue: queue + error: &error + ]; let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { @@ -130,8 +143,12 @@ pub unsafe fn string_from_objc(string: id) -> String { .to_string() } +extern "C" fn did_stop_with_error(this: &Object, _: Sel, stream: id, error: id) { + println!("did_stop_with_error"); +} + extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: NSInteger) { - println!("sample_output"); + println!("sample output"); } fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { From ef8a0dc175bf373a862467bd4af8008a0d5b7120 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 09:37:49 +0200 Subject: [PATCH 11/89] Fix bindgen warnings --- crates/capture/src/bindings.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index e845910603..afdef6d1da 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -1,3 +1,7 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); From 014246f56919c22b0eec55a5cd04aaaa317484ca Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 11:12:40 +0200 Subject: [PATCH 12/89] Provide all running applications to `SCContentFilter` to capture display --- crates/capture/build.rs | 4 +- crates/capture/src/bindings.rs | 4 - crates/capture/src/dummy.m | 15 ---- crates/capture/src/main.rs | 153 +++++++++++++++------------------ 4 files changed, 72 insertions(+), 104 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 6ab5a40e0f..51ff36f705 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -17,15 +17,13 @@ fn main() { .unwrap(); let sdk_path = sdk_path.trim_end(); + println!("cargo:rerun-if-changed=src/bindings.h"); let bindings = bindgen::Builder::default() .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") - .allowlist_type("CMSampleBufferRef") .allowlist_type("SCStreamOutputType") - .allowlist_var("_dispatch_main_q") - .allowlist_function("dispatch_async_f") .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index afdef6d1da..a111bede4d 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -5,7 +5,3 @@ use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); - -pub fn dispatch_get_main_queue() -> dispatch_queue_t { - unsafe { std::mem::transmute(&_dispatch_main_q) } -} diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m index 9e3a343ce5..0ae2bda6a7 100644 --- a/crates/capture/src/dummy.m +++ b/crates/capture/src/dummy.m @@ -4,19 +4,4 @@ @end @implementation MyClass - -- (void)stream:(SCStream *)stream - didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - ofType:(SCStreamOutputType)type { - printf("dummy capture handler called"); -} - -- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { - printf("dummy did stop with error called"); -} - -int main() { - [[MyClass alloc] init]; -} - @end diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 70a8ca62b2..c1c017c359 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,18 +1,22 @@ mod bindings; -use std::{slice, str, ptr::{self}}; - +use crate::bindings::{dispatch_queue_create, NSObject, SCStreamOutputType}; use block::ConcreteBlock; use cocoa::{ - base::{id, nil}, - foundation::{NSArray, NSString, NSUInteger, NSInteger}, + base::{id, nil, YES}, + foundation::{NSArray, NSBundle, NSString, NSUInteger}, }; use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; -use objc::{class, msg_send, sel, sel_impl, declare::ClassDecl, runtime::{Protocol, Object, Sel}}; +use objc::{ + class, + declare::ClassDecl, + msg_send, + runtime::{Object, Protocol, Sel}, + sel, sel_impl, +}; use simplelog::SimpleLogger; - -use crate::bindings::{dispatch_get_main_queue, dispatch_queue_create, NSObject, CMSampleBufferRef}; +use std::{ptr, slice, str}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -41,72 +45,51 @@ fn main() { println!("ERROR {}", string_from_objc(msg_send![error, localizedDescription])); return; } - + + let applications: id = msg_send![content, applications]; let displays: id = msg_send![content, displays]; - - if let Some(display) = (0..displays.count()) - .map(|ix| displays.objectAtIndex(ix)) - .next() - { - - let display_id: u32 = msg_send![display, displayID]; - println!("display id {:?}", display_id); - - // let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - // decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); - // decl.add_protocol(Protocol::get("SCStreamDelegate").unwrap()); - // decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, NSInteger)); - // decl.add_method(sel!(stream:didStopWithError:), did_stop_with_error as extern "C" fn(&Object, Sel, id, id)); - // let capture_output_class = decl.register(); - - // let output: id = msg_send![capture_output_class, alloc]; - // let output: id = msg_send![output, init]; - - let output: id = msg_send![class!(MyClass), alloc]; - let output: id = msg_send![output, init]; - - // Do we conform to the protocol? - let conforms: bool = msg_send![output, conformsToProtocol: Protocol::get("SCStreamOutput").unwrap()]; - dbg!(conforms); - assert!(conforms, "expect CaptureOutput instance to conform to SCStreamOutput protocol"); - - // Confirm we can send the protocol message to the object - let _: () = msg_send![output, stream:NSObject(ptr::null_mut()) didOutputSampleBuffer:NSObject(ptr::null_mut()) ofType:0]; - - let excluded_windows: id = msg_send![class!(NSArray), array]; - let filter: id = msg_send![class!(SCContentFilter), alloc]; - let filter: id = msg_send![filter, initWithDisplay: display excludingWindows: excluded_windows]; - let config: id = msg_send![class!(SCStreamConfiguration), alloc]; - let config: id = msg_send![config, init]; - // Configure the display content width and height. - let _: () = msg_send![config, setWidth: 800]; - let _: () = msg_send![config, setHeight: 600]; - let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; - let _: () = msg_send![config, setQueueDepth: 5]; - - let stream: id = msg_send![class!(SCStream), alloc]; - let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; - let error: id = nil; - let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); - - let _: () = msg_send![stream, - addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen - sampleHandlerQueue: queue - error: &error - ]; - - let start_capture_completion = ConcreteBlock::new(move |error: id| { - if !error.is_null() { - println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); - return; - } - - println!("starting capture"); - }); - - assert!(!stream.is_null()); - let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; - } + let display: id = displays.objectAtIndex(0); + + let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); + decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); + let capture_output_class = decl.register(); + + let output: id = msg_send![capture_output_class, alloc]; + let output: id = msg_send![output, init]; + + let filter: id = msg_send![class!(SCContentFilter), alloc]; + let filter: id = msg_send![filter, initWithDisplay: display includingApplications: applications exceptingWindows: nil]; + // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; + let config: id = msg_send![class!(SCStreamConfiguration), alloc]; + let config: id = msg_send![config, init]; + let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; + let _: () = msg_send![config, setQueueDepth: 6]; + let _: () = msg_send![config, setShowsCursor: YES]; + + let stream: id = msg_send![class!(SCStream), alloc]; + let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; + let error: id = nil; + let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); + + let _: () = msg_send![stream, + addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen + sampleHandlerQueue: queue + error: &error + ]; + + let start_capture_completion = ConcreteBlock::new(move |error: id| { + if !error.is_null() { + println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); + return; + } + + println!("starting capture"); + }); + + assert!(!stream.is_null()); + let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; + }); let _: id = msg_send![ @@ -136,18 +119,24 @@ impl gpui::View for ScreenCaptureView { } pub unsafe fn string_from_objc(string: id) -> String { - let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; - let bytes = string.UTF8String() as *const u8; - str::from_utf8(slice::from_raw_parts(bytes, len)) - .unwrap() - .to_string() + if string.is_null() { + Default::default() + } else { + let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; + let bytes = string.UTF8String() as *const u8; + str::from_utf8(slice::from_raw_parts(bytes, len)) + .unwrap() + .to_string() + } } -extern "C" fn did_stop_with_error(this: &Object, _: Sel, stream: id, error: id) { - println!("did_stop_with_error"); -} - -extern "C" fn sample_output(this: &Object, _: Sel, stream: id, buffer: id, kind: NSInteger) { +extern "C" fn sample_output( + _: &Object, + _: Sel, + _stream: id, + _buffer: id, + _kind: SCStreamOutputType, +) { println!("sample output"); } From 1611635e5fd957809790f25dfd5a0426490d0aaf Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 15:49:07 +0200 Subject: [PATCH 13/89] Capture display frames and access underlying IOSurface --- crates/capture/build.rs | 2 + crates/capture/src/main.rs | 156 ++++++++++++++++++++++++++++++++++++- 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 51ff36f705..550d1064ae 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -24,6 +24,8 @@ fn main() { .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") .allowlist_type("SCStreamOutputType") + .allowlist_type("SCFrameStatus") + .allowlist_var("SCStreamFrameInfo.*") .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index c1c017c359..08a6cf03b2 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -4,8 +4,10 @@ use crate::bindings::{dispatch_queue_create, NSObject, SCStreamOutputType}; use block::ConcreteBlock; use cocoa::{ base::{id, nil, YES}, - foundation::{NSArray, NSBundle, NSString, NSUInteger}, + foundation::{NSArray, NSString, NSUInteger}, }; +use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; +use core_media::{CMSampleBuffer, CMSampleBufferRef}; use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; use objc::{ @@ -49,6 +51,8 @@ fn main() { let applications: id = msg_send![content, applications]; let displays: id = msg_send![content, displays]; let display: id = displays.objectAtIndex(0); + let display_width: usize = msg_send![display, width]; + let display_height: usize = msg_send![display, height]; let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); @@ -63,6 +67,8 @@ fn main() { // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; let config: id = msg_send![class!(SCStreamConfiguration), alloc]; let config: id = msg_send![config, init]; + let _: () = msg_send![config, setWidth: display_width]; + let _: () = msg_send![config, setHeight: display_height]; let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; @@ -134,12 +140,156 @@ extern "C" fn sample_output( _: &Object, _: Sel, _stream: id, - _buffer: id, + buffer: id, _kind: SCStreamOutputType, ) { - println!("sample output"); + let buffer = unsafe { CMSampleBuffer::wrap_under_get_rule(buffer as CMSampleBufferRef) }; + let attachments = buffer.attachments(); + let attachments = attachments.first().expect("no attachments for sample"); + + unsafe { + let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; + let status = core_foundation::number::CFNumber::wrap_under_get_rule( + *attachments.get(string) as CFNumberRef, + ) + .to_i64() + .expect("invalid frame info status"); + + if status != bindings::SCFrameStatus_SCFrameStatusComplete { + println!("received incomplete frame"); + return; + } + } + + let image_buffer = buffer.image_buffer(); + dbg!(image_buffer.width(), image_buffer.height()); + let io_surface = image_buffer.io_surface(); } fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } + +mod core_media { + #![allow(non_snake_case)] + + use crate::core_video::{CVImageBuffer, CVImageBufferRef}; + use core_foundation::{ + array::{CFArray, CFArrayRef}, + base::{CFTypeID, TCFType}, + declare_TCFType, + dictionary::CFDictionary, + impl_CFTypeDescription, impl_TCFType, + string::CFString, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __CMSampleBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CMSampleBufferRef = *const __CMSampleBuffer; + + declare_TCFType!(CMSampleBuffer, CMSampleBufferRef); + impl_TCFType!(CMSampleBuffer, CMSampleBufferRef, CMSampleBufferGetTypeID); + impl_CFTypeDescription!(CMSampleBuffer); + + impl CMSampleBuffer { + pub fn attachments(&self) -> Vec> { + unsafe { + let attachments = + CMSampleBufferGetSampleAttachmentsArray(self.as_concrete_TypeRef(), true); + CFArray::::wrap_under_get_rule(attachments) + .into_iter() + .map(|attachments| { + CFDictionary::wrap_under_get_rule(attachments.as_concrete_TypeRef()) + }) + .collect() + } + } + + pub fn image_buffer(&self) -> CVImageBuffer { + unsafe { + CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( + self.as_concrete_TypeRef(), + )) + } + } + } + + extern "C" { + fn CMSampleBufferGetTypeID() -> CFTypeID; + fn CMSampleBufferGetSampleAttachmentsArray( + buffer: CMSampleBufferRef, + create_if_necessary: bool, + ) -> CFArrayRef; + fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; + } +} + +mod core_video { + #![allow(non_snake_case)] + + use crate::io_surface::{IOSurface, IOSurfaceRef}; + use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __CVImageBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CVImageBufferRef = *const __CVImageBuffer; + + declare_TCFType!(CVImageBuffer, CVImageBufferRef); + impl_TCFType!(CVImageBuffer, CVImageBufferRef, CVImageBufferGetTypeID); + impl_CFTypeDescription!(CVImageBuffer); + + impl CVImageBuffer { + pub fn io_surface(&self) -> IOSurface { + unsafe { + IOSurface::wrap_under_get_rule(CVPixelBufferGetIOSurface( + self.as_concrete_TypeRef(), + )) + } + } + + pub fn width(&self) -> usize { + unsafe { CVPixelBufferGetWidth(self.as_concrete_TypeRef()) } + } + + pub fn height(&self) -> usize { + unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } + } + } + + extern "C" { + fn CVImageBufferGetTypeID() -> CFTypeID; + fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; + fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + } +} + +mod io_surface { + #![allow(non_snake_case)] + + use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __IOSurface(c_void); + // The ref type must be a pointer to the underlying struct. + pub type IOSurfaceRef = *const __IOSurface; + + declare_TCFType!(IOSurface, IOSurfaceRef); + impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); + impl_CFTypeDescription!(IOSurface); + + extern "C" { + fn IOSurfaceGetTypeID() -> CFTypeID; + } +} From d473b52f5abeea170523763027524f1789c66afc Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 16:27:33 +0200 Subject: [PATCH 14/89] Remove `src/dummy.m` Co-Authored-By: Nathan Sobo --- crates/capture/build.rs | 7 ------- crates/capture/src/dummy.m | 7 ------- crates/capture/src/main.rs | 3 +-- 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 crates/capture/src/dummy.m diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 550d1064ae..a375d48f48 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -5,7 +5,6 @@ fn main() { println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-link-lib=framework=System"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); - println!("cargo:rustc-link-arg=-ObjC"); let sdk_path = String::from_utf8( Command::new("xcrun") @@ -36,10 +35,4 @@ fn main() { bindings .write_to_file(out_path.join("bindings.rs")) .expect("couldn't write dispatch bindings"); - - println!("cargo:rerun-if-changed=src/dummy.m"); - cc::Build::new() - .file("src/dummy.m") - .flag("-mmacosx-version-min=12.3") - .compile("dummy"); } diff --git a/crates/capture/src/dummy.m b/crates/capture/src/dummy.m deleted file mode 100644 index 0ae2bda6a7..0000000000 --- a/crates/capture/src/dummy.m +++ /dev/null @@ -1,7 +0,0 @@ -#import - -@interface MyClass : NSObject -@end - -@implementation MyClass -@end diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 08a6cf03b2..ac785a8d85 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -14,7 +14,7 @@ use objc::{ class, declare::ClassDecl, msg_send, - runtime::{Object, Protocol, Sel}, + runtime::{Object, Sel}, sel, sel_impl, }; use simplelog::SimpleLogger; @@ -55,7 +55,6 @@ fn main() { let display_height: usize = msg_send![display, height]; let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - decl.add_protocol(Protocol::get("SCStreamOutput").unwrap()); decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); let capture_output_class = decl.register(); From e12eaf8c58a0e0a9be8e6dfc1dc1b8a25a5eb573 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 18:17:54 +0200 Subject: [PATCH 15/89] Start on `Scene::push_surface` Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform/mac.rs | 1 + crates/gpui/src/platform/mac/renderer.rs | 4 ++++ crates/gpui/src/scene.rs | 14 +++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 1d2f5fc8ae..4ff73fcff2 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -13,6 +13,7 @@ use cocoa::base::{BOOL, NO, YES}; pub use dispatcher::Dispatcher; pub use fonts::FontSystem; use platform::{MacForegroundPlatform, MacPlatform}; +pub use renderer::Surface; use std::{rc::Rc, sync::Arc}; use window::Window; diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 8fdbda15e8..711d9e48b5 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -37,6 +37,10 @@ struct PathSprite { shader_data: shaders::GPUISprite, } +pub struct Surface { + pub bounds: RectF, +} + impl Renderer { pub fn new( device: metal::Device, diff --git a/crates/gpui/src/scene.rs b/crates/gpui/src/scene.rs index 086af5f64d..00b8d3c88b 100644 --- a/crates/gpui/src/scene.rs +++ b/crates/gpui/src/scene.rs @@ -10,7 +10,7 @@ use crate::{ fonts::{FontId, GlyphId}, geometry::{rect::RectF, vector::Vector2F}, json::ToJson, - platform::CursorStyle, + platform::{current::Surface, CursorStyle}, ImageData, }; pub use mouse_region::*; @@ -34,6 +34,7 @@ pub struct Layer { quads: Vec, underlines: Vec, images: Vec, + surfaces: Vec, shadows: Vec, glyphs: Vec, image_glyphs: Vec, @@ -249,6 +250,10 @@ impl Scene { self.active_layer().push_image(image) } + pub fn push_surface(&mut self, surface: Surface) { + self.active_layer().push_surface(surface) + } + pub fn push_underline(&mut self, underline: Underline) { self.active_layer().push_underline(underline) } @@ -329,6 +334,7 @@ impl Layer { quads: Default::default(), underlines: Default::default(), images: Default::default(), + surfaces: Default::default(), shadows: Default::default(), image_glyphs: Default::default(), glyphs: Default::default(), @@ -391,6 +397,12 @@ impl Layer { } } + fn push_surface(&mut self, surface: Surface) { + if can_draw(surface.bounds) { + self.surfaces.push(surface); + } + } + pub fn images(&self) -> &[Image] { self.images.as_slice() } From c4110edb7893601abdaa8f5f849df40c8064827a Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 19:05:33 +0200 Subject: [PATCH 16/89] Extract `io_surface` crate and invoke custom callback on frame sample Co-Authored-By: Nathan Sobo --- Cargo.lock | 11 +++++ crates/capture/Cargo.toml | 1 + crates/capture/build.rs | 3 +- crates/capture/src/bindings.rs | 5 ++ crates/capture/src/main.rs | 59 ++++++++++++------------ crates/gpui/Cargo.toml | 1 + crates/gpui/build.rs | 1 - crates/gpui/src/platform/mac.rs | 2 +- crates/gpui/src/platform/mac/renderer.rs | 1 + crates/io_surface/Cargo.toml | 13 ++++++ crates/io_surface/src/io_surface.rs | 21 +++++++++ 11 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 crates/io_surface/Cargo.toml create mode 100644 crates/io_surface/src/io_surface.rs diff --git a/Cargo.lock b/Cargo.lock index 7dff1a8bb8..4f6f6fe839 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -762,6 +762,7 @@ dependencies = [ "core-graphics", "foreign-types", "gpui", + "io_surface", "log", "objc", "simplelog", @@ -2228,6 +2229,7 @@ dependencies = [ "futures", "gpui_macros", "image", + "io_surface", "lazy_static", "log", "metal", @@ -2596,6 +2598,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "io_surface" +version = "0.1.0" +dependencies = [ + "block", + "core-foundation", + "objc", +] + [[package]] name = "iovec" version = "0.1.4" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index aade52f2fb..ebb35d8ef2 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -10,6 +10,7 @@ identifier = "dev.zed.Capture" [dependencies] gpui = { path = "../gpui" } +io_surface = { path = "../io_surface" } block = "0.1" cocoa = "0.24" diff --git a/crates/capture/build.rs b/crates/capture/build.rs index a375d48f48..8e95dca2ff 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -24,8 +24,9 @@ fn main() { .allowlist_function("CMTimeMake") .allowlist_type("SCStreamOutputType") .allowlist_type("SCFrameStatus") + .allowlist_type("dispatch_queue_t") .allowlist_var("SCStreamFrameInfo.*") - .allowlist_function("dispatch_queue_create") + .allowlist_var("_dispatch_main_q") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index a111bede4d..68487fce32 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -1,7 +1,12 @@ #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] #![allow(non_snake_case)] +#![allow(unused)] use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + +pub fn dispatch_get_main_queue() -> dispatch_queue_t { + unsafe { std::mem::transmute(&_dispatch_main_q) } +} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ac785a8d85..94561bde7e 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,6 +1,6 @@ mod bindings; -use crate::bindings::{dispatch_queue_create, NSObject, SCStreamOutputType}; +use crate::bindings::{dispatch_get_main_queue, SCStreamOutputType}; use block::ConcreteBlock; use cocoa::{ base::{id, nil, YES}, @@ -8,6 +8,7 @@ use cocoa::{ }; use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; use core_media::{CMSampleBuffer, CMSampleBufferRef}; +use gpui::elements::Canvas; use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; use log::LevelFilter; use objc::{ @@ -18,7 +19,7 @@ use objc::{ sel, sel_impl, }; use simplelog::SimpleLogger; -use std::{ptr, slice, str}; +use std::{ffi::c_void, slice, str}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -55,11 +56,17 @@ fn main() { let display_height: usize = msg_send![display, height]; let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); + decl.add_ivar::<*mut c_void>("callback"); decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); let capture_output_class = decl.register(); let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + let callback = Box::new(|buffer: CMSampleBufferRef| { + dbg!("HEY!"); + }) as Box; + let callback = Box::into_raw(Box::new(callback)); + (*output).set_ivar("callback", callback as *mut c_void); let filter: id = msg_send![class!(SCContentFilter), alloc]; let filter: id = msg_send![filter, initWithDisplay: display includingApplications: applications exceptingWindows: nil]; @@ -75,7 +82,7 @@ fn main() { let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; let error: id = nil; - let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut())); + let queue = dispatch_get_main_queue(); let _: () = msg_send![stream, addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen @@ -107,19 +114,29 @@ fn main() { }); } -struct ScreenCaptureView; +struct ScreenCaptureView { + surface: io_surface::IOSurface, +} impl gpui::Entity for ScreenCaptureView { type Event = (); } +// impl ScreenCaptureView { +// pub fn new() -> Self {} +// } + impl gpui::View for ScreenCaptureView { fn ui_name() -> &'static str { "View" } fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - Empty::new().boxed() + Canvas::new(|bounds, _, cx| { + + // cx.scene.push_surface(surface) + }) + .boxed() } } @@ -136,13 +153,18 @@ pub unsafe fn string_from_objc(string: id) -> String { } extern "C" fn sample_output( - _: &Object, + this: &Object, _: Sel, _stream: id, buffer: id, _kind: SCStreamOutputType, ) { let buffer = unsafe { CMSampleBuffer::wrap_under_get_rule(buffer as CMSampleBufferRef) }; + let callback = unsafe { *this.get_ivar::<*mut c_void>("callback") }; + let callback = callback as *mut Box; + let x = unsafe { callback.as_mut().unwrap() }; + (*x)(buffer.as_concrete_TypeRef()); + let attachments = buffer.attachments(); let attachments = attachments.first().expect("no attachments for sample"); @@ -228,11 +250,11 @@ mod core_media { mod core_video { #![allow(non_snake_case)] - use crate::io_surface::{IOSurface, IOSurfaceRef}; use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, }; + use io_surface::{IOSurface, IOSurfaceRef}; use std::ffi::c_void; #[repr(C)] @@ -269,26 +291,3 @@ mod core_video { fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; } } - -mod io_surface { - #![allow(non_snake_case)] - - use core_foundation::{ - base::{CFTypeID, TCFType}, - declare_TCFType, impl_CFTypeDescription, impl_TCFType, - }; - use std::ffi::c_void; - - #[repr(C)] - pub struct __IOSurface(c_void); - // The ref type must be a pointer to the underlying struct. - pub type IOSurfaceRef = *const __IOSurface; - - declare_TCFType!(IOSurface, IOSurfaceRef); - impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); - impl_CFTypeDescription!(IOSurface); - - extern "C" { - fn IOSurfaceGetTypeID() -> CFTypeID; - } -} diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 08c32135fb..6539c292fc 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -60,6 +60,7 @@ png = "0.16" simplelog = "0.9" [target.'cfg(target_os = "macos")'.dependencies] +io_surface = { path = "../io_surface" } anyhow = "1" block = "0.1" cocoa = "0.24" diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 967423d3f7..836d586c26 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -19,7 +19,6 @@ fn generate_dispatch_bindings() { .header("src/platform/mac/dispatch.h") .allowlist_var("_dispatch_main_q") .allowlist_function("dispatch_async_f") - .allowlist_function("dispatch_queue_create") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 4ff73fcff2..c1a4d45561 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -1,5 +1,5 @@ mod atlas; -pub mod dispatcher; +mod dispatcher; mod event; mod fonts; mod geometry; diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 711d9e48b5..25870c81fb 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -39,6 +39,7 @@ struct PathSprite { pub struct Surface { pub bounds: RectF, + pub native_surface: io_surface::IOSurface, } impl Renderer { diff --git a/crates/io_surface/Cargo.toml b/crates/io_surface/Cargo.toml new file mode 100644 index 0000000000..2d9324b6d8 --- /dev/null +++ b/crates/io_surface/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "io_surface" +version = "0.1.0" +edition = "2021" + +[lib] +path = "src/io_surface.rs" +doctest = false + +[dependencies] +block = "0.1" +core-foundation = "0.9.3" +objc = "0.2" \ No newline at end of file diff --git a/crates/io_surface/src/io_surface.rs b/crates/io_surface/src/io_surface.rs new file mode 100644 index 0000000000..13b1fd5ce1 --- /dev/null +++ b/crates/io_surface/src/io_surface.rs @@ -0,0 +1,21 @@ +#![allow(non_snake_case)] + +use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, +}; +use std::ffi::c_void; + +#[repr(C)] +pub struct __IOSurface(c_void); +// The ref type must be a pointer to the underlying struct. +pub type IOSurfaceRef = *const __IOSurface; + +declare_TCFType!(IOSurface, IOSurfaceRef); +impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); +impl_CFTypeDescription!(IOSurface); + +#[link(name = "IOSurface", kind = "framework")] +extern "C" { + fn IOSurfaceGetTypeID() -> CFTypeID; +} From d30e360664cc9b318aa045eaa16837754bc9b8bd Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 19:08:54 +0200 Subject: [PATCH 17/89] Retrieve IOSurface in Rust callback as opposed to doing so in delegate --- crates/capture/src/main.rs | 46 +++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 94561bde7e..ae02002e9e 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -63,7 +63,24 @@ fn main() { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; let callback = Box::new(|buffer: CMSampleBufferRef| { - dbg!("HEY!"); + let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); + let attachments = buffer.attachments(); + let attachments = attachments.first().expect("no attachments for sample"); + let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; + let status = core_foundation::number::CFNumber::wrap_under_get_rule( + *attachments.get(string) as CFNumberRef, + ) + .to_i64() + .expect("invalid frame info status"); + + if status != bindings::SCFrameStatus_SCFrameStatusComplete { + println!("received incomplete frame"); + return; + } + + let image_buffer = buffer.image_buffer(); + dbg!(image_buffer.width(), image_buffer.height()); + let io_surface = image_buffer.io_surface(); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -159,32 +176,11 @@ extern "C" fn sample_output( buffer: id, _kind: SCStreamOutputType, ) { - let buffer = unsafe { CMSampleBuffer::wrap_under_get_rule(buffer as CMSampleBufferRef) }; - let callback = unsafe { *this.get_ivar::<*mut c_void>("callback") }; - let callback = callback as *mut Box; - let x = unsafe { callback.as_mut().unwrap() }; - (*x)(buffer.as_concrete_TypeRef()); - - let attachments = buffer.attachments(); - let attachments = attachments.first().expect("no attachments for sample"); - unsafe { - let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; - let status = core_foundation::number::CFNumber::wrap_under_get_rule( - *attachments.get(string) as CFNumberRef, - ) - .to_i64() - .expect("invalid frame info status"); - - if status != bindings::SCFrameStatus_SCFrameStatusComplete { - println!("received incomplete frame"); - return; - } + let callback = *this.get_ivar::<*mut c_void>("callback"); + let callback = &mut *(callback as *mut Box); + (*callback)(buffer as CMSampleBufferRef); } - - let image_buffer = buffer.image_buffer(); - dbg!(image_buffer.width(), image_buffer.height()); - let io_surface = image_buffer.io_surface(); } fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { From 71d9a880d6ef80b5128dc0d594b592e7c76c3839 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 30 Aug 2022 19:28:17 +0200 Subject: [PATCH 18/89] WIP: Start pushing native surface to Scene This is segfaulting for some reason, so that's the next step to figure out. --- Cargo.lock | 3 ++ crates/capture/Cargo.toml | 3 ++ crates/capture/src/main.rs | 98 ++++++++++++++++++++++++++++---------- 3 files changed, 79 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f6f6fe839..aa030b9575 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,10 +761,13 @@ dependencies = [ "core-foundation", "core-graphics", "foreign-types", + "futures", "gpui", "io_surface", "log", "objc", + "parking_lot 0.11.2", + "postage", "simplelog", ] diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index ebb35d8ef2..8c6e901bbd 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -17,8 +17,11 @@ cocoa = "0.24" core-foundation = "0.9.3" core-graphics = "0.22.3" foreign-types = "0.3" +futures = "0.3" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" +parking_lot = "0.11.1" +postage = { version = "0.4.1", features = ["futures-traits"] } simplelog = "0.9" [build-dependencies] diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ae02002e9e..1b2ce6f8e8 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -8,8 +8,15 @@ use cocoa::{ }; use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; use core_media::{CMSampleBuffer, CMSampleBufferRef}; -use gpui::elements::Canvas; -use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem}; +use futures::StreamExt; +use gpui::{ + actions, + elements::{Canvas, *}, + keymap::Binding, + platform::current::Surface, + Menu, MenuItem, ViewContext, +}; +use io_surface::IOSurface; use log::LevelFilter; use objc::{ class, @@ -18,8 +25,9 @@ use objc::{ runtime::{Object, Sel}, sel, sel_impl, }; +use parking_lot::Mutex; use simplelog::SimpleLogger; -use std::{ffi::c_void, slice, str}; +use std::{cell::RefCell, ffi::c_void, slice, str}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -42,10 +50,29 @@ fn main() { }], }]); + cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); + }); +} + +struct ScreenCaptureView { + surface: Option, +} + +impl gpui::Entity for ScreenCaptureView { + type Event = (); +} + +impl ScreenCaptureView { + pub fn new(cx: &mut ViewContext) -> Self { + let (surface_tx, mut surface_rx) = postage::watch::channel::>(); + let surface_tx = RefCell::new(surface_tx); unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { if !error.is_null() { - println!("ERROR {}", string_from_objc(msg_send![error, localizedDescription])); + println!( + "ERROR {}", + string_from_objc(msg_send![error, localizedDescription]) + ); return; } @@ -57,11 +84,16 @@ fn main() { let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_ivar::<*mut c_void>("callback"); - decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType)); + decl.add_method( + sel!(stream:didOutputSampleBuffer:ofType:), + sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType), + ); let capture_output_class = decl.register(); let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + + // TODO: we could probably move this into a background queue. let callback = Box::new(|buffer: CMSampleBufferRef| { let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); let attachments = buffer.attachments(); @@ -79,8 +111,11 @@ fn main() { } let image_buffer = buffer.image_buffer(); - dbg!(image_buffer.width(), image_buffer.height()); let io_surface = image_buffer.io_surface(); + println!("before segfault"); + *surface_tx.borrow_mut().borrow_mut() = Some(io_surface); + + println!("!!!!!!!!!!!!!!!!!"); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -109,7 +144,10 @@ fn main() { let start_capture_completion = ConcreteBlock::new(move |error: id| { if !error.is_null() { - println!("error starting capture... error? {}", string_from_objc(msg_send![error, localizedDescription])); + println!( + "error starting capture... error? {}", + string_from_objc(msg_send![error, localizedDescription]) + ); return; } @@ -117,8 +155,10 @@ fn main() { }); assert!(!stream.is_null()); - let _: () = msg_send![stream, startCaptureWithCompletionHandler: start_capture_completion]; - + let _: () = msg_send![ + stream, + startCaptureWithCompletionHandler: start_capture_completion + ]; }); let _: id = msg_send![ @@ -127,31 +167,39 @@ fn main() { ]; } - // cx.add_window(Default::default(), |_| ScreenCaptureView); - }); -} + cx.spawn_weak(|this, mut cx| async move { + while let Some(surface) = surface_rx.next().await { + if let Some(this) = this.upgrade(&cx) { + this.update(&mut cx, |this, cx| { + this.surface = surface; + cx.notify(); + }) + } else { + break; + } + } + }) + .detach(); -struct ScreenCaptureView { - surface: io_surface::IOSurface, + Self { surface: None } + } } -impl gpui::Entity for ScreenCaptureView { - type Event = (); -} - -// impl ScreenCaptureView { -// pub fn new() -> Self {} -// } - impl gpui::View for ScreenCaptureView { fn ui_name() -> &'static str { "View" } fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - Canvas::new(|bounds, _, cx| { - - // cx.scene.push_surface(surface) + let surface = self.surface.clone(); + Canvas::new(move |bounds, _, cx| { + if let Some(native_surface) = surface.clone() { + dbg!("!!!!"); + cx.scene.push_surface(Surface { + bounds, + native_surface, + }); + } }) .boxed() } From db88ee2f4ce8fd30fe955141ea3a29620064582d Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 13:15:38 -0600 Subject: [PATCH 19/89] Handle captured frames in the background --- crates/capture/build.rs | 3 +-- crates/capture/src/bindings.rs | 4 ---- crates/capture/src/main.rs | 23 ++++++++++++----------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 8e95dca2ff..482e6487ce 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -22,11 +22,10 @@ fn main() { .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") .allowlist_function("CMTimeMake") + .allowlist_function("dispatch_queue_create") .allowlist_type("SCStreamOutputType") .allowlist_type("SCFrameStatus") - .allowlist_type("dispatch_queue_t") .allowlist_var("SCStreamFrameInfo.*") - .allowlist_var("_dispatch_main_q") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs index 68487fce32..a1c0b0da3e 100644 --- a/crates/capture/src/bindings.rs +++ b/crates/capture/src/bindings.rs @@ -6,7 +6,3 @@ use objc::*; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); - -pub fn dispatch_get_main_queue() -> dispatch_queue_t { - unsafe { std::mem::transmute(&_dispatch_main_q) } -} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 1b2ce6f8e8..e83135fc64 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,6 +1,6 @@ mod bindings; -use crate::bindings::{dispatch_get_main_queue, SCStreamOutputType}; +use crate::bindings::SCStreamOutputType; use block::ConcreteBlock; use cocoa::{ base::{id, nil, YES}, @@ -27,7 +27,7 @@ use objc::{ }; use parking_lot::Mutex; use simplelog::SimpleLogger; -use std::{cell::RefCell, ffi::c_void, slice, str}; +use std::{ffi::c_void, ptr, slice, str, sync::Arc}; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -65,7 +65,8 @@ impl gpui::Entity for ScreenCaptureView { impl ScreenCaptureView { pub fn new(cx: &mut ViewContext) -> Self { let (surface_tx, mut surface_rx) = postage::watch::channel::>(); - let surface_tx = RefCell::new(surface_tx); + let surface_tx = Arc::new(Mutex::new(surface_tx)); + unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { if !error.is_null() { @@ -92,9 +93,9 @@ impl ScreenCaptureView { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; + let surface_tx = surface_tx.clone(); - // TODO: we could probably move this into a background queue. - let callback = Box::new(|buffer: CMSampleBufferRef| { + let callback = Box::new(move |buffer: CMSampleBufferRef| { let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); let attachments = buffer.attachments(); let attachments = attachments.first().expect("no attachments for sample"); @@ -112,10 +113,7 @@ impl ScreenCaptureView { let image_buffer = buffer.image_buffer(); let io_surface = image_buffer.io_surface(); - println!("before segfault"); - *surface_tx.borrow_mut().borrow_mut() = Some(io_surface); - - println!("!!!!!!!!!!!!!!!!!"); + *surface_tx.lock().borrow_mut() = Some(io_surface); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -134,7 +132,10 @@ impl ScreenCaptureView { let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; let error: id = nil; - let queue = dispatch_get_main_queue(); + let queue = bindings::dispatch_queue_create( + ptr::null(), + bindings::NSObject(ptr::null_mut()), + ); let _: () = msg_send![stream, addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen @@ -172,6 +173,7 @@ impl ScreenCaptureView { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { this.surface = surface; + println!("NEW SURFACE!"); cx.notify(); }) } else { @@ -194,7 +196,6 @@ impl gpui::View for ScreenCaptureView { let surface = self.surface.clone(); Canvas::new(move |bounds, _, cx| { if let Some(native_surface) = surface.clone() { - dbg!("!!!!"); cx.scene.push_surface(Surface { bounds, native_surface, From 0430bbf7d95dee8605eac1ac31995f4724ca8c41 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 13:28:40 -0600 Subject: [PATCH 20/89] WIP: Start on Renderer::render_surfaces but really it's nothing --- crates/gpui/src/platform/mac/renderer.rs | 73 ++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 25870c81fb..41883a2f73 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -773,6 +773,79 @@ impl Renderer { } } + fn render_surfaces( + &mut self, + surfaces: &[Surface], + scale_factor: f32, + offset: &mut usize, + drawable_size: Vector2F, + command_encoder: &metal::RenderCommandEncoderRef, + ) { + if surfaces.is_empty() { + return; + } + + for surface in surfaces { + let origin = surface.bounds.origin() * scale_factor; + let target_size = surface.bounds.size() * scale_factor; + // let corner_radius = surface.corner_radius * scale_factor; + // let border_width = surface.border.width * scale_factor; + // let (alloc_id, atlas_bounds) = self.image_cache.render(&surface.native_surface); + } + + // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); + // command_encoder.set_vertex_buffer( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, + // Some(&self.unit_vertices), + // 0, + // ); + // command_encoder.set_vertex_bytes( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, + // mem::size_of::() as u64, + // [drawable_size.to_float2()].as_ptr() as *const c_void, + // ); + + // for (atlas_id, images) in images_by_atlas { + // align_offset(offset); + // let next_offset = *offset + images.len() * mem::size_of::(); + // assert!( + // next_offset <= INSTANCE_BUFFER_SIZE, + // "instance buffer exhausted" + // ); + + // let texture = self.image_cache.atlas_texture(atlas_id).unwrap(); + // command_encoder.set_vertex_buffer( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, + // Some(&self.instances), + // *offset as u64, + // ); + // command_encoder.set_vertex_bytes( + // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, + // mem::size_of::() as u64, + // [vec2i(texture.width() as i32, texture.height() as i32).to_float2()].as_ptr() + // as *const c_void, + // ); + // command_encoder.set_fragment_texture( + // shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, + // Some(texture), + // ); + + // unsafe { + // let buffer_contents = + // (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; + // std::ptr::copy_nonoverlapping(images.as_ptr(), buffer_contents, images.len()); + // } + + // command_encoder.draw_primitives_instanced( + // metal::MTLPrimitiveType::Triangle, + // 0, + // 6, + // images.len() as u64, + // ); + // *offset = next_offset; + // } + } + fn render_path_sprites( &mut self, layer_id: usize, From 531ffc01c91bb0bccc82be6f90e3ff0870d7ea4b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 20:34:59 -0600 Subject: [PATCH 21/89] Pass CVImageBuffers into GPUI instead of IOSurfaces --- Cargo.lock | 22 ++++---- crates/capture/Cargo.toml | 2 +- crates/capture/src/main.rs | 71 +++++------------------- crates/gpui/Cargo.toml | 2 +- crates/gpui/src/platform/mac/renderer.rs | 3 +- crates/io_surface/src/io_surface.rs | 21 ------- crates/{io_surface => media}/Cargo.toml | 4 +- crates/media/src/media.rs | 66 ++++++++++++++++++++++ 8 files changed, 95 insertions(+), 96 deletions(-) delete mode 100644 crates/io_surface/src/io_surface.rs rename crates/{io_surface => media}/Cargo.toml (74%) create mode 100644 crates/media/src/media.rs diff --git a/Cargo.lock b/Cargo.lock index aa030b9575..7b4fbdaaa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,8 +763,8 @@ dependencies = [ "foreign-types", "futures", "gpui", - "io_surface", "log", + "media", "objc", "parking_lot 0.11.2", "postage", @@ -2232,9 +2232,9 @@ dependencies = [ "futures", "gpui_macros", "image", - "io_surface", "lazy_static", "log", + "media", "metal", "num_cpus", "objc", @@ -2601,15 +2601,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "io_surface" -version = "0.1.0" -dependencies = [ - "block", - "core-foundation", - "objc", -] - [[package]] name = "iovec" version = "0.1.4" @@ -3039,6 +3030,15 @@ dependencies = [ "digest 0.10.3", ] +[[package]] +name = "media" +version = "0.1.0" +dependencies = [ + "block", + "core-foundation", + "objc", +] + [[package]] name = "memchr" version = "2.5.0" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 8c6e901bbd..f52fde195e 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -10,7 +10,7 @@ identifier = "dev.zed.Capture" [dependencies] gpui = { path = "../gpui" } -io_surface = { path = "../io_surface" } +media = { path = "../media" } block = "0.1" cocoa = "0.24" diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index e83135fc64..ed9f56cd18 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -16,8 +16,8 @@ use gpui::{ platform::current::Surface, Menu, MenuItem, ViewContext, }; -use io_surface::IOSurface; use log::LevelFilter; +use media::core_video::{self, CVImageBuffer}; use objc::{ class, declare::ClassDecl, @@ -55,7 +55,7 @@ fn main() { } struct ScreenCaptureView { - surface: Option, + image_buffer: Option, } impl gpui::Entity for ScreenCaptureView { @@ -64,8 +64,9 @@ impl gpui::Entity for ScreenCaptureView { impl ScreenCaptureView { pub fn new(cx: &mut ViewContext) -> Self { - let (surface_tx, mut surface_rx) = postage::watch::channel::>(); - let surface_tx = Arc::new(Mutex::new(surface_tx)); + let (image_buffer_tx, mut image_buffer_rx) = + postage::watch::channel::>(); + let image_buffer_tx = Arc::new(Mutex::new(image_buffer_tx)); unsafe { let block = ConcreteBlock::new(move |content: id, error: id| { @@ -93,7 +94,7 @@ impl ScreenCaptureView { let output: id = msg_send![capture_output_class, alloc]; let output: id = msg_send![output, init]; - let surface_tx = surface_tx.clone(); + let surface_tx = image_buffer_tx.clone(); let callback = Box::new(move |buffer: CMSampleBufferRef| { let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); @@ -112,8 +113,7 @@ impl ScreenCaptureView { } let image_buffer = buffer.image_buffer(); - let io_surface = image_buffer.io_surface(); - *surface_tx.lock().borrow_mut() = Some(io_surface); + *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; let callback = Box::into_raw(Box::new(callback)); (*output).set_ivar("callback", callback as *mut c_void); @@ -169,10 +169,10 @@ impl ScreenCaptureView { } cx.spawn_weak(|this, mut cx| async move { - while let Some(surface) = surface_rx.next().await { + while let Some(image_buffer) = image_buffer_rx.next().await { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { - this.surface = surface; + this.image_buffer = image_buffer; println!("NEW SURFACE!"); cx.notify(); }) @@ -183,7 +183,7 @@ impl ScreenCaptureView { }) .detach(); - Self { surface: None } + Self { image_buffer: None } } } @@ -193,12 +193,12 @@ impl gpui::View for ScreenCaptureView { } fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - let surface = self.surface.clone(); + let image_buffer = self.image_buffer.clone(); Canvas::new(move |bounds, _, cx| { - if let Some(native_surface) = surface.clone() { + if let Some(image_buffer) = image_buffer.clone() { cx.scene.push_surface(Surface { bounds, - native_surface, + image_buffer, }); } }) @@ -291,48 +291,3 @@ mod core_media { fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; } } - -mod core_video { - #![allow(non_snake_case)] - - use core_foundation::{ - base::{CFTypeID, TCFType}, - declare_TCFType, impl_CFTypeDescription, impl_TCFType, - }; - use io_surface::{IOSurface, IOSurfaceRef}; - use std::ffi::c_void; - - #[repr(C)] - pub struct __CVImageBuffer(c_void); - // The ref type must be a pointer to the underlying struct. - pub type CVImageBufferRef = *const __CVImageBuffer; - - declare_TCFType!(CVImageBuffer, CVImageBufferRef); - impl_TCFType!(CVImageBuffer, CVImageBufferRef, CVImageBufferGetTypeID); - impl_CFTypeDescription!(CVImageBuffer); - - impl CVImageBuffer { - pub fn io_surface(&self) -> IOSurface { - unsafe { - IOSurface::wrap_under_get_rule(CVPixelBufferGetIOSurface( - self.as_concrete_TypeRef(), - )) - } - } - - pub fn width(&self) -> usize { - unsafe { CVPixelBufferGetWidth(self.as_concrete_TypeRef()) } - } - - pub fn height(&self) -> usize { - unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } - } - } - - extern "C" { - fn CVImageBufferGetTypeID() -> CFTypeID; - fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; - fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; - fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; - } -} diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 6539c292fc..51bc416e19 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -60,7 +60,7 @@ png = "0.16" simplelog = "0.9" [target.'cfg(target_os = "macos")'.dependencies] -io_surface = { path = "../io_surface" } +media = { path = "../media" } anyhow = "1" block = "0.1" cocoa = "0.24" diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 41883a2f73..6c1d3cef21 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -39,7 +39,7 @@ struct PathSprite { pub struct Surface { pub bounds: RectF, - pub native_surface: io_surface::IOSurface, + pub image_buffer: media::core_video::CVImageBuffer, } impl Renderer { @@ -790,7 +790,6 @@ impl Renderer { let target_size = surface.bounds.size() * scale_factor; // let corner_radius = surface.corner_radius * scale_factor; // let border_width = surface.border.width * scale_factor; - // let (alloc_id, atlas_bounds) = self.image_cache.render(&surface.native_surface); } // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); diff --git a/crates/io_surface/src/io_surface.rs b/crates/io_surface/src/io_surface.rs deleted file mode 100644 index 13b1fd5ce1..0000000000 --- a/crates/io_surface/src/io_surface.rs +++ /dev/null @@ -1,21 +0,0 @@ -#![allow(non_snake_case)] - -use core_foundation::{ - base::{CFTypeID, TCFType}, - declare_TCFType, impl_CFTypeDescription, impl_TCFType, -}; -use std::ffi::c_void; - -#[repr(C)] -pub struct __IOSurface(c_void); -// The ref type must be a pointer to the underlying struct. -pub type IOSurfaceRef = *const __IOSurface; - -declare_TCFType!(IOSurface, IOSurfaceRef); -impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); -impl_CFTypeDescription!(IOSurface); - -#[link(name = "IOSurface", kind = "framework")] -extern "C" { - fn IOSurfaceGetTypeID() -> CFTypeID; -} diff --git a/crates/io_surface/Cargo.toml b/crates/media/Cargo.toml similarity index 74% rename from crates/io_surface/Cargo.toml rename to crates/media/Cargo.toml index 2d9324b6d8..ff472fe352 100644 --- a/crates/io_surface/Cargo.toml +++ b/crates/media/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "io_surface" +name = "media" version = "0.1.0" edition = "2021" [lib] -path = "src/io_surface.rs" +path = "src/media.rs" doctest = false [dependencies] diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs new file mode 100644 index 0000000000..435c03e936 --- /dev/null +++ b/crates/media/src/media.rs @@ -0,0 +1,66 @@ +#![allow(non_snake_case)] + +use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, +}; +use std::ffi::c_void; + +pub mod io_surface { + use super::*; + + #[repr(C)] + pub struct __IOSurface(c_void); + // The ref type must be a pointer to the underlying struct. + pub type IOSurfaceRef = *const __IOSurface; + + declare_TCFType!(IOSurface, IOSurfaceRef); + impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); + impl_CFTypeDescription!(IOSurface); + + #[link(name = "IOSurface", kind = "framework")] + extern "C" { + fn IOSurfaceGetTypeID() -> CFTypeID; + } +} + +pub mod core_video { + #![allow(non_snake_case)] + + use super::*; + use io_surface::{IOSurface, IOSurfaceRef}; + + #[repr(C)] + pub struct __CVImageBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CVImageBufferRef = *const __CVImageBuffer; + + declare_TCFType!(CVImageBuffer, CVImageBufferRef); + impl_TCFType!(CVImageBuffer, CVImageBufferRef, CVImageBufferGetTypeID); + impl_CFTypeDescription!(CVImageBuffer); + + impl CVImageBuffer { + pub fn io_surface(&self) -> IOSurface { + unsafe { + IOSurface::wrap_under_get_rule(CVPixelBufferGetIOSurface( + self.as_concrete_TypeRef(), + )) + } + } + + pub fn width(&self) -> usize { + unsafe { CVPixelBufferGetWidth(self.as_concrete_TypeRef()) } + } + + pub fn height(&self) -> usize { + unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } + } + } + + extern "C" { + fn CVImageBufferGetTypeID() -> CFTypeID; + fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; + fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + } +} From 37da841716cbe14d70cb05200ea3c4b0187eec84 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 30 Aug 2022 21:50:41 -0600 Subject: [PATCH 22/89] Start on using CVMetalTextureCache --- Cargo.lock | 1 + crates/gpui/src/platform/mac/renderer.rs | 30 ++++++- crates/media/Cargo.toml | 3 +- crates/media/src/media.rs | 100 +++++++++++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b4fbdaaa3..46abc68765 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3036,6 +3036,7 @@ version = "0.1.0" dependencies = [ "block", "core-foundation", + "metal", "objc", ] diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 6c1d3cef21..1023d5a5ef 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -9,10 +9,13 @@ use crate::{ scene::{Glyph, Icon, Image, ImageGlyph, Layer, Quad, Scene, Shadow, Underline}, }; use cocoa::foundation::NSUInteger; +use core_foundation::base::TCFType; +use foreign_types::ForeignTypeRef; use log::warn; +use media::core_video::{self, CVMetalTextureCache}; use metal::{MTLPixelFormat, MTLResourceOptions, NSRange}; use shaders::ToFloat2 as _; -use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, sync::Arc, vec}; +use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, ptr, sync::Arc, vec}; const SHADERS_METALLIB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/shaders.metallib")); const INSTANCE_BUFFER_SIZE: usize = 8192 * 1024; // This is an arbitrary decision. There's probably a more optimal value. @@ -29,6 +32,7 @@ pub struct Renderer { underline_pipeline_state: metal::RenderPipelineState, unit_vertices: metal::Buffer, instances: metal::Buffer, + cv_texture_cache: core_video::CVMetalTextureCache, } struct PathSprite { @@ -39,7 +43,7 @@ struct PathSprite { pub struct Surface { pub bounds: RectF, - pub image_buffer: media::core_video::CVImageBuffer, + pub image_buffer: core_video::CVImageBuffer, } impl Renderer { @@ -128,6 +132,7 @@ impl Renderer { "underline_fragment", pixel_format, ); + let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()); Self { sprite_cache, image_cache, @@ -140,6 +145,7 @@ impl Renderer { underline_pipeline_state, unit_vertices, instances, + cv_texture_cache, } } @@ -786,10 +792,26 @@ impl Renderer { } for surface in surfaces { - let origin = surface.bounds.origin() * scale_factor; - let target_size = surface.bounds.size() * scale_factor; + // let origin = surface.bounds.origin() * scale_factor; + // let target_size = surface.bounds.size() * scale_factor; // let corner_radius = surface.corner_radius * scale_factor; // let border_width = surface.border.width * scale_factor; + + let width = surface.image_buffer.width(); + let height = surface.image_buffer.height(); + + // We should add this method, but this return CVPixelFormatType and we need MTLPixelFormat + // I found at least one code example that manually maps them. Not sure what other options we have. + let pixel_format = surface.image_buffer.pixel_format_type(); + + let texture = self.cv_texture_cache.create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + pixel_format, + width, + height, + 0, + ); } // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index ff472fe352..06f73caaf5 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -10,4 +10,5 @@ doctest = false [dependencies] block = "0.1" core-foundation = "0.9.3" -objc = "0.2" \ No newline at end of file +metal = "0.21.0" +objc = "0.2" diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 435c03e936..855435359c 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -1,11 +1,15 @@ #![allow(non_snake_case)] +#![allow(non_camel_case_types)] use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, }; +use objc::runtime; use std::ffi::c_void; +pub type id = *mut runtime::Object; + pub mod io_surface { use super::*; @@ -27,8 +31,14 @@ pub mod io_surface { pub mod core_video { #![allow(non_snake_case)] + use std::ptr; + use super::*; + use core_foundation::{ + base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, + }; use io_surface::{IOSurface, IOSurfaceRef}; + use metal::{MTLDevice, MTLPixelFormat}; #[repr(C)] pub struct __CVImageBuffer(c_void); @@ -63,4 +73,94 @@ pub mod core_video { fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; } + + #[repr(C)] + pub struct __CVMetalTextureCache(c_void); + pub type CVMetalTextureCacheRef = *const __CVMetalTextureCache; + + declare_TCFType!(CVMetalTextureCache, CVMetalTextureCacheRef); + impl_TCFType!( + CVMetalTextureCache, + CVMetalTextureCacheRef, + CVMetalTextureCacheGetTypeID + ); + impl_CFTypeDescription!(CVMetalTextureCache); + + impl CVMetalTextureCache { + pub fn new(metal_device: *mut MTLDevice) -> Self { + unsafe { + let mut this = ptr::null(); + let result = CVMetalTextureCacheCreate( + kCFAllocatorDefault, + ptr::null_mut(), + metal_device, + ptr::null_mut(), + &mut this, + ); + // TODO: Check result + CVMetalTextureCache::wrap_under_create_rule(this) + } + } + + pub fn create_texture_from_image( + &self, + source: CVImageBufferRef, + texture_attributes: CFDictionaryRef, + pixel_format: MTLPixelFormat, + width: usize, + height: usize, + plane_index: usize, + ) -> CVMetalTexture { + unsafe { + let mut this = ptr::null(); + let result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, + self.as_concrete_TypeRef(), + source, + texture_attributes, + pixel_format, + width, + height, + plane_index, + &mut this, + ); + // TODO: Check result + CVMetalTexture::wrap_under_create_rule(this) + } + } + } + + extern "C" { + fn CVMetalTextureCacheGetTypeID() -> CFTypeID; + fn CVMetalTextureCacheCreate( + allocator: CFAllocatorRef, + cache_attributes: CFDictionaryRef, + metal_device: *const MTLDevice, + texture_attributes: CFDictionaryRef, + cache_out: *mut CVMetalTextureCacheRef, + ) -> i32; // TODO: This should be a CVReturn enum + fn CVMetalTextureCacheCreateTextureFromImage( + allocator: CFAllocatorRef, + texture_cache: CVMetalTextureCacheRef, + source_image: CVImageBufferRef, + texture_attributes: CFDictionaryRef, + pixel_format: MTLPixelFormat, + width: usize, + height: usize, + plane_index: usize, + texture_out: *mut CVMetalTextureRef, + ) -> i32; + } + + #[repr(C)] + pub struct __CVMetalTexture(c_void); + pub type CVMetalTextureRef = *const __CVMetalTexture; + + declare_TCFType!(CVMetalTexture, CVMetalTextureRef); + impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID); + impl_CFTypeDescription!(CVMetalTexture); + + extern "C" { + fn CVMetalTextureGetTypeID() -> CFTypeID; + } } From 79a7a0e0e709a61965c6c15070b14f94f3fe8b60 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 11:02:48 +0200 Subject: [PATCH 23/89] Capture screen in BGRA8 and render it in `capture` example app --- Cargo.lock | 3 +- crates/capture/Cargo.toml | 1 - crates/capture/src/main.rs | 5 +- crates/gpui/src/platform/mac/renderer.rs | 144 +++++++++++++---------- crates/gpui/src/scene.rs | 8 +- crates/media/Cargo.toml | 4 + crates/media/build.rs | 29 +++++ crates/media/src/bindings.h | 1 + crates/media/src/bindings.rs | 8 ++ crates/media/src/media.rs | 24 +++- 10 files changed, 156 insertions(+), 71 deletions(-) create mode 100644 crates/media/build.rs create mode 100644 crates/media/src/bindings.h create mode 100644 crates/media/src/bindings.rs diff --git a/Cargo.lock b/Cargo.lock index 46abc68765..b2d886db0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -756,7 +756,6 @@ version = "0.1.0" dependencies = [ "bindgen", "block", - "cc", "cocoa", "core-foundation", "core-graphics", @@ -3034,8 +3033,10 @@ dependencies = [ name = "media" version = "0.1.0" dependencies = [ + "bindgen", "block", "core-foundation", + "foreign-types", "metal", "objc", ] diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index f52fde195e..54b1d0990e 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -26,4 +26,3 @@ simplelog = "0.9" [build-dependencies] bindgen = "0.59.2" -cc = "1.0" \ No newline at end of file diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ed9f56cd18..af67ed9da4 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -128,6 +128,10 @@ impl ScreenCaptureView { let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; + let _: () = msg_send![ + config, + setPixelFormat: media::core_video::kCVPixelFormatType_32BGRA + ]; let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; @@ -173,7 +177,6 @@ impl ScreenCaptureView { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { this.image_buffer = image_buffer; - println!("NEW SURFACE!"); cx.notify(); }) } else { diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 1023d5a5ef..b0301e9cb6 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -383,6 +383,13 @@ impl Renderer { drawable_size, command_encoder, ); + self.render_surfaces( + layer.surfaces(), + scale_factor, + offset, + drawable_size, + command_encoder, + ); } command_encoder.end_encoding(); @@ -791,80 +798,87 @@ impl Renderer { return; } + command_encoder.set_render_pipeline_state(&self.image_pipeline_state); + command_encoder.set_vertex_buffer( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, + Some(&self.unit_vertices), + 0, + ); + command_encoder.set_vertex_bytes( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, + mem::size_of::() as u64, + [drawable_size.to_float2()].as_ptr() as *const c_void, + ); + for surface in surfaces { - // let origin = surface.bounds.origin() * scale_factor; - // let target_size = surface.bounds.size() * scale_factor; - // let corner_radius = surface.corner_radius * scale_factor; - // let border_width = surface.border.width * scale_factor; - - let width = surface.image_buffer.width(); - let height = surface.image_buffer.height(); - - // We should add this method, but this return CVPixelFormatType and we need MTLPixelFormat - // I found at least one code example that manually maps them. Not sure what other options we have. - let pixel_format = surface.image_buffer.pixel_format_type(); + let origin = surface.bounds.origin() * scale_factor; + let source_size = vec2i( + surface.image_buffer.width() as i32, + surface.image_buffer.height() as i32, + ); + let target_size = surface.bounds.size(); + let pixel_format = if surface.image_buffer.pixel_format_type() + == core_video::kCVPixelFormatType_32BGRA + { + MTLPixelFormat::BGRA8Unorm + } else { + panic!("unsupported pixel format") + }; let texture = self.cv_texture_cache.create_texture_from_image( surface.image_buffer.as_concrete_TypeRef(), ptr::null(), pixel_format, - width, - height, + source_size.x() as usize, + source_size.y() as usize, 0, ); + + align_offset(offset); + let next_offset = *offset + mem::size_of::(); + assert!( + next_offset <= INSTANCE_BUFFER_SIZE, + "instance buffer exhausted" + ); + + command_encoder.set_vertex_buffer( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, + Some(&self.instances), + *offset as u64, + ); + command_encoder.set_vertex_bytes( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, + mem::size_of::() as u64, + [source_size.to_float2()].as_ptr() as *const c_void, + ); + command_encoder.set_fragment_texture( + shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, + Some(texture.as_texture_ref()), + ); + + unsafe { + let buffer_contents = + (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; + std::ptr::write( + buffer_contents, + shaders::GPUIImage { + origin: origin.to_float2(), + target_size: target_size.to_float2(), + source_size: source_size.to_float2(), + atlas_origin: Default::default(), + border_top: Default::default(), + border_right: Default::default(), + border_bottom: Default::default(), + border_left: Default::default(), + border_color: Default::default(), + corner_radius: Default::default(), + }, + ); + } + + command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6); + *offset = next_offset; } - - // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); - // command_encoder.set_vertex_buffer( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, - // Some(&self.unit_vertices), - // 0, - // ); - // command_encoder.set_vertex_bytes( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, - // mem::size_of::() as u64, - // [drawable_size.to_float2()].as_ptr() as *const c_void, - // ); - - // for (atlas_id, images) in images_by_atlas { - // align_offset(offset); - // let next_offset = *offset + images.len() * mem::size_of::(); - // assert!( - // next_offset <= INSTANCE_BUFFER_SIZE, - // "instance buffer exhausted" - // ); - - // let texture = self.image_cache.atlas_texture(atlas_id).unwrap(); - // command_encoder.set_vertex_buffer( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, - // Some(&self.instances), - // *offset as u64, - // ); - // command_encoder.set_vertex_bytes( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, - // mem::size_of::() as u64, - // [vec2i(texture.width() as i32, texture.height() as i32).to_float2()].as_ptr() - // as *const c_void, - // ); - // command_encoder.set_fragment_texture( - // shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, - // Some(texture), - // ); - - // unsafe { - // let buffer_contents = - // (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; - // std::ptr::copy_nonoverlapping(images.as_ptr(), buffer_contents, images.len()); - // } - - // command_encoder.draw_primitives_instanced( - // metal::MTLPrimitiveType::Triangle, - // 0, - // 6, - // images.len() as u64, - // ); - // *offset = next_offset; - // } } fn render_path_sprites( diff --git a/crates/gpui/src/scene.rs b/crates/gpui/src/scene.rs index 00b8d3c88b..dbd46da1a0 100644 --- a/crates/gpui/src/scene.rs +++ b/crates/gpui/src/scene.rs @@ -397,14 +397,18 @@ impl Layer { } } + pub fn images(&self) -> &[Image] { + self.images.as_slice() + } + fn push_surface(&mut self, surface: Surface) { if can_draw(surface.bounds) { self.surfaces.push(surface); } } - pub fn images(&self) -> &[Image] { - self.images.as_slice() + pub fn surfaces(&self) -> &[Surface] { + self.surfaces.as_slice() } fn push_shadow(&mut self, shadow: Shadow) { diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 06f73caaf5..c2ca04abfa 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -10,5 +10,9 @@ doctest = false [dependencies] block = "0.1" core-foundation = "0.9.3" +foreign-types = "0.3" metal = "0.21.0" objc = "0.2" + +[build-dependencies] +bindgen = "0.59.2" diff --git a/crates/media/build.rs b/crates/media/build.rs new file mode 100644 index 0000000000..3446311d89 --- /dev/null +++ b/crates/media/build.rs @@ -0,0 +1,29 @@ +use std::{env, path::PathBuf, process::Command}; + +fn main() { + let sdk_path = String::from_utf8( + Command::new("xcrun") + .args(&["--sdk", "macosx", "--show-sdk-path"]) + .output() + .unwrap() + .stdout, + ) + .unwrap(); + let sdk_path = sdk_path.trim_end(); + + println!("cargo:rerun-if-changed=src/bindings.h"); + let bindings = bindgen::Builder::default() + .header("src/bindings.h") + .clang_arg(format!("-isysroot{}", sdk_path)) + .clang_arg("-xobjective-c") + .allowlist_var("kCVPixelFormatType_.*") + .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .layout_tests(false) + .generate() + .expect("unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("couldn't write dispatch bindings"); +} diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h new file mode 100644 index 0000000000..d5ba4eccd6 --- /dev/null +++ b/crates/media/src/bindings.h @@ -0,0 +1 @@ +#import \ No newline at end of file diff --git a/crates/media/src/bindings.rs b/crates/media/src/bindings.rs new file mode 100644 index 0000000000..a1c0b0da3e --- /dev/null +++ b/crates/media/src/bindings.rs @@ -0,0 +1,8 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(unused)] + +use objc::*; + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 855435359c..5db184f3b2 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -1,6 +1,8 @@ #![allow(non_snake_case)] #![allow(non_camel_case_types)] +mod bindings; + use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, @@ -34,11 +36,13 @@ pub mod core_video { use std::ptr; use super::*; + pub use crate::bindings::*; use core_foundation::{ base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, }; + use foreign_types::ForeignTypeRef; use io_surface::{IOSurface, IOSurfaceRef}; - use metal::{MTLDevice, MTLPixelFormat}; + use metal::{MTLDevice, MTLPixelFormat, MTLTexture}; #[repr(C)] pub struct __CVImageBuffer(c_void); @@ -65,13 +69,19 @@ pub mod core_video { pub fn height(&self) -> usize { unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } } + + pub fn pixel_format_type(&self) -> OSType { + unsafe { CVPixelBufferGetPixelFormatType(self.as_concrete_TypeRef()) } + } } + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVImageBufferGetTypeID() -> CFTypeID; fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetPixelFormatType(buffer: CVImageBufferRef) -> OSType; } #[repr(C)] @@ -130,6 +140,7 @@ pub mod core_video { } } + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVMetalTextureCacheGetTypeID() -> CFTypeID; fn CVMetalTextureCacheCreate( @@ -160,7 +171,18 @@ pub mod core_video { impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID); impl_CFTypeDescription!(CVMetalTexture); + impl CVMetalTexture { + pub fn as_texture_ref(&self) -> &metal::TextureRef { + unsafe { + let texture = CVMetalTextureGetTexture(self.as_concrete_TypeRef()); + &metal::TextureRef::from_ptr(texture as *mut _) + } + } + } + + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVMetalTextureGetTypeID() -> CFTypeID; + fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void; } } From fcf6aa15ebca1ccb09e14a89c936411046f8e253 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 11:09:14 +0200 Subject: [PATCH 24/89] Return results for fallible media APIs --- Cargo.lock | 1 + crates/gpui/src/platform/mac/renderer.rs | 21 ++++++++++-------- crates/media/Cargo.toml | 1 + crates/media/build.rs | 1 + crates/media/src/bindings.h | 3 ++- crates/media/src/media.rs | 28 ++++++++++++++---------- 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2d886db0a..669fedf659 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3033,6 +3033,7 @@ dependencies = [ name = "media" version = "0.1.0" dependencies = [ + "anyhow", "bindgen", "block", "core-foundation", diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index b0301e9cb6..8bf512a1a7 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -132,7 +132,7 @@ impl Renderer { "underline_fragment", pixel_format, ); - let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()); + let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()).unwrap(); Self { sprite_cache, image_cache, @@ -825,14 +825,17 @@ impl Renderer { panic!("unsupported pixel format") }; - let texture = self.cv_texture_cache.create_texture_from_image( - surface.image_buffer.as_concrete_TypeRef(), - ptr::null(), - pixel_format, - source_size.x() as usize, - source_size.y() as usize, - 0, - ); + let texture = self + .cv_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + pixel_format, + source_size.x() as usize, + source_size.y() as usize, + 0, + ) + .unwrap(); align_offset(offset); let next_offset = *offset + mem::size_of::(); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index c2ca04abfa..8773fbbe63 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -8,6 +8,7 @@ path = "src/media.rs" doctest = false [dependencies] +anyhow = "1.0" block = "0.1" core-foundation = "0.9.3" foreign-types = "0.3" diff --git a/crates/media/build.rs b/crates/media/build.rs index 3446311d89..50834ebe7f 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -17,6 +17,7 @@ fn main() { .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") .allowlist_var("kCVPixelFormatType_.*") + .allowlist_var("kCVReturn.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h index d5ba4eccd6..4e19d2701d 100644 --- a/crates/media/src/bindings.h +++ b/crates/media/src/bindings.h @@ -1 +1,2 @@ -#import \ No newline at end of file +#import +#import diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 5db184f3b2..b313f5f8fd 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -33,16 +33,16 @@ pub mod io_surface { pub mod core_video { #![allow(non_snake_case)] - use std::ptr; - use super::*; pub use crate::bindings::*; + use anyhow::{anyhow, Result}; use core_foundation::{ base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, }; use foreign_types::ForeignTypeRef; use io_surface::{IOSurface, IOSurfaceRef}; - use metal::{MTLDevice, MTLPixelFormat, MTLTexture}; + use metal::{MTLDevice, MTLPixelFormat}; + use std::ptr; #[repr(C)] pub struct __CVImageBuffer(c_void); @@ -97,7 +97,7 @@ pub mod core_video { impl_CFTypeDescription!(CVMetalTextureCache); impl CVMetalTextureCache { - pub fn new(metal_device: *mut MTLDevice) -> Self { + pub fn new(metal_device: *mut MTLDevice) -> Result { unsafe { let mut this = ptr::null(); let result = CVMetalTextureCacheCreate( @@ -107,8 +107,11 @@ pub mod core_video { ptr::null_mut(), &mut this, ); - // TODO: Check result - CVMetalTextureCache::wrap_under_create_rule(this) + if result == kCVReturnSuccess { + Ok(CVMetalTextureCache::wrap_under_create_rule(this)) + } else { + Err(anyhow!("could not create texture cache, code: {}", result)) + } } } @@ -120,7 +123,7 @@ pub mod core_video { width: usize, height: usize, plane_index: usize, - ) -> CVMetalTexture { + ) -> Result { unsafe { let mut this = ptr::null(); let result = CVMetalTextureCacheCreateTextureFromImage( @@ -134,8 +137,11 @@ pub mod core_video { plane_index, &mut this, ); - // TODO: Check result - CVMetalTexture::wrap_under_create_rule(this) + if result == kCVReturnSuccess { + Ok(CVMetalTexture::wrap_under_create_rule(this)) + } else { + Err(anyhow!("could not create texture, code: {}", result)) + } } } } @@ -149,7 +155,7 @@ pub mod core_video { metal_device: *const MTLDevice, texture_attributes: CFDictionaryRef, cache_out: *mut CVMetalTextureCacheRef, - ) -> i32; // TODO: This should be a CVReturn enum + ) -> CVReturn; fn CVMetalTextureCacheCreateTextureFromImage( allocator: CFAllocatorRef, texture_cache: CVMetalTextureCacheRef, @@ -160,7 +166,7 @@ pub mod core_video { height: usize, plane_index: usize, texture_out: *mut CVMetalTextureRef, - ) -> i32; + ) -> CVReturn; } #[repr(C)] From 3f66dd678a134bcf85e5481acf0d5edc8d2692a6 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 11:10:27 +0200 Subject: [PATCH 25/89] :fire: --- crates/media/src/media.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index b313f5f8fd..42a6eabac7 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -7,11 +7,8 @@ use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, }; -use objc::runtime; use std::ffi::c_void; -pub type id = *mut runtime::Object; - pub mod io_surface { use super::*; From a02e388ea2adc12579b334b32e3d47dc1c4ac658 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 12:02:57 +0200 Subject: [PATCH 26/89] Honor scale factor when rendering --- crates/capture/src/main.rs | 4 ++-- crates/gpui/src/platform/mac/renderer.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index af67ed9da4..36efe369f9 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -123,8 +123,8 @@ impl ScreenCaptureView { // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; let config: id = msg_send![class!(SCStreamConfiguration), alloc]; let config: id = msg_send![config, init]; - let _: () = msg_send![config, setWidth: display_width]; - let _: () = msg_send![config, setHeight: display_height]; + let _: () = msg_send![config, setWidth: display_width * 2]; + let _: () = msg_send![config, setHeight: display_height * 2]; let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 8bf512a1a7..ad0915ed5b 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -816,7 +816,7 @@ impl Renderer { surface.image_buffer.width() as i32, surface.image_buffer.height() as i32, ); - let target_size = surface.bounds.size(); + let target_size = surface.bounds.size() * scale_factor; let pixel_format = if surface.image_buffer.pixel_format_type() == core_video::kCVPixelFormatType_32BGRA { From f621d290fe5891aa9f3d4021e1059ccac7038fe6 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 14:33:58 +0200 Subject: [PATCH 27/89] Move `core_media` into `media` crate --- crates/capture/build.rs | 3 -- crates/capture/src/bindings.h | 1 - crates/capture/src/main.rs | 64 +++-------------------------------- crates/media/build.rs | 1 + crates/media/src/bindings.h | 1 + crates/media/src/media.rs | 61 ++++++++++++++++++++++++++++++++- 6 files changed, 67 insertions(+), 64 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 482e6487ce..7adb758dde 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,9 +1,7 @@ use std::{env, path::PathBuf, process::Command}; fn main() { - println!("cargo:rustc-link-lib=framework=CoreMedia"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); - println!("cargo:rustc-link-lib=framework=System"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); let sdk_path = String::from_utf8( @@ -21,7 +19,6 @@ fn main() { .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") - .allowlist_function("CMTimeMake") .allowlist_function("dispatch_queue_create") .allowlist_type("SCStreamOutputType") .allowlist_type("SCFrameStatus") diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h index 45ac30446b..5e3c525891 100644 --- a/crates/capture/src/bindings.h +++ b/crates/capture/src/bindings.h @@ -1,3 +1,2 @@ -#import #import #import diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 36efe369f9..0a05471f4f 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -7,7 +7,6 @@ use cocoa::{ foundation::{NSArray, NSString, NSUInteger}, }; use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; -use core_media::{CMSampleBuffer, CMSampleBufferRef}; use futures::StreamExt; use gpui::{ actions, @@ -17,7 +16,10 @@ use gpui::{ Menu, MenuItem, ViewContext, }; use log::LevelFilter; -use media::core_video::{self, CVImageBuffer}; +use media::{ + core_media::{CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, + core_video::{self, CVImageBuffer}, +}; use objc::{ class, declare::ClassDecl, @@ -125,7 +127,7 @@ impl ScreenCaptureView { let config: id = msg_send![config, init]; let _: () = msg_send![config, setWidth: display_width * 2]; let _: () = msg_send![config, setHeight: display_height * 2]; - let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; + let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; let _: () = msg_send![ @@ -238,59 +240,3 @@ extern "C" fn sample_output( fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } - -mod core_media { - #![allow(non_snake_case)] - - use crate::core_video::{CVImageBuffer, CVImageBufferRef}; - use core_foundation::{ - array::{CFArray, CFArrayRef}, - base::{CFTypeID, TCFType}, - declare_TCFType, - dictionary::CFDictionary, - impl_CFTypeDescription, impl_TCFType, - string::CFString, - }; - use std::ffi::c_void; - - #[repr(C)] - pub struct __CMSampleBuffer(c_void); - // The ref type must be a pointer to the underlying struct. - pub type CMSampleBufferRef = *const __CMSampleBuffer; - - declare_TCFType!(CMSampleBuffer, CMSampleBufferRef); - impl_TCFType!(CMSampleBuffer, CMSampleBufferRef, CMSampleBufferGetTypeID); - impl_CFTypeDescription!(CMSampleBuffer); - - impl CMSampleBuffer { - pub fn attachments(&self) -> Vec> { - unsafe { - let attachments = - CMSampleBufferGetSampleAttachmentsArray(self.as_concrete_TypeRef(), true); - CFArray::::wrap_under_get_rule(attachments) - .into_iter() - .map(|attachments| { - CFDictionary::wrap_under_get_rule(attachments.as_concrete_TypeRef()) - }) - .collect() - } - } - - pub fn image_buffer(&self) -> CVImageBuffer { - unsafe { - CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( - self.as_concrete_TypeRef(), - )) - } - } - } - - extern "C" { - fn CMSampleBufferGetTypeID() -> CFTypeID; - fn CMSampleBufferGetSampleAttachmentsArray( - buffer: CMSampleBufferRef, - create_if_necessary: bool, - ) -> CFArrayRef; - fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; - } -} diff --git a/crates/media/build.rs b/crates/media/build.rs index 50834ebe7f..0e610113d2 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -16,6 +16,7 @@ fn main() { .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") + .allowlist_function("CMTimeMake") .allowlist_var("kCVPixelFormatType_.*") .allowlist_var("kCVReturn.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h index 4e19d2701d..3a4689f708 100644 --- a/crates/media/src/bindings.h +++ b/crates/media/src/bindings.h @@ -1,2 +1,3 @@ +#import #import #import diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 42a6eabac7..86ba8d393e 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -31,7 +31,8 @@ pub mod core_video { #![allow(non_snake_case)] use super::*; - pub use crate::bindings::*; + pub use crate::bindings::kCVPixelFormatType_32BGRA; + use crate::bindings::{kCVReturnSuccess, CVReturn, OSType}; use anyhow::{anyhow, Result}; use core_foundation::{ base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, @@ -189,3 +190,61 @@ pub mod core_video { fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void; } } + +pub mod core_media { + #![allow(non_snake_case)] + + pub use crate::bindings::CMTimeMake; + use crate::core_video::{CVImageBuffer, CVImageBufferRef}; + use core_foundation::{ + array::{CFArray, CFArrayRef}, + base::{CFTypeID, TCFType}, + declare_TCFType, + dictionary::CFDictionary, + impl_CFTypeDescription, impl_TCFType, + string::CFString, + }; + use std::ffi::c_void; + + #[repr(C)] + pub struct __CMSampleBuffer(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CMSampleBufferRef = *const __CMSampleBuffer; + + declare_TCFType!(CMSampleBuffer, CMSampleBufferRef); + impl_TCFType!(CMSampleBuffer, CMSampleBufferRef, CMSampleBufferGetTypeID); + impl_CFTypeDescription!(CMSampleBuffer); + + impl CMSampleBuffer { + pub fn attachments(&self) -> Vec> { + unsafe { + let attachments = + CMSampleBufferGetSampleAttachmentsArray(self.as_concrete_TypeRef(), true); + CFArray::::wrap_under_get_rule(attachments) + .into_iter() + .map(|attachments| { + CFDictionary::wrap_under_get_rule(attachments.as_concrete_TypeRef()) + }) + .collect() + } + } + + pub fn image_buffer(&self) -> CVImageBuffer { + unsafe { + CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( + self.as_concrete_TypeRef(), + )) + } + } + } + + #[link(name = "CoreMedia", kind = "framework")] + extern "C" { + fn CMSampleBufferGetTypeID() -> CFTypeID; + fn CMSampleBufferGetSampleAttachmentsArray( + buffer: CMSampleBufferRef, + create_if_necessary: bool, + ) -> CFArrayRef; + fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; + } +} From 7054fa61f2678e6a3797bd96566835637d6a12ba Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 15:49:55 +0200 Subject: [PATCH 28/89] Start compressing captured frames as H264 --- crates/capture/src/main.rs | 19 +++- crates/media/build.rs | 7 ++ crates/media/src/bindings.h | 2 + crates/media/src/media.rs | 170 +++++++++++++++++++++++++++++++++++- 4 files changed, 195 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 0a05471f4f..a727dff37c 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -17,8 +17,9 @@ use gpui::{ }; use log::LevelFilter; use media::{ - core_media::{CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, + core_media::{kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, core_video::{self, CVImageBuffer}, + video_toolbox::VTCompressionSession, }; use objc::{ class, @@ -85,6 +86,14 @@ impl ScreenCaptureView { let display: id = displays.objectAtIndex(0); let display_width: usize = msg_send![display, width]; let display_height: usize = msg_send![display, height]; + let compression_session = VTCompressionSession::new( + display_width, + display_height, + kCMVideoCodecType_H264, + None, + ptr::null(), + ) + .unwrap(); let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_ivar::<*mut c_void>("callback"); @@ -114,7 +123,15 @@ impl ScreenCaptureView { return; } + let timing_info = buffer.sample_timing_info(0).unwrap(); let image_buffer = buffer.image_buffer(); + compression_session + .encode_frame( + image_buffer.as_concrete_TypeRef(), + timing_info.presentationTimeStamp, + timing_info.duration, + ) + .unwrap(); *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; let callback = Box::into_raw(Box::new(callback)); diff --git a/crates/media/build.rs b/crates/media/build.rs index 0e610113d2..22233be8f8 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -16,9 +16,16 @@ fn main() { .header("src/bindings.h") .clang_arg(format!("-isysroot{}", sdk_path)) .clang_arg("-xobjective-c") + .allowlist_type("CMItemIndex") + .allowlist_type("CMSampleTimingInfo") + .allowlist_type("CMVideoCodecType") + .allowlist_type("VTEncodeInfoFlags") .allowlist_function("CMTimeMake") .allowlist_var("kCVPixelFormatType_.*") .allowlist_var("kCVReturn.*") + .allowlist_var("VTEncodeInfoFlags_.*") + .allowlist_var("kCMVideoCodecType_.*") + .allowlist_var("kCMTime.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h index 3a4689f708..4df283d0c1 100644 --- a/crates/media/src/bindings.h +++ b/crates/media/src/bindings.h @@ -1,3 +1,5 @@ +#import #import #import #import +#import diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 86ba8d393e..6ff7799b49 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -194,11 +194,15 @@ pub mod core_video { pub mod core_media { #![allow(non_snake_case)] - pub use crate::bindings::CMTimeMake; + pub use crate::bindings::{ + kCMTimeInvalid, kCMVideoCodecType_H264, CMItemIndex, CMSampleTimingInfo, CMTime, + CMTimeMake, CMVideoCodecType, + }; use crate::core_video::{CVImageBuffer, CVImageBufferRef}; + use anyhow::{anyhow, Result}; use core_foundation::{ array::{CFArray, CFArrayRef}, - base::{CFTypeID, TCFType}, + base::{CFTypeID, OSStatus, TCFType}, declare_TCFType, dictionary::CFDictionary, impl_CFTypeDescription, impl_TCFType, @@ -236,6 +240,27 @@ pub mod core_media { )) } } + + pub fn sample_timing_info(&self, index: usize) -> Result { + unsafe { + let mut timing_info = CMSampleTimingInfo { + duration: kCMTimeInvalid, + presentationTimeStamp: kCMTimeInvalid, + decodeTimeStamp: kCMTimeInvalid, + }; + let result = CMSampleBufferGetSampleTimingInfo( + self.as_concrete_TypeRef(), + index as CMItemIndex, + &mut timing_info, + ); + + if result == 0 { + Ok(timing_info) + } else { + Err(anyhow!("error getting sample timing info, code {}", result)) + } + } + } } #[link(name = "CoreMedia", kind = "framework")] @@ -246,5 +271,146 @@ pub mod core_media { create_if_necessary: bool, ) -> CFArrayRef; fn CMSampleBufferGetImageBuffer(buffer: CMSampleBufferRef) -> CVImageBufferRef; + fn CMSampleBufferGetSampleTimingInfo( + buffer: CMSampleBufferRef, + index: CMItemIndex, + timing_info_out: *mut CMSampleTimingInfo, + ) -> OSStatus; + } +} + +pub mod video_toolbox { + #![allow(non_snake_case)] + + use super::*; + use crate::{ + core_media::{CMSampleBufferRef, CMTime, CMVideoCodecType}, + core_video::CVImageBufferRef, + }; + use anyhow::{anyhow, Result}; + use bindings::VTEncodeInfoFlags; + use core_foundation::{ + base::OSStatus, + dictionary::{CFDictionary, CFDictionaryRef, CFMutableDictionary}, + mach_port::CFAllocatorRef, + }; + use std::ptr; + + #[repr(C)] + pub struct __VTCompressionSession(c_void); + // The ref type must be a pointer to the underlying struct. + pub type VTCompressionSessionRef = *const __VTCompressionSession; + + declare_TCFType!(VTCompressionSession, VTCompressionSessionRef); + impl_TCFType!( + VTCompressionSession, + VTCompressionSessionRef, + VTCompressionSessionGetTypeID + ); + impl_CFTypeDescription!(VTCompressionSession); + + impl VTCompressionSession { + pub fn new( + width: usize, + height: usize, + codec: CMVideoCodecType, + callback: VTCompressionOutputCallback, + callback_data: *const c_void, + ) -> Result { + unsafe { + let mut this = ptr::null(); + let result = VTCompressionSessionCreate( + ptr::null(), + width as i32, + height as i32, + codec, + ptr::null(), + ptr::null(), + ptr::null(), + Some(Self::output), + callback_data, + &mut this, + ); + + if result == 0 { + Ok(Self::wrap_under_create_rule(this)) + } else { + Err(anyhow!( + "error creating compression session, code {}", + result + )) + } + } + } + + extern "C" fn output( + outputCallbackRefCon: *mut c_void, + sourceFrameRefCon: *mut c_void, + status: OSStatus, + infoFlags: VTEncodeInfoFlags, + sampleBuffer: CMSampleBufferRef, + ) { + println!("YO!"); + } + + pub fn encode_frame( + &self, + buffer: CVImageBufferRef, + presentation_timestamp: CMTime, + duration: CMTime, + ) -> Result<()> { + unsafe { + let result = VTCompressionSessionEncodeFrame( + self.as_concrete_TypeRef(), + buffer, + presentation_timestamp, + duration, + ptr::null(), + ptr::null(), + ptr::null_mut(), + ); + if result == 0 { + Ok(()) + } else { + Err(anyhow!("error encoding frame, code {}", result)) + } + } + } + } + + type VTCompressionOutputCallback = Option< + unsafe extern "C" fn( + outputCallbackRefCon: *mut c_void, + sourceFrameRefCon: *mut c_void, + status: OSStatus, + infoFlags: VTEncodeInfoFlags, + sampleBuffer: CMSampleBufferRef, + ), + >; + + #[link(name = "VideoToolbox", kind = "framework")] + extern "C" { + fn VTCompressionSessionGetTypeID() -> CFTypeID; + fn VTCompressionSessionCreate( + allocator: CFAllocatorRef, + width: i32, + height: i32, + codec_type: CMVideoCodecType, + encoder_specification: CFDictionaryRef, + source_image_buffer_attributes: CFDictionaryRef, + compressed_data_allocator: CFAllocatorRef, + output_callback: VTCompressionOutputCallback, + output_callback_ref_con: *const c_void, + compression_session_out: *mut VTCompressionSessionRef, + ) -> OSStatus; + fn VTCompressionSessionEncodeFrame( + session: VTCompressionSessionRef, + image_buffer: CVImageBufferRef, + presentation_timestamp: CMTime, + duration: CMTime, + frame_properties: CFDictionaryRef, + source_frame_ref_con: *const c_void, + output_flags: *mut VTEncodeInfoFlags, + ) -> OSStatus; } } From 600029a9189dccd2f817c5934ecab061214eebf7 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 18:02:05 +0200 Subject: [PATCH 29/89] WIP: Start converting H264 samples to Annex-B NALs Co-Authored-By: Nathan Sobo --- crates/media/build.rs | 1 + crates/media/src/media.rs | 117 +++++++++++++++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/crates/media/build.rs b/crates/media/build.rs index 22233be8f8..1ee8f23e41 100644 --- a/crates/media/build.rs +++ b/crates/media/build.rs @@ -26,6 +26,7 @@ fn main() { .allowlist_var("VTEncodeInfoFlags_.*") .allowlist_var("kCMVideoCodecType_.*") .allowlist_var("kCMTime.*") + .allowlist_var("kCMSampleAttachmentKey_.*") .parse_callbacks(Box::new(bindgen::CargoCallbacks)) .layout_tests(false) .generate() diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 6ff7799b49..c6aa1e3d0a 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -208,7 +208,7 @@ pub mod core_media { impl_CFTypeDescription, impl_TCFType, string::CFString, }; - use std::ffi::c_void; + use std::{ffi::c_void, ptr}; #[repr(C)] pub struct __CMSampleBuffer(c_void); @@ -261,6 +261,14 @@ pub mod core_media { } } } + + pub fn format_description(&self) -> CMFormatDescription { + unsafe { + CMFormatDescription::wrap_under_get_rule(CMSampleBufferGetFormatDescription( + self.as_concrete_TypeRef(), + )) + } + } } #[link(name = "CoreMedia", kind = "framework")] @@ -276,6 +284,71 @@ pub mod core_media { index: CMItemIndex, timing_info_out: *mut CMSampleTimingInfo, ) -> OSStatus; + fn CMSampleBufferGetFormatDescription(buffer: CMSampleBufferRef) -> CMFormatDescriptionRef; + } + + #[repr(C)] + pub struct __CMFormatDescription(c_void); + // The ref type must be a pointer to the underlying struct. + pub type CMFormatDescriptionRef = *const __CMFormatDescription; + + declare_TCFType!(CMFormatDescription, CMFormatDescriptionRef); + impl_TCFType!( + CMFormatDescription, + CMFormatDescriptionRef, + CMFormatDescriptionGetTypeID + ); + impl_CFTypeDescription!(CMFormatDescription); + + impl CMFormatDescription { + pub fn h264_parameter_set_count(&self) -> usize { + unsafe { + let mut count = 0; + let result = CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + self.as_concrete_TypeRef(), + 0, + ptr::null_mut(), + ptr::null_mut(), + &mut count, + ptr::null_mut(), + ); + assert_eq!(result, 0); + count + } + } + + pub fn h264_parameter_set_at_index(&self, index: usize) -> Result<&[u8]> { + unsafe { + let mut bytes = ptr::null(); + let mut len = 0; + let result = CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + self.as_concrete_TypeRef(), + index, + &mut bytes, + &mut len, + ptr::null_mut(), + ptr::null_mut(), + ); + if result == 0 { + Ok(std::slice::from_raw_parts(bytes, len)) + } else { + Err(anyhow!("error getting parameter set, code: {}", result)) + } + } + } + } + + #[link(name = "CoreMedia", kind = "framework")] + extern "C" { + fn CMFormatDescriptionGetTypeID() -> CFTypeID; + fn CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + video_desc: CMFormatDescriptionRef, + parameter_set_index: usize, + parameter_set_pointer_out: *mut *const u8, + parameter_set_size_out: *mut usize, + parameter_set_count_out: *mut usize, + NALUnitHeaderLengthOut: *mut isize, + ) -> OSStatus; } } @@ -284,15 +357,17 @@ pub mod video_toolbox { use super::*; use crate::{ - core_media::{CMSampleBufferRef, CMTime, CMVideoCodecType}, + core_media::{CMSampleBuffer, CMSampleBufferRef, CMTime, CMVideoCodecType}, core_video::CVImageBufferRef, }; use anyhow::{anyhow, Result}; use bindings::VTEncodeInfoFlags; use core_foundation::{ base::OSStatus, - dictionary::{CFDictionary, CFDictionaryRef, CFMutableDictionary}, + dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, + number::{CFBooleanGetValue, CFBooleanRef}, + string::CFStringRef, }; use std::ptr; @@ -343,13 +418,39 @@ pub mod video_toolbox { } } - extern "C" fn output( - outputCallbackRefCon: *mut c_void, - sourceFrameRefCon: *mut c_void, + unsafe extern "C" fn output( + output_callback_ref_con: *mut c_void, + source_frame_ref_con: *mut c_void, status: OSStatus, - infoFlags: VTEncodeInfoFlags, - sampleBuffer: CMSampleBufferRef, + info_flags: VTEncodeInfoFlags, + sample_buffer: CMSampleBufferRef, ) { + if status != 0 { + println!("error encoding frame, code: {}", status); + return; + } + let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + + let mut is_iframe = false; + let attachments = sample_buffer.attachments(); + if let Some(attachments) = attachments.first() { + is_iframe = attachments + .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) + .map_or(true, |not_sync| { + CFBooleanGetValue(*not_sync as CFBooleanRef) + }); + } + + const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; + if is_iframe { + let format_description = sample_buffer.format_description(); + for ix in 0..format_description.h264_parameter_set_count() { + let parameter_set = format_description.h264_parameter_set_at_index(ix); + stream.extend(START_CODE); + stream.extend(parameter_set); + } + } + println!("YO!"); } From 047b5114f1158a35201558b2c3c2b12c5aa92b53 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 20:03:47 +0200 Subject: [PATCH 30/89] Start on a new, more abstract `CompressionSession` primitive Co-Authored-By: Nathan Sobo --- Cargo.lock | 7 +- crates/capture/Cargo.toml | 2 + crates/capture/src/compression_session.rs | 178 ++++++++++++++++++++++ crates/capture/src/main.rs | 55 +++++-- crates/media/Cargo.toml | 1 + crates/media/src/media.rs | 54 +------ 6 files changed, 236 insertions(+), 61 deletions(-) create mode 100644 crates/capture/src/compression_session.rs diff --git a/Cargo.lock b/Cargo.lock index 669fedf659..6b34c02f0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,9 +663,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] name = "bzip2-sys" @@ -754,8 +754,10 @@ dependencies = [ name = "capture" version = "0.1.0" dependencies = [ + "anyhow", "bindgen", "block", + "bytes", "cocoa", "core-foundation", "core-graphics", @@ -3036,6 +3038,7 @@ dependencies = [ "anyhow", "bindgen", "block", + "bytes", "core-foundation", "foreign-types", "metal", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 54b1d0990e..c2bee8312a 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -12,7 +12,9 @@ identifier = "dev.zed.Capture" gpui = { path = "../gpui" } media = { path = "../media" } +anyhow = "1.0.38" block = "0.1" +bytes = "1.2" cocoa = "0.24" core-foundation = "0.9.3" core-graphics = "0.22.3" diff --git a/crates/capture/src/compression_session.rs b/crates/capture/src/compression_session.rs new file mode 100644 index 0000000000..1a771d057b --- /dev/null +++ b/crates/capture/src/compression_session.rs @@ -0,0 +1,178 @@ +use anyhow::Result; +use core_foundation::base::{OSStatus, TCFType}; +use media::{ + core_media::{CMSampleBufferRef, CMSampleTimingInfo, CMVideoCodecType}, + core_video::CVImageBuffer, + video_toolbox::{VTCompressionSession, VTEncodeInfoFlags}, +}; +use std::ffi::c_void; + +pub struct CompressionSession { + session: VTCompressionSession, + output_callback: Box, +} + +impl + CompressionSession +{ + pub fn new(width: usize, height: usize, codec: CMVideoCodecType, callback: F) -> Result { + let callback = Box::new(callback); + let session = VTCompressionSession::new( + width, + height, + codec, + Some(Self::output_callback), + callback.as_ref() as *const _ as *const c_void, + )?; + Ok(Self { + session, + output_callback: callback, + }) + } + + pub fn encode_frame(&self, buffer: &CVImageBuffer, timing: CMSampleTimingInfo) -> Result<()> { + self.session.encode_frame( + buffer.as_concrete_TypeRef(), + timing.presentationTimeStamp, + timing.duration, + ) + } + + extern "C" fn output_callback( + output_callback_ref_con: *mut c_void, + _: *mut c_void, + status: OSStatus, + flags: VTEncodeInfoFlags, + sample_buffer: CMSampleBufferRef, + ) { + let callback = unsafe { &mut *(output_callback_ref_con as *mut F) }; + callback(status, flags, sample_buffer); + } +} + +// unsafe extern "C" fn output( +// output_callback_ref_con: *mut c_void, +// source_frame_ref_con: *mut c_void, +// status: OSStatus, +// info_flags: VTEncodeInfoFlags, +// sample_buffer: CMSampleBufferRef, +// ) { +// if status != 0 { +// println!("error encoding frame, code: {}", status); +// return; +// } +// let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + +// let mut is_iframe = false; +// let attachments = sample_buffer.attachments(); +// if let Some(attachments) = attachments.first() { +// is_iframe = attachments +// .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) +// .map_or(true, |not_sync| { +// CFBooleanGetValue(*not_sync as CFBooleanRef) +// }); +// } + +// const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; +// if is_iframe { +// let format_description = sample_buffer.format_description(); +// for ix in 0..format_description.h264_parameter_set_count() { +// let parameter_set = format_description.h264_parameter_set_at_index(ix); +// stream.extend(START_CODE); +// stream.extend(parameter_set); +// } +// } + +// println!("YO!"); +// } + +// static void videoFrameFinishedEncoding(void *outputCallbackRefCon, +// void *sourceFrameRefCon, +// OSStatus status, +// VTEncodeInfoFlags infoFlags, +// CMSampleBufferRef sampleBuffer) { +// // Check if there were any errors encoding +// if (status != noErr) { +// NSLog(@"Error encoding video, err=%lld", (int64_t)status); +// return; +// } + +// // In this example we will use a NSMutableData object to store the +// // elementary stream. +// NSMutableData *elementaryStream = [NSMutableData data]; + +// // Find out if the sample buffer contains an I-Frame. +// // If so we will write the SPS and PPS NAL units to the elementary stream. +// BOOL isIFrame = NO; +// CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); +// if (CFArrayGetCount(attachmentsArray)) { +// CFBooleanRef notSync; +// CFDictionaryRef dict = CFArrayGetValueAtIndex(attachmentsArray, 0); +// BOOL keyExists = CFDictionaryGetValueIfPresent(dict, +// kCMSampleAttachmentKey_NotSync, +// (const void **)¬Sync); +// // An I-Frame is a sync frame +// isIFrame = !keyExists || !CFBooleanGetValue(notSync); +// } + +// // This is the start code that we will write to +// // the elementary stream before every NAL unit +// static const size_t startCodeLength = 4; +// static const uint8_t startCode[] = {0x00, 0x00, 0x00, 0x01}; + +// // Write the SPS and PPS NAL units to the elementary stream before every I-Frame +// if (isIFrame) { +// CMFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sampleBuffer); + +// // Find out how many parameter sets there are +// size_t numberOfParameterSets; +// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, +// 0, NULL, NULL, +// &numberOfParameterSets, +// NULL); + +// // Write each parameter set to the elementary stream +// for (int i = 0; i < numberOfParameterSets; i++) { +// const uint8_t *parameterSetPointer; +// size_t parameterSetLength; +// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, +// i, +// ¶meterSetPointer, +// ¶meterSetLength, +// NULL, NULL); + +// // Write the parameter set to the elementary stream +// [elementaryStream appendBytes:startCode length:startCodeLength]; +// [elementaryStream appendBytes:parameterSetPointer length:parameterSetLength]; +// } +// } + +// // Get a pointer to the raw AVCC NAL unit data in the sample buffer +// size_t blockBufferLength; +// uint8_t *bufferDataPointer = NULL; +// CMBlockBufferGetDataPointer(CMSampleBufferGetDataBuffer(sampleBuffer), +// 0, +// NULL, +// &blockBufferLength, +// (char **)&bufferDataPointer); + +// // Loop through all the NAL units in the block buffer +// // and write them to the elementary stream with +// // start codes instead of AVCC length headers +// size_t bufferOffset = 0; +// static const int AVCCHeaderLength = 4; +// while (bufferOffset < blockBufferLength - AVCCHeaderLength) { +// // Read the NAL unit length +// uint32_t NALUnitLength = 0; +// memcpy(&NALUnitLength, bufferDataPointer + bufferOffset, AVCCHeaderLength); +// // Convert the length value from Big-endian to Little-endian +// NALUnitLength = CFSwapInt32BigToHost(NALUnitLength); +// // Write start code to the elementary stream +// [elementaryStream appendBytes:startCode length:startCodeLength]; +// // Write the NAL unit without the AVCC length header to the elementary stream +// [elementaryStream appendBytes:bufferDataPointer + bufferOffset + AVCCHeaderLength +// length:NALUnitLength]; +// // Move to the next NAL unit in the block buffer +// bufferOffset += AVCCHeaderLength + NALUnitLength; +// } +// } diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index a727dff37c..b1d8f11795 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,12 +1,18 @@ mod bindings; +mod compression_session; -use crate::bindings::SCStreamOutputType; +use crate::{bindings::SCStreamOutputType, compression_session::CompressionSession}; use block::ConcreteBlock; +use bytes::BytesMut; use cocoa::{ base::{id, nil, YES}, foundation::{NSArray, NSString, NSUInteger}, }; -use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef}; +use core_foundation::{ + base::TCFType, + number::{CFBooleanGetValue, CFBooleanRef, CFNumberRef}, + string::CFStringRef, +}; use futures::StreamExt; use gpui::{ actions, @@ -17,7 +23,10 @@ use gpui::{ }; use log::LevelFilter; use media::{ - core_media::{kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, CMTimeMake}, + core_media::{ + kCMSampleAttachmentKey_NotSync, kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, + CMTimeMake, + }, core_video::{self, CVImageBuffer}, video_toolbox::VTCompressionSession, }; @@ -86,12 +95,40 @@ impl ScreenCaptureView { let display: id = displays.objectAtIndex(0); let display_width: usize = msg_send![display, width]; let display_height: usize = msg_send![display, height]; - let compression_session = VTCompressionSession::new( + let mut compression_buffer = BytesMut::new(); + let compression_session = CompressionSession::new( display_width, display_height, kCMVideoCodecType_H264, - None, - ptr::null(), + move |status, flags, sample_buffer| { + if status != 0 { + println!("error encoding frame, code: {}", status); + return; + } + let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + + let mut is_iframe = false; + let attachments = sample_buffer.attachments(); + if let Some(attachments) = attachments.first() { + is_iframe = attachments + .find(kCMSampleAttachmentKey_NotSync as CFStringRef) + .map_or(true, |not_sync| { + CFBooleanGetValue(*not_sync as CFBooleanRef) + }); + } + + const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; + if is_iframe { + let format_description = sample_buffer.format_description(); + for ix in 0..format_description.h264_parameter_set_count() { + let parameter_set = + format_description.h264_parameter_set_at_index(ix).unwrap(); + compression_buffer.extend_from_slice(&START_CODE); + compression_buffer.extend_from_slice(parameter_set); + let nal_unit = compression_buffer.split(); + } + } + }, ) .unwrap(); @@ -126,11 +163,7 @@ impl ScreenCaptureView { let timing_info = buffer.sample_timing_info(0).unwrap(); let image_buffer = buffer.image_buffer(); compression_session - .encode_frame( - image_buffer.as_concrete_TypeRef(), - timing_info.presentationTimeStamp, - timing_info.duration, - ) + .encode_frame(&image_buffer, timing_info) .unwrap(); *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 8773fbbe63..aad2b74c02 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -10,6 +10,7 @@ doctest = false [dependencies] anyhow = "1.0" block = "0.1" +bytes = "1.2" core-foundation = "0.9.3" foreign-types = "0.3" metal = "0.21.0" diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index c6aa1e3d0a..5c5f43af9d 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -195,8 +195,8 @@ pub mod core_media { #![allow(non_snake_case)] pub use crate::bindings::{ - kCMTimeInvalid, kCMVideoCodecType_H264, CMItemIndex, CMSampleTimingInfo, CMTime, - CMTimeMake, CMVideoCodecType, + kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264, CMItemIndex, + CMSampleTimingInfo, CMTime, CMTimeMake, CMVideoCodecType, }; use crate::core_video::{CVImageBuffer, CVImageBufferRef}; use anyhow::{anyhow, Result}; @@ -357,18 +357,12 @@ pub mod video_toolbox { use super::*; use crate::{ - core_media::{CMSampleBuffer, CMSampleBufferRef, CMTime, CMVideoCodecType}, + core_media::{CMSampleBufferRef, CMTime, CMVideoCodecType}, core_video::CVImageBufferRef, }; use anyhow::{anyhow, Result}; - use bindings::VTEncodeInfoFlags; - use core_foundation::{ - base::OSStatus, - dictionary::CFDictionaryRef, - mach_port::CFAllocatorRef, - number::{CFBooleanGetValue, CFBooleanRef}, - string::CFStringRef, - }; + pub use bindings::VTEncodeInfoFlags; + use core_foundation::{base::OSStatus, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef}; use std::ptr; #[repr(C)] @@ -402,7 +396,7 @@ pub mod video_toolbox { ptr::null(), ptr::null(), ptr::null(), - Some(Self::output), + callback, callback_data, &mut this, ); @@ -418,42 +412,6 @@ pub mod video_toolbox { } } - unsafe extern "C" fn output( - output_callback_ref_con: *mut c_void, - source_frame_ref_con: *mut c_void, - status: OSStatus, - info_flags: VTEncodeInfoFlags, - sample_buffer: CMSampleBufferRef, - ) { - if status != 0 { - println!("error encoding frame, code: {}", status); - return; - } - let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - - let mut is_iframe = false; - let attachments = sample_buffer.attachments(); - if let Some(attachments) = attachments.first() { - is_iframe = attachments - .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) - .map_or(true, |not_sync| { - CFBooleanGetValue(*not_sync as CFBooleanRef) - }); - } - - const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; - if is_iframe { - let format_description = sample_buffer.format_description(); - for ix in 0..format_description.h264_parameter_set_count() { - let parameter_set = format_description.h264_parameter_set_at_index(ix); - stream.extend(START_CODE); - stream.extend(parameter_set); - } - } - - println!("YO!"); - } - pub fn encode_frame( &self, buffer: CVImageBufferRef, From b51abc5a639883de5416f260cc146012068791d2 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 31 Aug 2022 13:36:29 -0600 Subject: [PATCH 31/89] Read the frame data out of the CMSampleBuffer Still not sending it anywhere, but think I'm reading it correctly. --- Cargo.lock | 1 + crates/capture/Cargo.toml | 5 +--- crates/capture/script/capture | 5 ---- crates/capture/src/main.rs | 20 +++++++++++++++ crates/media/src/media.rs | 48 ++++++++++++++++++++++++++++++++++- 5 files changed, 69 insertions(+), 10 deletions(-) delete mode 100755 crates/capture/script/capture diff --git a/Cargo.lock b/Cargo.lock index 6b34c02f0d..60e54aeb9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -757,6 +757,7 @@ dependencies = [ "anyhow", "bindgen", "block", + "byteorder", "bytes", "cocoa", "core-foundation", diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index c2bee8312a..cdae75fc2b 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -4,10 +4,6 @@ version = "0.1.0" edition = "2021" description = "An example of screen capture" -[package.metadata.bundle] -name = "Capture" -identifier = "dev.zed.Capture" - [dependencies] gpui = { path = "../gpui" } media = { path = "../media" } @@ -15,6 +11,7 @@ media = { path = "../media" } anyhow = "1.0.38" block = "0.1" bytes = "1.2" +byteorder = "1.4" cocoa = "0.24" core-foundation = "0.9.3" core-graphics = "0.22.3" diff --git a/crates/capture/script/capture b/crates/capture/script/capture deleted file mode 100755 index 0e6cf1f409..0000000000 --- a/crates/capture/script/capture +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cargo bundle -TTY=`tty` -open ../../target/debug/bundle/osx/Capture.app --stdout $TTY --stderr $TTY diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index b1d8f11795..97a5557c66 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -3,6 +3,7 @@ mod compression_session; use crate::{bindings::SCStreamOutputType, compression_session::CompressionSession}; use block::ConcreteBlock; +use byteorder::{BigEndian, ReadBytesExt}; use bytes::BytesMut; use cocoa::{ base::{id, nil, YES}, @@ -128,6 +129,25 @@ impl ScreenCaptureView { let nal_unit = compression_buffer.split(); } } + + let data = sample_buffer.data(); + let mut data = data.bytes(); + + const AVCC_HEADER_LENGTH: usize = 4; + while data.len() - AVCC_HEADER_LENGTH > 0 { + let nal_unit_len = match data.read_u32::() { + Ok(len) => len as usize, + Err(error) => { + log::error!("error decoding nal unit length: {}", error); + return; + } + }; + compression_buffer.extend_from_slice(&START_CODE); + compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); + data = &data[nal_unit_len..]; + + let nal_unit = compression_buffer.split(); + } }, ) .unwrap(); diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 5c5f43af9d..ebe3ef7f4d 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -269,6 +269,14 @@ pub mod core_media { )) } } + + pub fn data(&self) -> CMBlockBuffer { + unsafe { + CMBlockBuffer::wrap_under_get_rule(CMSampleBufferGetDataBuffer( + self.as_concrete_TypeRef(), + )) + } + } } #[link(name = "CoreMedia", kind = "framework")] @@ -285,11 +293,11 @@ pub mod core_media { timing_info_out: *mut CMSampleTimingInfo, ) -> OSStatus; fn CMSampleBufferGetFormatDescription(buffer: CMSampleBufferRef) -> CMFormatDescriptionRef; + fn CMSampleBufferGetDataBuffer(sample_buffer: CMSampleBufferRef) -> CMBlockBufferRef; } #[repr(C)] pub struct __CMFormatDescription(c_void); - // The ref type must be a pointer to the underlying struct. pub type CMFormatDescriptionRef = *const __CMFormatDescription; declare_TCFType!(CMFormatDescription, CMFormatDescriptionRef); @@ -350,6 +358,44 @@ pub mod core_media { NALUnitHeaderLengthOut: *mut isize, ) -> OSStatus; } + + #[repr(C)] + pub struct __CMBlockBuffer(c_void); + pub type CMBlockBufferRef = *const __CMBlockBuffer; + + declare_TCFType!(CMBlockBuffer, CMBlockBufferRef); + impl_TCFType!(CMBlockBuffer, CMBlockBufferRef, CMBlockBufferGetTypeID); + impl_CFTypeDescription!(CMBlockBuffer); + + impl CMBlockBuffer { + pub fn bytes(&self) -> &[u8] { + unsafe { + let mut bytes = ptr::null(); + let mut len = 0; + let result = CMBlockBufferGetDataPointer( + self.as_concrete_TypeRef(), + 0, + &mut 0, + &mut len, + &mut bytes, + ); + assert!(result == 0, "could not get block buffer data"); + std::slice::from_raw_parts(bytes, len) + } + } + } + + #[link(name = "CoreMedia", kind = "framework")] + extern "C" { + fn CMBlockBufferGetTypeID() -> CFTypeID; + fn CMBlockBufferGetDataPointer( + buffer: CMBlockBufferRef, + offset: usize, + length_at_offset_out: *mut usize, + total_length_out: *mut usize, + data_pointer_out: *mut *const u8, + ) -> OSStatus; + } } pub mod video_toolbox { From 6c28b2172847fdb849b565655c0151fefe2c0eec Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 1 Sep 2022 17:52:12 +0200 Subject: [PATCH 32/89] WIP: Start binding LiveKit --- LiveKitObjC/LKRoom.m | 28 ++ LiveKitObjC/LiveKitObjC-Bridging-Header.h | 3 + .../LiveKitObjC.xcodeproj/project.pbxproj | 342 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcshareddata/IDEWorkspaceChecks.plist | 8 + .../xcshareddata/swiftpm/Package.resolved | 50 +++ LiveKitObjC/Room.swift | 23 ++ crates/capture/build.rs | 2 + crates/capture/src/main.rs | 106 +++--- 9 files changed, 517 insertions(+), 52 deletions(-) create mode 100644 LiveKitObjC/LKRoom.m create mode 100644 LiveKitObjC/LiveKitObjC-Bridging-Header.h create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved create mode 100644 LiveKitObjC/Room.swift diff --git a/LiveKitObjC/LKRoom.m b/LiveKitObjC/LKRoom.m new file mode 100644 index 0000000000..ea4d7d4402 --- /dev/null +++ b/LiveKitObjC/LKRoom.m @@ -0,0 +1,28 @@ +// +// LKRoom.m +// LiveKitObjC +// +// Created by Antonio Scandurra on 01/09/22. +// + +#import +#import + +@interface LKRoom: NSObject { +} + +@property (nonatomic, retain) SLKRoom* room; +@end + +@implementation LKRoom +-(id)init { + if (self = [super init]) { + self.room = [[SLKRoom alloc] init]; + } + return self; +} + +-(void)connectWithURL:(NSString *)url token:(NSString *)token callback:(void(^)(void))callback { + [self.room connectWithUrl:url token:token callback:callback]; +} +@end diff --git a/LiveKitObjC/LiveKitObjC-Bridging-Header.h b/LiveKitObjC/LiveKitObjC-Bridging-Header.h new file mode 100644 index 0000000000..e11d920b12 --- /dev/null +++ b/LiveKitObjC/LiveKitObjC-Bridging-Header.h @@ -0,0 +1,3 @@ +// +// Use this file to import your target's public headers that you would like to expose to Swift. +// diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj new file mode 100644 index 0000000000..93053cb68d --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -0,0 +1,342 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 55; + objects = { + +/* Begin PBXBuildFile section */ + AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; + AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; + AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */; }; + AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBDD28C121E6001AD7BE /* LiveKit */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libLiveKitObjC.a; sourceTree = BUILT_PRODUCTS_DIR; }; + AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "LiveKitObjC-Bridging-Header.h"; sourceTree = ""; }; + AFA4DBD828C0F87F001AD7BE /* Room.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Room.swift; sourceTree = ""; }; + AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = LKRoom.m; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + AFA4DBCB28C0F7F5001AD7BE /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */, + AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + AFA4DBC428C0F7F5001AD7BE = { + isa = PBXGroup; + children = ( + AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */, + AFA4DBD828C0F87F001AD7BE /* Room.swift */, + AFA4DBCE28C0F7F5001AD7BE /* Products */, + AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */, + ); + sourceTree = ""; + }; + AFA4DBCE28C0F7F5001AD7BE /* Products */ = { + isa = PBXGroup; + children = ( + AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXHeadersBuildPhase section */ + AFA4DBC928C0F7F5001AD7BE /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXHeadersBuildPhase section */ + +/* Begin PBXNativeTarget section */ + AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */ = { + isa = PBXNativeTarget; + buildConfigurationList = AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */; + buildPhases = ( + AFA4DBC928C0F7F5001AD7BE /* Headers */, + AFA4DBCA28C0F7F5001AD7BE /* Sources */, + AFA4DBCB28C0F7F5001AD7BE /* Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = LiveKitObjC; + packageProductDependencies = ( + AFA4DBD528C0F839001AD7BE /* LiveKit */, + AFA4DBDD28C121E6001AD7BE /* LiveKit */, + ); + productName = LiveKitObjC; + productReference = AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */; + productType = "com.apple.product-type.library.static"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + AFA4DBC528C0F7F5001AD7BE /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 1340; + TargetAttributes = { + AFA4DBCC28C0F7F5001AD7BE = { + CreatedOnToolsVersion = 13.4.1; + LastSwiftMigration = 1340; + }; + }; + }; + buildConfigurationList = AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */; + compatibilityVersion = "Xcode 13.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = AFA4DBC428C0F7F5001AD7BE; + packageReferences = ( + AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, + AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, + ); + productRefGroup = AFA4DBCE28C0F7F5001AD7BE /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXSourcesBuildPhase section */ + AFA4DBCA28C0F7F5001AD7BE /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */, + AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + AFA4DBCF28C0F7F5001AD7BE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 12.3; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + }; + name = Debug; + }; + AFA4DBD028C0F7F5001AD7BE /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 12.3; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = macosx; + }; + name = Release; + }; + AFA4DBD228C0F7F5001AD7BE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = NO; + EXECUTABLE_PREFIX = lib; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + "@loader_path/../Frameworks", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + AFA4DBD328C0F7F5001AD7BE /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = NO; + EXECUTABLE_PREFIX = lib; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + "@loader_path/../Frameworks", + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AFA4DBCF28C0F7F5001AD7BE /* Debug */, + AFA4DBD028C0F7F5001AD7BE /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AFA4DBD228C0F7F5001AD7BE /* Debug */, + AFA4DBD328C0F7F5001AD7BE /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + +/* Begin XCRemoteSwiftPackageReference section */ + AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/livekit/client-sdk-swift"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 1.0.0; + }; + }; + AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/livekit/client-sdk-swift"; + requirement = { + branch = main; + kind = branch; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + AFA4DBD528C0F839001AD7BE /* LiveKit */ = { + isa = XCSwiftPackageProductDependency; + package = AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; + productName = LiveKit; + }; + AFA4DBDD28C121E6001AD7BE /* LiveKit */ = { + isa = XCSwiftPackageProductDependency; + package = AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; + productName = LiveKit; + }; +/* End XCSwiftPackageProductDependency section */ + }; + rootObject = AFA4DBC528C0F7F5001AD7BE /* Project object */; +} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000000..919434a625 --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000000..18d981003d --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved new file mode 100644 index 0000000000..904852142c --- /dev/null +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -0,0 +1,50 @@ +{ + "pins" : [ + { + "identity" : "client-sdk-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/client-sdk-swift", + "state" : { + "branch" : "main", + "revision" : "e3de0d06e14825a7c9f1204f18f66898afb56cd6" + } + }, + { + "identity" : "promises", + "kind" : "remoteSourceControl", + "location" : "https://github.com/google/promises.git", + "state" : { + "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", + "version" : "2.1.1" + } + }, + { + "identity" : "specs", + "kind" : "remoteSourceControl", + "location" : "https://github.com/webrtc-sdk/Specs.git", + "state" : { + "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", + "version" : "104.5112.2" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", + "version" : "1.4.4" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", + "version" : "1.20.1" + } + } + ], + "version" : 2 +} diff --git a/LiveKitObjC/Room.swift b/LiveKitObjC/Room.swift new file mode 100644 index 0000000000..1019dc7894 --- /dev/null +++ b/LiveKitObjC/Room.swift @@ -0,0 +1,23 @@ +// +// LKRoom.swift +// LiveKitObjC +// +// Created by Antonio Scandurra on 01/09/22. +// + +import Foundation +import LiveKit + +@objc public class SLKRoom: NSObject, RoomDelegate { + lazy var room = Room(delegate: self) + + @objc public func connect( + url: String, + token: String, + callback: @convention(block) @escaping () -> Void + ) { + self.room.connect(url, token).then { room in + callback() + } + } +} diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 7adb758dde..e88423bed1 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,6 +1,8 @@ use std::{env, path::PathBuf, process::Command}; fn main() { + println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); + println!("cargo:rustc-link-lib=static=LiveKitObjC"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 97a5557c66..e456b032ea 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -48,6 +48,8 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); fn main() { + class!(LKRoom); + SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); gpui::App::new(()).unwrap().run(|cx| { @@ -97,60 +99,60 @@ impl ScreenCaptureView { let display_width: usize = msg_send![display, width]; let display_height: usize = msg_send![display, height]; let mut compression_buffer = BytesMut::new(); - let compression_session = CompressionSession::new( - display_width, - display_height, - kCMVideoCodecType_H264, - move |status, flags, sample_buffer| { - if status != 0 { - println!("error encoding frame, code: {}", status); - return; - } - let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + // let compression_session = CompressionSession::new( + // display_width, + // display_height, + // kCMVideoCodecType_H264, + // move |status, flags, sample_buffer| { + // if status != 0 { + // println!("error encoding frame, code: {}", status); + // return; + // } + // let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - let mut is_iframe = false; - let attachments = sample_buffer.attachments(); - if let Some(attachments) = attachments.first() { - is_iframe = attachments - .find(kCMSampleAttachmentKey_NotSync as CFStringRef) - .map_or(true, |not_sync| { - CFBooleanGetValue(*not_sync as CFBooleanRef) - }); - } + // let mut is_iframe = false; + // let attachments = sample_buffer.attachments(); + // if let Some(attachments) = attachments.first() { + // is_iframe = attachments + // .find(kCMSampleAttachmentKey_NotSync as CFStringRef) + // .map_or(true, |not_sync| { + // CFBooleanGetValue(*not_sync as CFBooleanRef) + // }); + // } - const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; - if is_iframe { - let format_description = sample_buffer.format_description(); - for ix in 0..format_description.h264_parameter_set_count() { - let parameter_set = - format_description.h264_parameter_set_at_index(ix).unwrap(); - compression_buffer.extend_from_slice(&START_CODE); - compression_buffer.extend_from_slice(parameter_set); - let nal_unit = compression_buffer.split(); - } - } + // const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; + // if is_iframe { + // let format_description = sample_buffer.format_description(); + // for ix in 0..format_description.h264_parameter_set_count() { + // let parameter_set = + // format_description.h264_parameter_set_at_index(ix).unwrap(); + // compression_buffer.extend_from_slice(&START_CODE); + // compression_buffer.extend_from_slice(parameter_set); + // let nal_unit = compression_buffer.split(); + // } + // } - let data = sample_buffer.data(); - let mut data = data.bytes(); + // let data = sample_buffer.data(); + // let mut data = data.bytes(); - const AVCC_HEADER_LENGTH: usize = 4; - while data.len() - AVCC_HEADER_LENGTH > 0 { - let nal_unit_len = match data.read_u32::() { - Ok(len) => len as usize, - Err(error) => { - log::error!("error decoding nal unit length: {}", error); - return; - } - }; - compression_buffer.extend_from_slice(&START_CODE); - compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); - data = &data[nal_unit_len..]; + // const AVCC_HEADER_LENGTH: usize = 4; + // while data.len() - AVCC_HEADER_LENGTH > 0 { + // let nal_unit_len = match data.read_u32::() { + // Ok(len) => len as usize, + // Err(error) => { + // log::error!("error decoding nal unit length: {}", error); + // return; + // } + // }; + // compression_buffer.extend_from_slice(&START_CODE); + // compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); + // data = &data[nal_unit_len..]; - let nal_unit = compression_buffer.split(); - } - }, - ) - .unwrap(); + // let nal_unit = compression_buffer.split(); + // } + // }, + // ) + // .unwrap(); let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); decl.add_ivar::<*mut c_void>("callback"); @@ -182,9 +184,9 @@ impl ScreenCaptureView { let timing_info = buffer.sample_timing_info(0).unwrap(); let image_buffer = buffer.image_buffer(); - compression_session - .encode_frame(&image_buffer, timing_info) - .unwrap(); + // compression_session + // .encode_frame(&image_buffer, timing_info) + // .unwrap(); *surface_tx.lock().borrow_mut() = Some(image_buffer); }) as Box; let callback = Box::into_raw(Box::new(callback)); From 46abb1cbf601ab70ccd972c3f8e9f20a06aa1c22 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 1 Sep 2022 19:55:02 +0200 Subject: [PATCH 33/89] WIP --- LiveKitObjC/LKRoom.m | 4 ++++ .../LiveKitObjC.xcodeproj/project.pbxproj | 23 +++++-------------- .../xcshareddata/swiftpm/Package.resolved | 4 ++-- crates/capture/build.rs | 1 + crates/capture/src/main.rs | 8 ++++++- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/LiveKitObjC/LKRoom.m b/LiveKitObjC/LKRoom.m index ea4d7d4402..02fc01d2f4 100644 --- a/LiveKitObjC/LKRoom.m +++ b/LiveKitObjC/LKRoom.m @@ -26,3 +26,7 @@ [self.room connectWithUrl:url token:token callback:callback]; } @end + +LKRoom* BuildLKRoom() { + return [[LKRoom alloc] init]; +} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj index 93053cb68d..9c7a6136b3 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -10,7 +10,6 @@ AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */; }; - AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBDD28C121E6001AD7BE /* LiveKit */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -26,7 +25,6 @@ buildActionMask = 2147483647; files = ( AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */, - AFA4DBDE28C121E6001AD7BE /* LiveKit in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -79,7 +77,6 @@ name = LiveKitObjC; packageProductDependencies = ( AFA4DBD528C0F839001AD7BE /* LiveKit */, - AFA4DBDD28C121E6001AD7BE /* LiveKit */, ); productName = LiveKitObjC; productReference = AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */; @@ -111,7 +108,6 @@ mainGroup = AFA4DBC428C0F7F5001AD7BE; packageReferences = ( AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, - AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, ); productRefGroup = AFA4DBCE28C0F7F5001AD7BE /* Products */; projectDirPath = ""; @@ -247,10 +243,13 @@ AFA4DBD228C0F7F5001AD7BE /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; + KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", @@ -267,10 +266,13 @@ AFA4DBD328C0F7F5001AD7BE /* Release */ = { isa = XCBuildConfiguration; buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; + KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", @@ -315,14 +317,6 @@ minimumVersion = 1.0.0; }; }; - AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/livekit/client-sdk-swift"; - requirement = { - branch = main; - kind = branch; - }; - }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ @@ -331,11 +325,6 @@ package = AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; productName = LiveKit; }; - AFA4DBDD28C121E6001AD7BE /* LiveKit */ = { - isa = XCSwiftPackageProductDependency; - package = AFA4DBDC28C121E6001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; - productName = LiveKit; - }; /* End XCSwiftPackageProductDependency section */ }; rootObject = AFA4DBC528C0F7F5001AD7BE /* Project object */; diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 904852142c..21468123fb 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift", "state" : { - "branch" : "main", - "revision" : "e3de0d06e14825a7c9f1204f18f66898afb56cd6" + "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", + "version" : "1.0.4" } }, { diff --git a/crates/capture/build.rs b/crates/capture/build.rs index e88423bed1..630cf1ed6b 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,6 +1,7 @@ use std::{env, path::PathBuf, process::Command}; fn main() { + println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); println!("cargo:rustc-link-lib=static=LiveKitObjC"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index e456b032ea..6a4c53aede 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -47,8 +47,14 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); +extern "C" { + fn BuildLKRoom() -> *const c_void; +} + fn main() { - class!(LKRoom); + unsafe { + BuildLKRoom(); + } SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); From 5fec784580e6d43689cbfc7046f96149c71e499b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 2 Sep 2022 11:42:39 +0200 Subject: [PATCH 34/89] Link Swift stdlib --- Cargo.lock | 2 + .../LiveKitObjC.xcodeproj/project.pbxproj | 8 --- crates/capture/Cargo.toml | 2 + crates/capture/build.rs | 56 +++++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60e54aeb9f..bf60a08367 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,6 +770,8 @@ dependencies = [ "objc", "parking_lot 0.11.2", "postage", + "serde", + "serde_json", "simplelog", ] diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj index 9c7a6136b3..db7445a28f 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -243,11 +243,8 @@ AFA4DBD228C0F7F5001AD7BE /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; - BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( @@ -256,7 +253,6 @@ "@loader_path/../Frameworks", ); PRODUCT_NAME = "$(TARGET_NAME)"; - SKIP_INSTALL = YES; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; @@ -266,11 +262,8 @@ AFA4DBD328C0F7F5001AD7BE /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; - BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = NO; EXECUTABLE_PREFIX = lib; KEEP_PRIVATE_EXTERNS = NO; LD_RUNPATH_SEARCH_PATHS = ( @@ -279,7 +272,6 @@ "@loader_path/../Frameworks", ); PRODUCT_NAME = "$(TARGET_NAME)"; - SKIP_INSTALL = YES; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_VERSION = 5.0; }; diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index cdae75fc2b..339198f764 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -25,3 +25,5 @@ simplelog = "0.9" [build-dependencies] bindgen = "0.59.2" +serde = { version = "1.0", features = ["derive", "rc"] } +serde_json = { version = "1.0", features = ["preserve_order"] } \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 630cf1ed6b..b16ed18960 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,6 +1,62 @@ +use serde::Deserialize; use std::{env, path::PathBuf, process::Command}; +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftTargetInfo { + pub triple: String, + pub unversioned_triple: String, + pub module_triple: String, + pub swift_runtime_compatibility_version: String, + #[serde(rename = "librariesRequireRPath")] + pub libraries_require_rpath: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftPaths { + pub runtime_library_paths: Vec, + pub runtime_library_import_paths: Vec, + pub runtime_resource_path: String, +} + +#[derive(Debug, Deserialize)] +pub struct SwiftTarget { + pub target: SwiftTargetInfo, + pub paths: SwiftPaths, +} + +const MACOS_TARGET_VERSION: &str = "12"; + +pub fn link_swift_libs() { + let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + if arch == "aarch64" { + arch = "arm64".into(); + } + let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); + + let swift_target_info_str = Command::new("swift") + .args(&["-target", &target, "-print-target-info"]) + .output() + .unwrap() + .stdout; + + let swift_target_info: SwiftTarget = serde_json::from_slice(&swift_target_info_str).unwrap(); + if swift_target_info.target.libraries_require_rpath { + panic!("Libraries require RPath! Change minimum MacOS value to fix.") + } + + swift_target_info + .paths + .runtime_library_paths + .iter() + .for_each(|path| { + println!("cargo:rustc-link-search=native={}", path); + }); +} + fn main() { + link_swift_libs(); println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); println!("cargo:rustc-link-lib=static=LiveKitObjC"); From 52f32b50b296fdb693896057510b31268ed6985a Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 2 Sep 2022 18:35:16 +0200 Subject: [PATCH 35/89] WIP: Talk to Swift via C without involving Objective-C Co-Authored-By: Nathan Sobo --- LiveKitObjC/LKRoom.m | 32 ------------------ .../LiveKitObjC.xcodeproj/project.pbxproj | 12 +++---- LiveKitObjC/Room.swift | 26 +++++++++++++-- crates/capture/build.rs | 4 ++- crates/capture/src/main.rs | 33 +++++++++++++++---- 5 files changed, 60 insertions(+), 47 deletions(-) delete mode 100644 LiveKitObjC/LKRoom.m diff --git a/LiveKitObjC/LKRoom.m b/LiveKitObjC/LKRoom.m deleted file mode 100644 index 02fc01d2f4..0000000000 --- a/LiveKitObjC/LKRoom.m +++ /dev/null @@ -1,32 +0,0 @@ -// -// LKRoom.m -// LiveKitObjC -// -// Created by Antonio Scandurra on 01/09/22. -// - -#import -#import - -@interface LKRoom: NSObject { -} - -@property (nonatomic, retain) SLKRoom* room; -@end - -@implementation LKRoom --(id)init { - if (self = [super init]) { - self.room = [[SLKRoom alloc] init]; - } - return self; -} - --(void)connectWithURL:(NSString *)url token:(NSString *)token callback:(void(^)(void))callback { - [self.room connectWithUrl:url token:token callback:callback]; -} -@end - -LKRoom* BuildLKRoom() { - return [[LKRoom alloc] init]; -} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj index db7445a28f..c1293713ec 100644 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj @@ -9,14 +9,12 @@ /* Begin PBXBuildFile section */ AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; - AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libLiveKitObjC.a; sourceTree = BUILT_PRODUCTS_DIR; }; AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "LiveKitObjC-Bridging-Header.h"; sourceTree = ""; }; AFA4DBD828C0F87F001AD7BE /* Room.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Room.swift; sourceTree = ""; }; - AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = LKRoom.m; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -34,7 +32,6 @@ AFA4DBC428C0F7F5001AD7BE = { isa = PBXGroup; children = ( - AFA4DBDA28C0FBC0001AD7BE /* LKRoom.m */, AFA4DBD828C0F87F001AD7BE /* Room.swift */, AFA4DBCE28C0F7F5001AD7BE /* Products */, AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */, @@ -124,7 +121,6 @@ buildActionMask = 2147483647; files = ( AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */, - AFA4DBDB28C0FBC0001AD7BE /* LKRoom.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -245,13 +241,15 @@ buildSettings = { CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = YES; EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = NO; + KEEP_PRIVATE_EXTERNS = YES; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", "@loader_path/../Frameworks", ); + PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; @@ -264,13 +262,15 @@ buildSettings = { CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; + DEFINES_MODULE = YES; EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = NO; + KEEP_PRIVATE_EXTERNS = YES; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", "@loader_path/../Frameworks", ); + PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; SWIFT_VERSION = 5.0; diff --git a/LiveKitObjC/Room.swift b/LiveKitObjC/Room.swift index 1019dc7894..9397b82649 100644 --- a/LiveKitObjC/Room.swift +++ b/LiveKitObjC/Room.swift @@ -8,10 +8,18 @@ import Foundation import LiveKit -@objc public class SLKRoom: NSObject, RoomDelegate { +public class LKRoom: RoomDelegate { lazy var room = Room(delegate: self) - @objc public func connect( + init() { + print("INIT!\n"); + } + + deinit { + print("DEINIT!\n"); + } + + public func connect( url: String, token: String, callback: @convention(block) @escaping () -> Void @@ -20,4 +28,18 @@ import LiveKit callback() } } + } + + +@_cdecl("LKRoomCreate") +public func LKRoomCreate() -> UnsafeMutableRawPointer { + Unmanaged.passRetained(LKRoom()).toOpaque() +} + +@_cdecl("LKRoomDestroy") +public func LKRoomDestroy(ptr: UnsafeRawPointer) { + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); +} + + diff --git a/crates/capture/build.rs b/crates/capture/build.rs index b16ed18960..58cf51203a 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -58,9 +58,11 @@ pub fn link_swift_libs() { fn main() { link_swift_libs(); println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); - println!("cargo:rustc-link-search=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug"); + println!("cargo:rustc-link-search=native=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/libs"); + println!("cargo:rustc-link-search=framework=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/frameworks"); println!("cargo:rustc-link-lib=static=LiveKitObjC"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); + println!("cargo:rustc-link-lib=framework=WebRTC"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); let sdk_path = String::from_utf8( diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 6a4c53aede..f952f3ba22 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -10,7 +10,7 @@ use cocoa::{ foundation::{NSArray, NSString, NSUInteger}, }; use core_foundation::{ - base::TCFType, + base::{CFRelease, TCFType}, number::{CFBooleanGetValue, CFBooleanRef, CFNumberRef}, string::CFStringRef, }; @@ -35,7 +35,7 @@ use objc::{ class, declare::ClassDecl, msg_send, - runtime::{Object, Sel}, + runtime::{Class, Object, Sel}, sel, sel_impl, }; use parking_lot::Mutex; @@ -48,13 +48,34 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); extern "C" { - fn BuildLKRoom() -> *const c_void; + fn LKRoomCreate() -> *const c_void; + fn LKRoomDestroy(ptr: *const c_void); +} + +struct Room { + native_room: *const c_void, +} + +impl Room { + pub fn new() -> Self { + Self { + native_room: unsafe { LKRoomCreate() }, + } + } +} + +impl Drop for Room { + fn drop(&mut self) { + unsafe { LKRoomDestroy(self.native_room) } + } } fn main() { - unsafe { - BuildLKRoom(); - } + println!("Creating room..."); + let room = Room::new(); + + println!("Dropping room..."); + drop(room); SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); From 4bcc008cbf4a9e87ddd069867dc7b01636d313c3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 2 Sep 2022 12:56:38 -0600 Subject: [PATCH 36/89] WIP: Start on live_kit crate that uses a C-based bridge --- Cargo.lock | 11 ++- crates/capture/Cargo.toml | 3 +- crates/capture/build.rs | 69 ++------------ crates/capture/src/main.rs | 24 +---- crates/live_kit/Cargo.toml | 15 +++ crates/live_kit/LiveKitBridge/.gitignore | 9 ++ .../live_kit/LiveKitBridge/Package.resolved | 50 ++++++++++ crates/live_kit/LiveKitBridge/Package.swift | 27 ++++++ crates/live_kit/LiveKitBridge/README.md | 3 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 12 +++ crates/live_kit/build.rs | 95 +++++++++++++++++++ crates/live_kit/src/live_kit.rs | 24 +++++ 12 files changed, 254 insertions(+), 88 deletions(-) create mode 100644 crates/live_kit/Cargo.toml create mode 100644 crates/live_kit/LiveKitBridge/.gitignore create mode 100644 crates/live_kit/LiveKitBridge/Package.resolved create mode 100644 crates/live_kit/LiveKitBridge/Package.swift create mode 100644 crates/live_kit/LiveKitBridge/README.md create mode 100644 crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift create mode 100644 crates/live_kit/build.rs create mode 100644 crates/live_kit/src/live_kit.rs diff --git a/Cargo.lock b/Cargo.lock index bf60a08367..1e176cda42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,13 +765,12 @@ dependencies = [ "foreign-types", "futures", "gpui", + "live_kit", "log", "media", "objc", "parking_lot 0.11.2", "postage", - "serde", - "serde_json", "simplelog", ] @@ -2918,6 +2917,14 @@ dependencies = [ "rand_chacha 0.3.1", ] +[[package]] +name = "live_kit" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "lock_api" version = "0.4.7" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index 339198f764..d572c6077c 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -6,6 +6,7 @@ description = "An example of screen capture" [dependencies] gpui = { path = "../gpui" } +live_kit = { path = "../live_kit" } media = { path = "../media" } anyhow = "1.0.38" @@ -25,5 +26,3 @@ simplelog = "0.9" [build-dependencies] bindgen = "0.59.2" -serde = { version = "1.0", features = ["derive", "rc"] } -serde_json = { version = "1.0", features = ["preserve_order"] } \ No newline at end of file diff --git a/crates/capture/build.rs b/crates/capture/build.rs index 58cf51203a..e2ffda4c53 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,68 +1,15 @@ -use serde::Deserialize; use std::{env, path::PathBuf, process::Command}; -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftTargetInfo { - pub triple: String, - pub unversioned_triple: String, - pub module_triple: String, - pub swift_runtime_compatibility_version: String, - #[serde(rename = "librariesRequireRPath")] - pub libraries_require_rpath: bool, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftPaths { - pub runtime_library_paths: Vec, - pub runtime_library_import_paths: Vec, - pub runtime_resource_path: String, -} - -#[derive(Debug, Deserialize)] -pub struct SwiftTarget { - pub target: SwiftTargetInfo, - pub paths: SwiftPaths, -} - -const MACOS_TARGET_VERSION: &str = "12"; - -pub fn link_swift_libs() { - let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - if arch == "aarch64" { - arch = "arm64".into(); - } - let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); - - let swift_target_info_str = Command::new("swift") - .args(&["-target", &target, "-print-target-info"]) - .output() - .unwrap() - .stdout; - - let swift_target_info: SwiftTarget = serde_json::from_slice(&swift_target_info_str).unwrap(); - if swift_target_info.target.libraries_require_rpath { - panic!("Libraries require RPath! Change minimum MacOS value to fix.") - } - - swift_target_info - .paths - .runtime_library_paths - .iter() - .for_each(|path| { - println!("cargo:rustc-link-search=native={}", path); - }); -} - fn main() { - link_swift_libs(); - println!("cargo:rerun-if-changed=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/Debug/libLiveKitObjC.a"); - println!("cargo:rustc-link-search=native=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/libs"); - println!("cargo:rustc-link-search=framework=/Users/as-cii/Library/Developer/Xcode/DerivedData/LiveKitObjC-ftgpxknhsgkrocbhhgjkyyvkgkbj/Build/Products/frameworks"); - println!("cargo:rustc-link-lib=static=LiveKitObjC"); + println!( + "cargo:rustc-link-search=framework={}", + "crates/live_kit/LiveKitBridge/.build/arm64-apple-macosx/debug" + ); + + // Find frameworks as a sibling of the executable at runtime + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); - println!("cargo:rustc-link-lib=framework=WebRTC"); println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); let sdk_path = String::from_utf8( diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index f952f3ba22..75df9be368 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -22,6 +22,7 @@ use gpui::{ platform::current::Surface, Menu, MenuItem, ViewContext, }; +use live_kit::Room; use log::LevelFilter; use media::{ core_media::{ @@ -47,29 +48,6 @@ const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); -extern "C" { - fn LKRoomCreate() -> *const c_void; - fn LKRoomDestroy(ptr: *const c_void); -} - -struct Room { - native_room: *const c_void, -} - -impl Room { - pub fn new() -> Self { - Self { - native_room: unsafe { LKRoomCreate() }, - } - } -} - -impl Drop for Room { - fn drop(&mut self) { - unsafe { LKRoomDestroy(self.native_room) } - } -} - fn main() { println!("Creating room..."); let room = Room::new(); diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml new file mode 100644 index 0000000000..15f7683456 --- /dev/null +++ b/crates/live_kit/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "live_kit" +version = "0.1.0" +edition = "2021" +description = "Bindings to LiveKit Swift client SDK" + +[lib] +path = "src/live_kit.rs" +doctest = false + +[dependencies] + +[build-dependencies] +serde = { version = "1.0", features = ["derive", "rc"] } +serde_json = { version = "1.0", features = ["preserve_order"] } diff --git a/crates/live_kit/LiveKitBridge/.gitignore b/crates/live_kit/LiveKitBridge/.gitignore new file mode 100644 index 0000000000..3b29812086 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +/.build +/Packages +/*.xcodeproj +xcuserdata/ +DerivedData/ +.swiftpm/config/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved new file mode 100644 index 0000000000..df0b5d6243 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -0,0 +1,50 @@ +{ + "pins" : [ + { + "identity" : "client-sdk-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/client-sdk-swift.git", + "state" : { + "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", + "version" : "1.0.4" + } + }, + { + "identity" : "promises", + "kind" : "remoteSourceControl", + "location" : "https://github.com/google/promises.git", + "state" : { + "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", + "version" : "2.1.1" + } + }, + { + "identity" : "specs", + "kind" : "remoteSourceControl", + "location" : "https://github.com/webrtc-sdk/Specs.git", + "state" : { + "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", + "version" : "104.5112.2" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", + "version" : "1.4.4" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", + "version" : "1.20.1" + } + } + ], + "version" : 2 +} diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift new file mode 100644 index 0000000000..9e7039998d --- /dev/null +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -0,0 +1,27 @@ +// swift-tools-version: 5.6 + +import PackageDescription + +let package = Package( + name: "LiveKitBridge", + platforms: [ + .macOS(.v10_15) + ], + products: [ + // Products define the executables and libraries a package produces, and make them visible to other packages. + .library( + name: "LiveKitBridge", + type: .static, + targets: ["LiveKitBridge"]), + ], + dependencies: [ + .package(url: "https://github.com/livekit/client-sdk-swift.git", from: "1.0.0"), + ], + targets: [ + // Targets are the basic building blocks of a package. A target can define a module or a test suite. + // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "LiveKitBridge", + dependencies: [.product(name: "LiveKit", package: "client-sdk-swift")]), + ] +) diff --git a/crates/live_kit/LiveKitBridge/README.md b/crates/live_kit/LiveKitBridge/README.md new file mode 100644 index 0000000000..b982c67286 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/README.md @@ -0,0 +1,3 @@ +# LiveKitBridge + +A description of this package. diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift new file mode 100644 index 0000000000..11df49e6c6 --- /dev/null +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -0,0 +1,12 @@ +import Foundation +import LiveKit + +@_cdecl("LKRoomCreate") +public func LKRoomCreate() -> UnsafeMutableRawPointer { + Unmanaged.passRetained(Room()).toOpaque() +} + +@_cdecl("LKRoomDestroy") +public func LKRoomDestroy(ptr: UnsafeRawPointer) { + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); +} diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs new file mode 100644 index 0000000000..dab126d12c --- /dev/null +++ b/crates/live_kit/build.rs @@ -0,0 +1,95 @@ +use serde::Deserialize; +use std::{env, path::PathBuf, process::Command}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftTargetInfo { + pub triple: String, + pub unversioned_triple: String, + pub module_triple: String, + pub swift_runtime_compatibility_version: String, + #[serde(rename = "librariesRequireRPath")] + pub libraries_require_rpath: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SwiftPaths { + pub runtime_library_paths: Vec, + pub runtime_library_import_paths: Vec, + pub runtime_resource_path: String, +} + +#[derive(Debug, Deserialize)] +pub struct SwiftTarget { + pub target: SwiftTargetInfo, + pub paths: SwiftPaths, +} + +const MACOS_TARGET_VERSION: &str = "12"; + +fn main() { + build_bridge(); + link_swift_stdlib(); +} + +fn build_bridge() { + let profile = env::var("PROFILE").unwrap(); + let package_name = "LiveKitBridge"; + let package_root = env::current_dir().unwrap().join(package_name); + if !Command::new("swift") + .args(&["build", "-c", &profile]) + .current_dir(&package_root) + .status() + .unwrap() + .success() + { + panic!( + "Failed to compile swift package in {}", + package_root.display() + ); + } + + let swift_target_info = get_swift_target(); + let swift_out_dir_path = format!( + "{}/.build/{}/{}", + package_root.display(), + swift_target_info.target.unversioned_triple, + profile + ); + + println!("cargo:rustc-link-search=native={}", swift_out_dir_path); + println!( + "cargo:rustc-link-search=framework={}", + "/Users/nathan/src/zed/crates/live_kit/frameworks" + ); + println!("cargo:rustc-link-lib=static={}", package_name); + println!("cargo:rustc-link-lib=framework=WebRTC"); +} + +fn link_swift_stdlib() { + let target = get_swift_target(); + if target.target.libraries_require_rpath { + panic!("Libraries require RPath! Change minimum MacOS value to fix.") + } + + target.paths.runtime_library_paths.iter().for_each(|path| { + println!("cargo:rustc-link-search=native={}", path); + }); +} + +fn get_swift_target() -> SwiftTarget { + let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + if arch == "aarch64" { + arch = "arm64".into(); + } + let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); + + let swift_target_info_str = Command::new("swift") + .args(&["-target", &target, "-print-target-info"]) + .output() + .unwrap() + .stdout; + + serde_json::from_slice(&swift_target_info_str).unwrap() +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs new file mode 100644 index 0000000000..3dcaaa74ed --- /dev/null +++ b/crates/live_kit/src/live_kit.rs @@ -0,0 +1,24 @@ +use std::ffi::c_void; + +extern "C" { + fn LKRoomCreate() -> *const c_void; + fn LKRoomDestroy(ptr: *const c_void); +} + +pub struct Room { + native_room: *const c_void, +} + +impl Room { + pub fn new() -> Self { + Self { + native_room: unsafe { LKRoomCreate() }, + } + } +} + +impl Drop for Room { + fn drop(&mut self) { + unsafe { LKRoomDestroy(self.native_room) } + } +} From 3c2566fc11ed3c575e0379f42deb5935c1d8f779 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 08:56:49 -0600 Subject: [PATCH 37/89] Copy WebRTC.framework when building livekit crate We determine the location of the target executable directory in a somewhat hacky way, but it seems reasonably stable. Co-Authored-By: Antonio Scandurra --- crates/capture/build.rs | 7 +--- crates/live_kit/build.rs | 88 +++++++++++++++++++++++++++------------- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/crates/capture/build.rs b/crates/capture/build.rs index e2ffda4c53..c637f2ce9b 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,12 +1,7 @@ use std::{env, path::PathBuf, process::Command}; fn main() { - println!( - "cargo:rustc-link-search=framework={}", - "crates/live_kit/LiveKitBridge/.build/arm64-apple-macosx/debug" - ); - - // Find frameworks as a sibling of the executable at runtime + // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index dab126d12c..ca9c2d5cb9 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -29,53 +29,70 @@ pub struct SwiftTarget { const MACOS_TARGET_VERSION: &str = "12"; fn main() { - build_bridge(); - link_swift_stdlib(); + let swift_target = get_swift_target(); + + build_bridge(&swift_target); + link_swift_stdlib(&swift_target); + link_webrtc_framework(&swift_target); } -fn build_bridge() { - let profile = env::var("PROFILE").unwrap(); - let package_name = "LiveKitBridge"; - let package_root = env::current_dir().unwrap().join(package_name); +fn build_bridge(swift_target: &SwiftTarget) { + let swift_package_root = swift_package_root(); if !Command::new("swift") - .args(&["build", "-c", &profile]) - .current_dir(&package_root) + .args(&["build", "-c", &env::var("PROFILE").unwrap()]) + .current_dir(&swift_package_root) .status() .unwrap() .success() { panic!( "Failed to compile swift package in {}", - package_root.display() + swift_package_root.display() ); } - let swift_target_info = get_swift_target(); - let swift_out_dir_path = format!( - "{}/.build/{}/{}", - package_root.display(), - swift_target_info.target.unversioned_triple, - profile - ); - - println!("cargo:rustc-link-search=native={}", swift_out_dir_path); println!( - "cargo:rustc-link-search=framework={}", - "/Users/nathan/src/zed/crates/live_kit/frameworks" + "cargo:rustc-link-search=native={}", + swift_target.out_dir_path().display() ); - println!("cargo:rustc-link-lib=static={}", package_name); - println!("cargo:rustc-link-lib=framework=WebRTC"); + println!("cargo:rustc-link-lib=static={}", SWIFT_PACKAGE_NAME); } -fn link_swift_stdlib() { - let target = get_swift_target(); - if target.target.libraries_require_rpath { +fn link_swift_stdlib(swift_target: &SwiftTarget) { + if swift_target.target.libraries_require_rpath { panic!("Libraries require RPath! Change minimum MacOS value to fix.") } - target.paths.runtime_library_paths.iter().for_each(|path| { - println!("cargo:rustc-link-search=native={}", path); - }); + swift_target + .paths + .runtime_library_paths + .iter() + .for_each(|path| { + println!("cargo:rustc-link-search=native={}", path); + }); +} + +fn link_webrtc_framework(swift_target: &SwiftTarget) { + let swift_out_dir_path = swift_target.out_dir_path(); + println!("cargo:rustc-link-lib=framework=WebRTC"); + println!( + "cargo:rustc-link-search=framework={}", + swift_out_dir_path.display() + ); + + let source_path = swift_out_dir_path.join("WebRTC.framework"); + let target_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); + assert!( + Command::new("cp") + .arg("-r") + .args(&[&source_path, &target_path]) + .status() + .unwrap() + .success(), + "could not copy WebRTC.framework from {:?} to {:?}", + source_path, + target_path + ); } fn get_swift_target() -> SwiftTarget { @@ -93,3 +110,18 @@ fn get_swift_target() -> SwiftTarget { serde_json::from_slice(&swift_target_info_str).unwrap() } + +const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; + +fn swift_package_root() -> PathBuf { + env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) +} + +impl SwiftTarget { + fn out_dir_path(&self) -> PathBuf { + swift_package_root() + .join(".build") + .join(&self.target.unversioned_triple) + .join(env::var("PROFILE").unwrap()) + } +} From 45d83b557b77ee7cc7cd255aafbc333277ad82b3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 09:05:53 -0600 Subject: [PATCH 38/89] Remove LiveKitObjC experiment Co-Authored-By: Antonio Scandurra --- LiveKitObjC/LiveKitObjC-Bridging-Header.h | 3 - .../LiveKitObjC.xcodeproj/project.pbxproj | 323 ------------------ .../contents.xcworkspacedata | 7 - .../xcshareddata/IDEWorkspaceChecks.plist | 8 - .../xcshareddata/swiftpm/Package.resolved | 50 --- LiveKitObjC/Room.swift | 45 --- 6 files changed, 436 deletions(-) delete mode 100644 LiveKitObjC/LiveKitObjC-Bridging-Header.h delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist delete mode 100644 LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved delete mode 100644 LiveKitObjC/Room.swift diff --git a/LiveKitObjC/LiveKitObjC-Bridging-Header.h b/LiveKitObjC/LiveKitObjC-Bridging-Header.h deleted file mode 100644 index e11d920b12..0000000000 --- a/LiveKitObjC/LiveKitObjC-Bridging-Header.h +++ /dev/null @@ -1,3 +0,0 @@ -// -// Use this file to import your target's public headers that you would like to expose to Swift. -// diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj b/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj deleted file mode 100644 index c1293713ec..0000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.pbxproj +++ /dev/null @@ -1,323 +0,0 @@ -// !$*UTF8*$! -{ - archiveVersion = 1; - classes = { - }; - objectVersion = 55; - objects = { - -/* Begin PBXBuildFile section */ - AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = AFA4DBD528C0F839001AD7BE /* LiveKit */; }; - AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFA4DBD828C0F87F001AD7BE /* Room.swift */; }; -/* End PBXBuildFile section */ - -/* Begin PBXFileReference section */ - AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libLiveKitObjC.a; sourceTree = BUILT_PRODUCTS_DIR; }; - AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "LiveKitObjC-Bridging-Header.h"; sourceTree = ""; }; - AFA4DBD828C0F87F001AD7BE /* Room.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Room.swift; sourceTree = ""; }; -/* End PBXFileReference section */ - -/* Begin PBXFrameworksBuildPhase section */ - AFA4DBCB28C0F7F5001AD7BE /* Frameworks */ = { - isa = PBXFrameworksBuildPhase; - buildActionMask = 2147483647; - files = ( - AFA4DBD628C0F839001AD7BE /* LiveKit in Frameworks */, - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXFrameworksBuildPhase section */ - -/* Begin PBXGroup section */ - AFA4DBC428C0F7F5001AD7BE = { - isa = PBXGroup; - children = ( - AFA4DBD828C0F87F001AD7BE /* Room.swift */, - AFA4DBCE28C0F7F5001AD7BE /* Products */, - AFA4DBD728C0F87F001AD7BE /* LiveKitObjC-Bridging-Header.h */, - ); - sourceTree = ""; - }; - AFA4DBCE28C0F7F5001AD7BE /* Products */ = { - isa = PBXGroup; - children = ( - AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */, - ); - name = Products; - sourceTree = ""; - }; -/* End PBXGroup section */ - -/* Begin PBXHeadersBuildPhase section */ - AFA4DBC928C0F7F5001AD7BE /* Headers */ = { - isa = PBXHeadersBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXHeadersBuildPhase section */ - -/* Begin PBXNativeTarget section */ - AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */ = { - isa = PBXNativeTarget; - buildConfigurationList = AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */; - buildPhases = ( - AFA4DBC928C0F7F5001AD7BE /* Headers */, - AFA4DBCA28C0F7F5001AD7BE /* Sources */, - AFA4DBCB28C0F7F5001AD7BE /* Frameworks */, - ); - buildRules = ( - ); - dependencies = ( - ); - name = LiveKitObjC; - packageProductDependencies = ( - AFA4DBD528C0F839001AD7BE /* LiveKit */, - ); - productName = LiveKitObjC; - productReference = AFA4DBCD28C0F7F5001AD7BE /* libLiveKitObjC.a */; - productType = "com.apple.product-type.library.static"; - }; -/* End PBXNativeTarget section */ - -/* Begin PBXProject section */ - AFA4DBC528C0F7F5001AD7BE /* Project object */ = { - isa = PBXProject; - attributes = { - BuildIndependentTargetsInParallel = 1; - LastUpgradeCheck = 1340; - TargetAttributes = { - AFA4DBCC28C0F7F5001AD7BE = { - CreatedOnToolsVersion = 13.4.1; - LastSwiftMigration = 1340; - }; - }; - }; - buildConfigurationList = AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */; - compatibilityVersion = "Xcode 13.0"; - developmentRegion = en; - hasScannedForEncodings = 0; - knownRegions = ( - en, - Base, - ); - mainGroup = AFA4DBC428C0F7F5001AD7BE; - packageReferences = ( - AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */, - ); - productRefGroup = AFA4DBCE28C0F7F5001AD7BE /* Products */; - projectDirPath = ""; - projectRoot = ""; - targets = ( - AFA4DBCC28C0F7F5001AD7BE /* LiveKitObjC */, - ); - }; -/* End PBXProject section */ - -/* Begin PBXSourcesBuildPhase section */ - AFA4DBCA28C0F7F5001AD7BE /* Sources */ = { - isa = PBXSourcesBuildPhase; - buildActionMask = 2147483647; - files = ( - AFA4DBD928C0F87F001AD7BE /* Room.swift in Sources */, - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXSourcesBuildPhase section */ - -/* Begin XCBuildConfiguration section */ - AFA4DBCF28C0F7F5001AD7BE /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - ALWAYS_SEARCH_USER_PATHS = NO; - CLANG_ANALYZER_NONNULL = YES; - CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; - CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; - CLANG_ENABLE_OBJC_WEAK = YES; - CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; - CLANG_WARN_BOOL_CONVERSION = YES; - CLANG_WARN_COMMA = YES; - CLANG_WARN_CONSTANT_CONVERSION = YES; - CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; - CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; - CLANG_WARN_DOCUMENTATION_COMMENTS = YES; - CLANG_WARN_EMPTY_BODY = YES; - CLANG_WARN_ENUM_CONVERSION = YES; - CLANG_WARN_INFINITE_RECURSION = YES; - CLANG_WARN_INT_CONVERSION = YES; - CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; - CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; - CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; - CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; - CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; - CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; - CLANG_WARN_STRICT_PROTOTYPES = YES; - CLANG_WARN_SUSPICIOUS_MOVE = YES; - CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; - CLANG_WARN_UNREACHABLE_CODE = YES; - CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; - COPY_PHASE_STRIP = NO; - DEBUG_INFORMATION_FORMAT = dwarf; - ENABLE_STRICT_OBJC_MSGSEND = YES; - ENABLE_TESTABILITY = YES; - GCC_C_LANGUAGE_STANDARD = gnu11; - GCC_DYNAMIC_NO_PIC = NO; - GCC_NO_COMMON_BLOCKS = YES; - GCC_OPTIMIZATION_LEVEL = 0; - GCC_PREPROCESSOR_DEFINITIONS = ( - "DEBUG=1", - "$(inherited)", - ); - GCC_WARN_64_TO_32_BIT_CONVERSION = YES; - GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; - GCC_WARN_UNDECLARED_SELECTOR = YES; - GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; - GCC_WARN_UNUSED_FUNCTION = YES; - GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 12.3; - MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; - MTL_FAST_MATH = YES; - ONLY_ACTIVE_ARCH = YES; - SDKROOT = macosx; - }; - name = Debug; - }; - AFA4DBD028C0F7F5001AD7BE /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - ALWAYS_SEARCH_USER_PATHS = NO; - CLANG_ANALYZER_NONNULL = YES; - CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; - CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; - CLANG_ENABLE_OBJC_WEAK = YES; - CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; - CLANG_WARN_BOOL_CONVERSION = YES; - CLANG_WARN_COMMA = YES; - CLANG_WARN_CONSTANT_CONVERSION = YES; - CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; - CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; - CLANG_WARN_DOCUMENTATION_COMMENTS = YES; - CLANG_WARN_EMPTY_BODY = YES; - CLANG_WARN_ENUM_CONVERSION = YES; - CLANG_WARN_INFINITE_RECURSION = YES; - CLANG_WARN_INT_CONVERSION = YES; - CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; - CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; - CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; - CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; - CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; - CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; - CLANG_WARN_STRICT_PROTOTYPES = YES; - CLANG_WARN_SUSPICIOUS_MOVE = YES; - CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; - CLANG_WARN_UNREACHABLE_CODE = YES; - CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; - COPY_PHASE_STRIP = NO; - DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - ENABLE_NS_ASSERTIONS = NO; - ENABLE_STRICT_OBJC_MSGSEND = YES; - GCC_C_LANGUAGE_STANDARD = gnu11; - GCC_NO_COMMON_BLOCKS = YES; - GCC_WARN_64_TO_32_BIT_CONVERSION = YES; - GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; - GCC_WARN_UNDECLARED_SELECTOR = YES; - GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; - GCC_WARN_UNUSED_FUNCTION = YES; - GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 12.3; - MTL_ENABLE_DEBUG_INFO = NO; - MTL_FAST_MATH = YES; - SDKROOT = macosx; - }; - name = Release; - }; - AFA4DBD228C0F7F5001AD7BE /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - CLANG_ENABLE_MODULES = YES; - CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = YES; - EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = YES; - LD_RUNPATH_SEARCH_PATHS = ( - "$(inherited)", - "@executable_path/../Frameworks", - "@loader_path/../Frameworks", - ); - PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; - PRODUCT_NAME = "$(TARGET_NAME)"; - SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; - SWIFT_OPTIMIZATION_LEVEL = "-Onone"; - SWIFT_VERSION = 5.0; - }; - name = Debug; - }; - AFA4DBD328C0F7F5001AD7BE /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - CLANG_ENABLE_MODULES = YES; - CODE_SIGN_STYLE = Automatic; - DEFINES_MODULE = YES; - EXECUTABLE_PREFIX = lib; - KEEP_PRIVATE_EXTERNS = YES; - LD_RUNPATH_SEARCH_PATHS = ( - "$(inherited)", - "@executable_path/../Frameworks", - "@loader_path/../Frameworks", - ); - PRESERVE_DEAD_CODE_INITS_AND_TERMS = YES; - PRODUCT_NAME = "$(TARGET_NAME)"; - SWIFT_OBJC_BRIDGING_HEADER = "LiveKitObjC-Bridging-Header.h"; - SWIFT_VERSION = 5.0; - }; - name = Release; - }; -/* End XCBuildConfiguration section */ - -/* Begin XCConfigurationList section */ - AFA4DBC828C0F7F5001AD7BE /* Build configuration list for PBXProject "LiveKitObjC" */ = { - isa = XCConfigurationList; - buildConfigurations = ( - AFA4DBCF28C0F7F5001AD7BE /* Debug */, - AFA4DBD028C0F7F5001AD7BE /* Release */, - ); - defaultConfigurationIsVisible = 0; - defaultConfigurationName = Release; - }; - AFA4DBD128C0F7F5001AD7BE /* Build configuration list for PBXNativeTarget "LiveKitObjC" */ = { - isa = XCConfigurationList; - buildConfigurations = ( - AFA4DBD228C0F7F5001AD7BE /* Debug */, - AFA4DBD328C0F7F5001AD7BE /* Release */, - ); - defaultConfigurationIsVisible = 0; - defaultConfigurationName = Release; - }; -/* End XCConfigurationList section */ - -/* Begin XCRemoteSwiftPackageReference section */ - AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/livekit/client-sdk-swift"; - requirement = { - kind = upToNextMajorVersion; - minimumVersion = 1.0.0; - }; - }; -/* End XCRemoteSwiftPackageReference section */ - -/* Begin XCSwiftPackageProductDependency section */ - AFA4DBD528C0F839001AD7BE /* LiveKit */ = { - isa = XCSwiftPackageProductDependency; - package = AFA4DBD428C0F839001AD7BE /* XCRemoteSwiftPackageReference "client-sdk-swift" */; - productName = LiveKit; - }; -/* End XCSwiftPackageProductDependency section */ - }; - rootObject = AFA4DBC528C0F7F5001AD7BE /* Project object */; -} diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata deleted file mode 100644 index 919434a625..0000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/contents.xcworkspacedata +++ /dev/null @@ -1,7 +0,0 @@ - - - - - diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist deleted file mode 100644 index 18d981003d..0000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +++ /dev/null @@ -1,8 +0,0 @@ - - - - - IDEDidComputeMac32BitWarning - - - diff --git a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved deleted file mode 100644 index 21468123fb..0000000000 --- a/LiveKitObjC/LiveKitObjC.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ /dev/null @@ -1,50 +0,0 @@ -{ - "pins" : [ - { - "identity" : "client-sdk-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/livekit/client-sdk-swift", - "state" : { - "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", - "version" : "1.0.4" - } - }, - { - "identity" : "promises", - "kind" : "remoteSourceControl", - "location" : "https://github.com/google/promises.git", - "state" : { - "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", - "version" : "2.1.1" - } - }, - { - "identity" : "specs", - "kind" : "remoteSourceControl", - "location" : "https://github.com/webrtc-sdk/Specs.git", - "state" : { - "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", - "version" : "104.5112.2" - } - }, - { - "identity" : "swift-log", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-log.git", - "state" : { - "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", - "version" : "1.4.4" - } - }, - { - "identity" : "swift-protobuf", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-protobuf.git", - "state" : { - "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", - "version" : "1.20.1" - } - } - ], - "version" : 2 -} diff --git a/LiveKitObjC/Room.swift b/LiveKitObjC/Room.swift deleted file mode 100644 index 9397b82649..0000000000 --- a/LiveKitObjC/Room.swift +++ /dev/null @@ -1,45 +0,0 @@ -// -// LKRoom.swift -// LiveKitObjC -// -// Created by Antonio Scandurra on 01/09/22. -// - -import Foundation -import LiveKit - -public class LKRoom: RoomDelegate { - lazy var room = Room(delegate: self) - - init() { - print("INIT!\n"); - } - - deinit { - print("DEINIT!\n"); - } - - public func connect( - url: String, - token: String, - callback: @convention(block) @escaping () -> Void - ) { - self.room.connect(url, token).then { room in - callback() - } - } - -} - - -@_cdecl("LKRoomCreate") -public func LKRoomCreate() -> UnsafeMutableRawPointer { - Unmanaged.passRetained(LKRoom()).toOpaque() -} - -@_cdecl("LKRoomDestroy") -public func LKRoomDestroy(ptr: UnsafeRawPointer) { - let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); -} - - From 5347c7d678d506cc0e0beb3c938b4d8eb18354cb Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 13:19:17 -0600 Subject: [PATCH 39/89] Connect to LiveKit room in capture example --- Cargo.lock | 21 ++ crates/capture/Cargo.toml | 4 + crates/capture/build.rs | 35 +-- crates/capture/src/bindings.h | 2 - crates/capture/src/bindings.rs | 8 - crates/capture/src/compression_session.rs | 178 ----------- crates/capture/src/live_kit_token.rs | 71 +++++ crates/capture/src/main.rs | 277 ++---------------- crates/live_kit/Cargo.toml | 2 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 11 + crates/live_kit/build.rs | 1 + crates/live_kit/src/live_kit.rs | 37 ++- 12 files changed, 173 insertions(+), 474 deletions(-) delete mode 100644 crates/capture/src/bindings.h delete mode 100644 crates/capture/src/bindings.rs delete mode 100644 crates/capture/src/compression_session.rs create mode 100644 crates/capture/src/live_kit_token.rs diff --git a/Cargo.lock b/Cargo.lock index 1e176cda42..bafbaaf93c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,12 +765,16 @@ dependencies = [ "foreign-types", "futures", "gpui", + "hmac 0.12.1", + "jwt", "live_kit", "log", "media", "objc", "parking_lot 0.11.2", "postage", + "serde", + "sha2 0.10.2", "simplelog", ] @@ -2747,6 +2751,21 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41ee439ee368ba4a77ac70d04f14015415af8600d6c894dc1f11bd79758c57d5" +[[package]] +name = "jwt" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" +dependencies = [ + "base64", + "crypto-common", + "digest 0.10.3", + "hmac 0.12.1", + "serde", + "serde_json", + "sha2 0.10.2", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -2921,6 +2940,8 @@ dependencies = [ name = "live_kit" version = "0.1.0" dependencies = [ + "core-foundation", + "futures", "serde", "serde_json", ] diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index d572c6077c..f8ed31097a 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -18,10 +18,14 @@ core-foundation = "0.9.3" core-graphics = "0.22.3" foreign-types = "0.3" futures = "0.3" +hmac = "0.12" +jwt = "0.16" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" parking_lot = "0.11.1" postage = { version = "0.4.1", features = ["futures-traits"] } +serde = { version = "1.0", features = ["derive", "rc"] } +sha2 = "0.10" simplelog = "0.9" [build-dependencies] diff --git a/crates/capture/build.rs b/crates/capture/build.rs index c637f2ce9b..41f60ff486 100644 --- a/crates/capture/build.rs +++ b/crates/capture/build.rs @@ -1,38 +1,7 @@ -use std::{env, path::PathBuf, process::Command}; - fn main() { // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); - println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=12.3"); - - let sdk_path = String::from_utf8( - Command::new("xcrun") - .args(&["--sdk", "macosx", "--show-sdk-path"]) - .output() - .unwrap() - .stdout, - ) - .unwrap(); - let sdk_path = sdk_path.trim_end(); - - println!("cargo:rerun-if-changed=src/bindings.h"); - let bindings = bindgen::Builder::default() - .header("src/bindings.h") - .clang_arg(format!("-isysroot{}", sdk_path)) - .clang_arg("-xobjective-c") - .allowlist_function("dispatch_queue_create") - .allowlist_type("SCStreamOutputType") - .allowlist_type("SCFrameStatus") - .allowlist_var("SCStreamFrameInfo.*") - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) - .layout_tests(false) - .generate() - .expect("unable to generate bindings"); - - let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); - bindings - .write_to_file(out_path.join("bindings.rs")) - .expect("couldn't write dispatch bindings"); + // Register exported Objective-C selectors, protocols, etc + println!("cargo:rustc-link-arg=-Wl,-ObjC"); } diff --git a/crates/capture/src/bindings.h b/crates/capture/src/bindings.h deleted file mode 100644 index 5e3c525891..0000000000 --- a/crates/capture/src/bindings.h +++ /dev/null @@ -1,2 +0,0 @@ -#import -#import diff --git a/crates/capture/src/bindings.rs b/crates/capture/src/bindings.rs deleted file mode 100644 index a1c0b0da3e..0000000000 --- a/crates/capture/src/bindings.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![allow(non_upper_case_globals)] -#![allow(non_camel_case_types)] -#![allow(non_snake_case)] -#![allow(unused)] - -use objc::*; - -include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/crates/capture/src/compression_session.rs b/crates/capture/src/compression_session.rs deleted file mode 100644 index 1a771d057b..0000000000 --- a/crates/capture/src/compression_session.rs +++ /dev/null @@ -1,178 +0,0 @@ -use anyhow::Result; -use core_foundation::base::{OSStatus, TCFType}; -use media::{ - core_media::{CMSampleBufferRef, CMSampleTimingInfo, CMVideoCodecType}, - core_video::CVImageBuffer, - video_toolbox::{VTCompressionSession, VTEncodeInfoFlags}, -}; -use std::ffi::c_void; - -pub struct CompressionSession { - session: VTCompressionSession, - output_callback: Box, -} - -impl - CompressionSession -{ - pub fn new(width: usize, height: usize, codec: CMVideoCodecType, callback: F) -> Result { - let callback = Box::new(callback); - let session = VTCompressionSession::new( - width, - height, - codec, - Some(Self::output_callback), - callback.as_ref() as *const _ as *const c_void, - )?; - Ok(Self { - session, - output_callback: callback, - }) - } - - pub fn encode_frame(&self, buffer: &CVImageBuffer, timing: CMSampleTimingInfo) -> Result<()> { - self.session.encode_frame( - buffer.as_concrete_TypeRef(), - timing.presentationTimeStamp, - timing.duration, - ) - } - - extern "C" fn output_callback( - output_callback_ref_con: *mut c_void, - _: *mut c_void, - status: OSStatus, - flags: VTEncodeInfoFlags, - sample_buffer: CMSampleBufferRef, - ) { - let callback = unsafe { &mut *(output_callback_ref_con as *mut F) }; - callback(status, flags, sample_buffer); - } -} - -// unsafe extern "C" fn output( -// output_callback_ref_con: *mut c_void, -// source_frame_ref_con: *mut c_void, -// status: OSStatus, -// info_flags: VTEncodeInfoFlags, -// sample_buffer: CMSampleBufferRef, -// ) { -// if status != 0 { -// println!("error encoding frame, code: {}", status); -// return; -// } -// let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - -// let mut is_iframe = false; -// let attachments = sample_buffer.attachments(); -// if let Some(attachments) = attachments.first() { -// is_iframe = attachments -// .find(bindings::kCMSampleAttachmentKey_NotSync as CFStringRef) -// .map_or(true, |not_sync| { -// CFBooleanGetValue(*not_sync as CFBooleanRef) -// }); -// } - -// const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; -// if is_iframe { -// let format_description = sample_buffer.format_description(); -// for ix in 0..format_description.h264_parameter_set_count() { -// let parameter_set = format_description.h264_parameter_set_at_index(ix); -// stream.extend(START_CODE); -// stream.extend(parameter_set); -// } -// } - -// println!("YO!"); -// } - -// static void videoFrameFinishedEncoding(void *outputCallbackRefCon, -// void *sourceFrameRefCon, -// OSStatus status, -// VTEncodeInfoFlags infoFlags, -// CMSampleBufferRef sampleBuffer) { -// // Check if there were any errors encoding -// if (status != noErr) { -// NSLog(@"Error encoding video, err=%lld", (int64_t)status); -// return; -// } - -// // In this example we will use a NSMutableData object to store the -// // elementary stream. -// NSMutableData *elementaryStream = [NSMutableData data]; - -// // Find out if the sample buffer contains an I-Frame. -// // If so we will write the SPS and PPS NAL units to the elementary stream. -// BOOL isIFrame = NO; -// CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); -// if (CFArrayGetCount(attachmentsArray)) { -// CFBooleanRef notSync; -// CFDictionaryRef dict = CFArrayGetValueAtIndex(attachmentsArray, 0); -// BOOL keyExists = CFDictionaryGetValueIfPresent(dict, -// kCMSampleAttachmentKey_NotSync, -// (const void **)¬Sync); -// // An I-Frame is a sync frame -// isIFrame = !keyExists || !CFBooleanGetValue(notSync); -// } - -// // This is the start code that we will write to -// // the elementary stream before every NAL unit -// static const size_t startCodeLength = 4; -// static const uint8_t startCode[] = {0x00, 0x00, 0x00, 0x01}; - -// // Write the SPS and PPS NAL units to the elementary stream before every I-Frame -// if (isIFrame) { -// CMFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sampleBuffer); - -// // Find out how many parameter sets there are -// size_t numberOfParameterSets; -// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, -// 0, NULL, NULL, -// &numberOfParameterSets, -// NULL); - -// // Write each parameter set to the elementary stream -// for (int i = 0; i < numberOfParameterSets; i++) { -// const uint8_t *parameterSetPointer; -// size_t parameterSetLength; -// CMVideoFormatDescriptionGetH264ParameterSetAtIndex(description, -// i, -// ¶meterSetPointer, -// ¶meterSetLength, -// NULL, NULL); - -// // Write the parameter set to the elementary stream -// [elementaryStream appendBytes:startCode length:startCodeLength]; -// [elementaryStream appendBytes:parameterSetPointer length:parameterSetLength]; -// } -// } - -// // Get a pointer to the raw AVCC NAL unit data in the sample buffer -// size_t blockBufferLength; -// uint8_t *bufferDataPointer = NULL; -// CMBlockBufferGetDataPointer(CMSampleBufferGetDataBuffer(sampleBuffer), -// 0, -// NULL, -// &blockBufferLength, -// (char **)&bufferDataPointer); - -// // Loop through all the NAL units in the block buffer -// // and write them to the elementary stream with -// // start codes instead of AVCC length headers -// size_t bufferOffset = 0; -// static const int AVCCHeaderLength = 4; -// while (bufferOffset < blockBufferLength - AVCCHeaderLength) { -// // Read the NAL unit length -// uint32_t NALUnitLength = 0; -// memcpy(&NALUnitLength, bufferDataPointer + bufferOffset, AVCCHeaderLength); -// // Convert the length value from Big-endian to Little-endian -// NALUnitLength = CFSwapInt32BigToHost(NALUnitLength); -// // Write start code to the elementary stream -// [elementaryStream appendBytes:startCode length:startCodeLength]; -// // Write the NAL unit without the AVCC length header to the elementary stream -// [elementaryStream appendBytes:bufferDataPointer + bufferOffset + AVCCHeaderLength -// length:NALUnitLength]; -// // Move to the next NAL unit in the block buffer -// bufferOffset += AVCCHeaderLength + NALUnitLength; -// } -// } diff --git a/crates/capture/src/live_kit_token.rs b/crates/capture/src/live_kit_token.rs new file mode 100644 index 0000000000..be4fc4f4a2 --- /dev/null +++ b/crates/capture/src/live_kit_token.rs @@ -0,0 +1,71 @@ +use anyhow::Result; +use hmac::{Hmac, Mac}; +use jwt::SignWithKey; +use serde::Serialize; +use sha2::Sha256; +use std::{ + ops::Add, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +static DEFAULT_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6 hours + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ClaimGrants<'a> { + iss: &'a str, + sub: &'a str, + iat: u64, + exp: u64, + nbf: u64, + jwtid: &'a str, + video: VideoGrant<'a>, +} + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct VideoGrant<'a> { + room_create: Option, + room_join: Option, + room_list: Option, + room_record: Option, + room_admin: Option, + room: Option<&'a str>, + can_publish: Option, + can_subscribe: Option, + can_publish_data: Option, + hidden: Option, + recorder: Option, +} + +pub fn create_token( + api_key: &str, + secret_key: &str, + room_name: &str, + participant_name: &str, +) -> Result { + let secret_key: Hmac = Hmac::new_from_slice(secret_key.as_bytes())?; + + let now = SystemTime::now(); + + let claims = ClaimGrants { + iss: api_key, + sub: participant_name, + iat: now.duration_since(UNIX_EPOCH).unwrap().as_secs(), + exp: now + .add(DEFAULT_TTL) + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + nbf: 0, + jwtid: participant_name, + video: VideoGrant { + room: Some(room_name), + room_join: Some(true), + can_publish: Some(true), + can_subscribe: Some(true), + ..Default::default() + }, + }; + Ok(claims.sign_with_key(&secret_key)?) +} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 75df9be368..83454ba77a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,20 +1,5 @@ -mod bindings; -mod compression_session; +mod live_kit_token; -use crate::{bindings::SCStreamOutputType, compression_session::CompressionSession}; -use block::ConcreteBlock; -use byteorder::{BigEndian, ReadBytesExt}; -use bytes::BytesMut; -use cocoa::{ - base::{id, nil, YES}, - foundation::{NSArray, NSString, NSUInteger}, -}; -use core_foundation::{ - base::{CFRelease, TCFType}, - number::{CFBooleanGetValue, CFBooleanRef, CFNumberRef}, - string::CFStringRef, -}; -use futures::StreamExt; use gpui::{ actions, elements::{Canvas, *}, @@ -24,37 +9,12 @@ use gpui::{ }; use live_kit::Room; use log::LevelFilter; -use media::{ - core_media::{ - kCMSampleAttachmentKey_NotSync, kCMVideoCodecType_H264, CMSampleBuffer, CMSampleBufferRef, - CMTimeMake, - }, - core_video::{self, CVImageBuffer}, - video_toolbox::VTCompressionSession, -}; -use objc::{ - class, - declare::ClassDecl, - msg_send, - runtime::{Class, Object, Sel}, - sel, sel_impl, -}; -use parking_lot::Mutex; +use media::core_video::CVImageBuffer; use simplelog::SimpleLogger; -use std::{ffi::c_void, ptr, slice, str, sync::Arc}; - -#[allow(non_upper_case_globals)] -const NSUTF8StringEncoding: NSUInteger = 4; actions!(capture, [Quit]); fn main() { - println!("Creating room..."); - let room = Room::new(); - - println!("Dropping room..."); - drop(room); - SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); gpui::App::new(()).unwrap().run(|cx| { @@ -70,12 +30,32 @@ fn main() { }], }]); - cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); + let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); + let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); + + let token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant", + ) + .unwrap(); + + let room = live_kit::Room::new(); + cx.spawn(|cx| async move { + println!("connecting..."); + room.connect("wss://zed.livekit.cloud", &token).await; + println!("connected!"); + drop(room); + }) + .detach(); + + // cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); }); } struct ScreenCaptureView { - image_buffer: Option, + image_buffer: Option, } impl gpui::Entity for ScreenCaptureView { @@ -83,188 +63,7 @@ impl gpui::Entity for ScreenCaptureView { } impl ScreenCaptureView { - pub fn new(cx: &mut ViewContext) -> Self { - let (image_buffer_tx, mut image_buffer_rx) = - postage::watch::channel::>(); - let image_buffer_tx = Arc::new(Mutex::new(image_buffer_tx)); - - unsafe { - let block = ConcreteBlock::new(move |content: id, error: id| { - if !error.is_null() { - println!( - "ERROR {}", - string_from_objc(msg_send![error, localizedDescription]) - ); - return; - } - - let applications: id = msg_send![content, applications]; - let displays: id = msg_send![content, displays]; - let display: id = displays.objectAtIndex(0); - let display_width: usize = msg_send![display, width]; - let display_height: usize = msg_send![display, height]; - let mut compression_buffer = BytesMut::new(); - // let compression_session = CompressionSession::new( - // display_width, - // display_height, - // kCMVideoCodecType_H264, - // move |status, flags, sample_buffer| { - // if status != 0 { - // println!("error encoding frame, code: {}", status); - // return; - // } - // let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); - - // let mut is_iframe = false; - // let attachments = sample_buffer.attachments(); - // if let Some(attachments) = attachments.first() { - // is_iframe = attachments - // .find(kCMSampleAttachmentKey_NotSync as CFStringRef) - // .map_or(true, |not_sync| { - // CFBooleanGetValue(*not_sync as CFBooleanRef) - // }); - // } - - // const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01]; - // if is_iframe { - // let format_description = sample_buffer.format_description(); - // for ix in 0..format_description.h264_parameter_set_count() { - // let parameter_set = - // format_description.h264_parameter_set_at_index(ix).unwrap(); - // compression_buffer.extend_from_slice(&START_CODE); - // compression_buffer.extend_from_slice(parameter_set); - // let nal_unit = compression_buffer.split(); - // } - // } - - // let data = sample_buffer.data(); - // let mut data = data.bytes(); - - // const AVCC_HEADER_LENGTH: usize = 4; - // while data.len() - AVCC_HEADER_LENGTH > 0 { - // let nal_unit_len = match data.read_u32::() { - // Ok(len) => len as usize, - // Err(error) => { - // log::error!("error decoding nal unit length: {}", error); - // return; - // } - // }; - // compression_buffer.extend_from_slice(&START_CODE); - // compression_buffer.extend_from_slice(&data[..nal_unit_len as usize]); - // data = &data[nal_unit_len..]; - - // let nal_unit = compression_buffer.split(); - // } - // }, - // ) - // .unwrap(); - - let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap(); - decl.add_ivar::<*mut c_void>("callback"); - decl.add_method( - sel!(stream:didOutputSampleBuffer:ofType:), - sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType), - ); - let capture_output_class = decl.register(); - - let output: id = msg_send![capture_output_class, alloc]; - let output: id = msg_send![output, init]; - let surface_tx = image_buffer_tx.clone(); - - let callback = Box::new(move |buffer: CMSampleBufferRef| { - let buffer = CMSampleBuffer::wrap_under_get_rule(buffer); - let attachments = buffer.attachments(); - let attachments = attachments.first().expect("no attachments for sample"); - let string = bindings::SCStreamFrameInfoStatus.0 as CFStringRef; - let status = core_foundation::number::CFNumber::wrap_under_get_rule( - *attachments.get(string) as CFNumberRef, - ) - .to_i64() - .expect("invalid frame info status"); - - if status != bindings::SCFrameStatus_SCFrameStatusComplete { - println!("received incomplete frame"); - return; - } - - let timing_info = buffer.sample_timing_info(0).unwrap(); - let image_buffer = buffer.image_buffer(); - // compression_session - // .encode_frame(&image_buffer, timing_info) - // .unwrap(); - *surface_tx.lock().borrow_mut() = Some(image_buffer); - }) as Box; - let callback = Box::into_raw(Box::new(callback)); - (*output).set_ivar("callback", callback as *mut c_void); - - let filter: id = msg_send![class!(SCContentFilter), alloc]; - let filter: id = msg_send![filter, initWithDisplay: display includingApplications: applications exceptingWindows: nil]; - // let filter: id = msg_send![filter, initWithDesktopIndependentWindow: window]; - let config: id = msg_send![class!(SCStreamConfiguration), alloc]; - let config: id = msg_send![config, init]; - let _: () = msg_send![config, setWidth: display_width * 2]; - let _: () = msg_send![config, setHeight: display_height * 2]; - let _: () = msg_send![config, setMinimumFrameInterval: CMTimeMake(1, 60)]; - let _: () = msg_send![config, setQueueDepth: 6]; - let _: () = msg_send![config, setShowsCursor: YES]; - let _: () = msg_send![ - config, - setPixelFormat: media::core_video::kCVPixelFormatType_32BGRA - ]; - - let stream: id = msg_send![class!(SCStream), alloc]; - let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; - let error: id = nil; - let queue = bindings::dispatch_queue_create( - ptr::null(), - bindings::NSObject(ptr::null_mut()), - ); - - let _: () = msg_send![stream, - addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen - sampleHandlerQueue: queue - error: &error - ]; - - let start_capture_completion = ConcreteBlock::new(move |error: id| { - if !error.is_null() { - println!( - "error starting capture... error? {}", - string_from_objc(msg_send![error, localizedDescription]) - ); - return; - } - - println!("starting capture"); - }); - - assert!(!stream.is_null()); - let _: () = msg_send![ - stream, - startCaptureWithCompletionHandler: start_capture_completion - ]; - }); - - let _: id = msg_send![ - class!(SCShareableContent), - getShareableContentWithCompletionHandler: block - ]; - } - - cx.spawn_weak(|this, mut cx| async move { - while let Some(image_buffer) = image_buffer_rx.next().await { - if let Some(this) = this.upgrade(&cx) { - this.update(&mut cx, |this, cx| { - this.image_buffer = image_buffer; - cx.notify(); - }) - } else { - break; - } - } - }) - .detach(); - + pub fn new(_: &mut ViewContext) -> Self { Self { image_buffer: None } } } @@ -288,32 +87,6 @@ impl gpui::View for ScreenCaptureView { } } -pub unsafe fn string_from_objc(string: id) -> String { - if string.is_null() { - Default::default() - } else { - let len = msg_send![string, lengthOfBytesUsingEncoding: NSUTF8StringEncoding]; - let bytes = string.UTF8String() as *const u8; - str::from_utf8(slice::from_raw_parts(bytes, len)) - .unwrap() - .to_string() - } -} - -extern "C" fn sample_output( - this: &Object, - _: Sel, - _stream: id, - buffer: id, - _kind: SCStreamOutputType, -) { - unsafe { - let callback = *this.get_ivar::<*mut c_void>("callback"); - let callback = &mut *(callback as *mut Box); - (*callback)(buffer as CMSampleBufferRef); - } -} - fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index 15f7683456..8cb4df32cc 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -9,6 +9,8 @@ path = "src/live_kit.rs" doctest = false [dependencies] +core-foundation = "0.9.3" +futures = "0.3" [build-dependencies] serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 11df49e6c6..6a935d7ec4 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -10,3 +10,14 @@ public func LKRoomCreate() -> UnsafeMutableRawPointer { public func LKRoomDestroy(ptr: UnsafeRawPointer) { let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); } + +@_cdecl("LKRoomConnect") +public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { + let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); + + room.connect(url as String, token as String).then { _ in + callback(callback_data); + }.catch { error in + print(error); + }; +} diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index ca9c2d5cb9..06f49c6618 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -37,6 +37,7 @@ fn main() { } fn build_bridge(swift_target: &SwiftTarget) { + println!("cargo:rerun-if-changed={}", SWIFT_PACKAGE_NAME); let swift_package_root = swift_package_root(); if !Command::new("swift") .args(&["build", "-c", &env::var("PROFILE").unwrap()]) diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 3dcaaa74ed..346ba2173b 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,8 +1,20 @@ +use core_foundation::{ + base::TCFType, + string::{CFString, CFStringRef}, +}; +use futures::{channel::oneshot, Future}; use std::ffi::c_void; extern "C" { fn LKRoomCreate() -> *const c_void; - fn LKRoomDestroy(ptr: *const c_void); + fn LKRoomDestroy(room: *const c_void); + fn LKRoomConnect( + room: *const c_void, + url: CFStringRef, + token: CFStringRef, + callback: extern "C" fn(*mut c_void) -> (), + callback_data: *mut c_void, + ); } pub struct Room { @@ -15,6 +27,29 @@ impl Room { native_room: unsafe { LKRoomCreate() }, } } + + pub fn connect(&self, url: &str, token: &str) -> impl Future { + let url = CFString::new(url); + let token = CFString::new(token); + + let (tx, rx) = oneshot::channel(); + extern "C" fn did_connect(tx: *mut c_void) { + let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; + let _ = tx.send(()); + } + + unsafe { + LKRoomConnect( + self.native_room, + url.as_concrete_TypeRef(), + token.as_concrete_TypeRef(), + did_connect, + Box::into_raw(Box::new(tx)) as *mut c_void, + ) + } + + async { rx.await.unwrap() } + } } impl Drop for Room { From 7bf64ec23e883cd87e43c0d112056ac8384295cb Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 6 Sep 2022 21:19:00 -0600 Subject: [PATCH 40/89] Add ability to get a screen share track for a window And also list windows --- Cargo.lock | 1 + crates/capture/src/main.rs | 20 +++-- crates/live_kit/Cargo.toml | 1 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 16 ++-- crates/live_kit/src/live_kit.rs | 78 ++++++++++++++++--- 5 files changed, 93 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bafbaaf93c..a9448065c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2941,6 +2941,7 @@ name = "live_kit" version = "0.1.0" dependencies = [ "core-foundation", + "core-graphics", "futures", "serde", "serde_json", diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 83454ba77a..07d284b7b7 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -7,7 +7,7 @@ use gpui::{ platform::current::Surface, Menu, MenuItem, ViewContext, }; -use live_kit::Room; +use live_kit::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; use simplelog::SimpleLogger; @@ -42,13 +42,17 @@ fn main() { .unwrap(); let room = live_kit::Room::new(); - cx.spawn(|cx| async move { - println!("connecting..."); - room.connect("wss://zed.livekit.cloud", &token).await; - println!("connected!"); - drop(room); - }) - .detach(); + cx.foreground() + .spawn(async move { + println!("connecting..."); + room.connect("wss://zed.livekit.cloud", &token).await; + let windows = live_kit::list_windows(); + println!("connected! {:?}", windows); + + let window_id = windows.iter().next().unwrap().id; + let track = LocalVideoTrack::screen_share_for_window(window_id); + }) + .detach(); // cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); }); diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index 8cb4df32cc..325753b7f9 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -10,6 +10,7 @@ doctest = false [dependencies] core-foundation = "0.9.3" +core-graphics = "0.22.3" futures = "0.3" [build-dependencies] diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 6a935d7ec4..a5783d69b4 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,16 +1,16 @@ import Foundation import LiveKit +@_cdecl("LKRelease") +public func LKRelease(ptr: UnsafeRawPointer) { + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); +} + @_cdecl("LKRoomCreate") public func LKRoomCreate() -> UnsafeMutableRawPointer { Unmanaged.passRetained(Room()).toOpaque() } -@_cdecl("LKRoomDestroy") -public func LKRoomDestroy(ptr: UnsafeRawPointer) { - let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); -} - @_cdecl("LKRoomConnect") public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); @@ -21,3 +21,9 @@ public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString print(error); }; } + +@_cdecl("LKCreateScreenShareTrackForWindow") +public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutableRawPointer { + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)); + return Unmanaged.passRetained(track).toOpaque() +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 346ba2173b..7ea94298fd 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,13 +1,20 @@ use core_foundation::{ - base::TCFType, + array::CFArray, + base::{TCFType, TCFTypeRef}, + dictionary::CFDictionary, + number::CFNumber, string::{CFString, CFStringRef}, }; +use core_graphics::window::{ + kCGNullWindowID, kCGWindowListOptionExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, + kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, +}; use futures::{channel::oneshot, Future}; use std::ffi::c_void; extern "C" { + fn LKRelease(object: *const c_void); fn LKRoomCreate() -> *const c_void; - fn LKRoomDestroy(room: *const c_void); fn LKRoomConnect( room: *const c_void, url: CFStringRef, @@ -15,17 +22,14 @@ extern "C" { callback: extern "C" fn(*mut c_void) -> (), callback_data: *mut c_void, ); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } -pub struct Room { - native_room: *const c_void, -} +pub struct Room(*const c_void); impl Room { pub fn new() -> Self { - Self { - native_room: unsafe { LKRoomCreate() }, - } + Self(unsafe { LKRoomCreate() }) } pub fn connect(&self, url: &str, token: &str) -> impl Future { @@ -40,7 +44,7 @@ impl Room { unsafe { LKRoomConnect( - self.native_room, + self.0, url.as_concrete_TypeRef(), token.as_concrete_TypeRef(), did_connect, @@ -54,6 +58,60 @@ impl Room { impl Drop for Room { fn drop(&mut self) { - unsafe { LKRoomDestroy(self.native_room) } + unsafe { LKRelease(self.0) } + } +} + +pub struct LocalVideoTrack(*const c_void); + +impl LocalVideoTrack { + pub fn screen_share_for_window(window_id: u32) -> Self { + Self(unsafe { LKCreateScreenShareTrackForWindow(window_id) }) + } +} + +impl Drop for LocalVideoTrack { + fn drop(&mut self) { + unsafe { LKRelease(self.0) } + } +} + +#[derive(Debug)] +pub struct WindowInfo { + pub id: u32, + pub owner_pid: i32, + pub owner_name: Option, +} + +pub fn list_windows() -> Vec { + unsafe { + let dicts = CFArray::::wrap_under_get_rule(CGWindowListCopyWindowInfo( + kCGWindowListOptionOnScreenOnly | kCGWindowListOptionExcludeDesktopElements, + kCGNullWindowID, + )); + + dicts + .iter() + .map(|dict| { + let id = + CFNumber::wrap_under_get_rule(*dict.get(kCGWindowNumber.as_void_ptr()) as _) + .to_i64() + .unwrap() as u32; + + let owner_pid = + CFNumber::wrap_under_get_rule(*dict.get(kCGWindowOwnerPID.as_void_ptr()) as _) + .to_i32() + .unwrap(); + + let owner_name = dict + .find(kCGWindowOwnerName.as_void_ptr()) + .map(|name| CFString::wrap_under_get_rule(*name as _).to_string()); + WindowInfo { + id, + owner_pid, + owner_name, + } + }) + .collect() } } From df3ab13441b6f668a86feb5953535c755641369c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 14:52:11 +0200 Subject: [PATCH 41/89] Publish screen-sharing video track on the newly-created Room --- crates/capture/src/main.rs | 3 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 11 +++++ crates/live_kit/build.rs | 6 ++- crates/live_kit/src/live_kit.rs | 48 +++++++++++++++---- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 07d284b7b7..66a01d976b 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -41,7 +41,7 @@ fn main() { ) .unwrap(); - let room = live_kit::Room::new(); + let room = Room::new(); cx.foreground() .spawn(async move { println!("connecting..."); @@ -51,6 +51,7 @@ fn main() { let window_id = windows.iter().next().unwrap().id; let track = LocalVideoTrack::screen_share_for_window(window_id); + room.publish_video_track(&track).await; }) .detach(); diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index a5783d69b4..5d30cc4d28 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -22,6 +22,17 @@ public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString }; } +@_cdecl("LKRoomPublishVideoTrack") +public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { + let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); + let track = Unmanaged.fromOpaque(track).takeUnretainedValue(); + room.localParticipant?.publishVideoTrack(track: track).then { _ in + callback(callback_data); + }.catch { error in + print(error); + }; +} + @_cdecl("LKCreateScreenShareTrackForWindow") public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutableRawPointer { let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)); diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index 06f49c6618..4f23cc4e92 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -37,7 +37,11 @@ fn main() { } fn build_bridge(swift_target: &SwiftTarget) { - println!("cargo:rerun-if-changed={}", SWIFT_PACKAGE_NAME); + println!("cargo:rerun-if-changed={}/Sources", SWIFT_PACKAGE_NAME); + println!( + "cargo:rerun-if-changed={}/Package.swift", + SWIFT_PACKAGE_NAME + ); let swift_package_root = swift_package_root(); if !Command::new("swift") .args(&["build", "-c", &env::var("PROFILE").unwrap()]) diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 7ea94298fd..a3b06cfcf6 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -14,6 +14,7 @@ use std::ffi::c_void; extern "C" { fn LKRelease(object: *const c_void); + fn LKRoomCreate() -> *const c_void; fn LKRoomConnect( room: *const c_void, @@ -22,6 +23,13 @@ extern "C" { callback: extern "C" fn(*mut c_void) -> (), callback_data: *mut c_void, ); + fn LKRoomPublishVideoTrack( + room: *const c_void, + track: *const c_void, + callback: extern "C" fn(*mut c_void) -> (), + callback_data: *mut c_void, + ); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } @@ -35,25 +43,49 @@ impl Room { pub fn connect(&self, url: &str, token: &str) -> impl Future { let url = CFString::new(url); let token = CFString::new(token); - - let (tx, rx) = oneshot::channel(); - extern "C" fn did_connect(tx: *mut c_void) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; - let _ = tx.send(()); - } - + let (did_connect, tx, rx) = Self::build_done_callback(); unsafe { LKRoomConnect( self.0, url.as_concrete_TypeRef(), token.as_concrete_TypeRef(), did_connect, - Box::into_raw(Box::new(tx)) as *mut c_void, + tx, ) } async { rx.await.unwrap() } } + + pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future { + let (did_publish, tx, rx) = Self::build_done_callback(); + unsafe { + LKRoomPublishVideoTrack( + self.0, + track.0, + did_publish, + Box::into_raw(Box::new(tx)) as *mut c_void, + ) + } + async { rx.await.unwrap() } + } + + fn build_done_callback() -> ( + extern "C" fn(*mut c_void), + *mut c_void, + oneshot::Receiver<()>, + ) { + let (tx, rx) = oneshot::channel(); + extern "C" fn done_callback(tx: *mut c_void) { + let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; + let _ = tx.send(()); + } + ( + done_callback, + Box::into_raw(Box::new(tx)) as *mut c_void, + rx, + ) + } } impl Drop for Room { From b154c3c9ee9116652607342d14934cc4c6b82396 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 15:18:03 +0200 Subject: [PATCH 42/89] Expose Swift errors as `anyhow::Result` in live_kit --- Cargo.lock | 1 + crates/capture/src/main.rs | 6 ++-- crates/live_kit/Cargo.toml | 1 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 12 ++++---- crates/live_kit/src/live_kit.rs | 28 +++++++++++-------- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9448065c9..9dbdfe1fc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2940,6 +2940,7 @@ dependencies = [ name = "live_kit" version = "0.1.0" dependencies = [ + "anyhow", "core-foundation", "core-graphics", "futures", diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 66a01d976b..82a6934fa7 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -45,13 +45,15 @@ fn main() { cx.foreground() .spawn(async move { println!("connecting..."); - room.connect("wss://zed.livekit.cloud", &token).await; + room.connect("wss://zed.livekit.cloud", &token) + .await + .unwrap(); let windows = live_kit::list_windows(); println!("connected! {:?}", windows); let window_id = windows.iter().next().unwrap().id; let track = LocalVideoTrack::screen_share_for_window(window_id); - room.publish_video_track(&track).await; + room.publish_video_track(&track).await.unwrap(); }) .detach(); diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index 325753b7f9..d98b7eae88 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -9,6 +9,7 @@ path = "src/live_kit.rs" doctest = false [dependencies] +anyhow = "1.0.38" core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 5d30cc4d28..a33aeeb6de 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -12,24 +12,24 @@ public func LKRoomCreate() -> UnsafeMutableRawPointer { } @_cdecl("LKRoomConnect") -public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { +public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); room.connect(url as String, token as String).then { _ in - callback(callback_data); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?); }.catch { error in - print(error); + callback(callback_data, error.localizedDescription as CFString); }; } @_cdecl("LKRoomPublishVideoTrack") -public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer) -> Void, callback_data: UnsafeRawPointer) { +public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); let track = Unmanaged.fromOpaque(track).takeUnretainedValue(); room.localParticipant?.publishVideoTrack(track: track).then { _ in - callback(callback_data); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?); }.catch { error in - print(error); + callback(callback_data, error.localizedDescription as CFString); }; } diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index a3b06cfcf6..0db2ac1e34 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,3 +1,4 @@ +use anyhow::{anyhow, Context, Result}; use core_foundation::{ array::CFArray, base::{TCFType, TCFTypeRef}, @@ -20,13 +21,13 @@ extern "C" { room: *const c_void, url: CFStringRef, token: CFStringRef, - callback: extern "C" fn(*mut c_void) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef) -> (), callback_data: *mut c_void, ); fn LKRoomPublishVideoTrack( room: *const c_void, track: *const c_void, - callback: extern "C" fn(*mut c_void) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef) -> (), callback_data: *mut c_void, ); @@ -40,7 +41,7 @@ impl Room { Self(unsafe { LKRoomCreate() }) } - pub fn connect(&self, url: &str, token: &str) -> impl Future { + pub fn connect(&self, url: &str, token: &str) -> impl Future> { let url = CFString::new(url); let token = CFString::new(token); let (did_connect, tx, rx) = Self::build_done_callback(); @@ -54,10 +55,10 @@ impl Room { ) } - async { rx.await.unwrap() } + async { rx.await.unwrap().context("error connecting to room") } } - pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future { + pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future> { let (did_publish, tx, rx) = Self::build_done_callback(); unsafe { LKRoomPublishVideoTrack( @@ -67,18 +68,23 @@ impl Room { Box::into_raw(Box::new(tx)) as *mut c_void, ) } - async { rx.await.unwrap() } + async { rx.await.unwrap().context("error publishing video track") } } fn build_done_callback() -> ( - extern "C" fn(*mut c_void), + extern "C" fn(*mut c_void, CFStringRef), *mut c_void, - oneshot::Receiver<()>, + oneshot::Receiver>, ) { let (tx, rx) = oneshot::channel(); - extern "C" fn done_callback(tx: *mut c_void) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<()>) }; - let _ = tx.send(()); + extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) { + let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; + if error.is_null() { + let _ = tx.send(Ok(())); + } else { + let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; + let _ = tx.send(Err(anyhow!(error))); + } } ( done_callback, From 376e674748b16f0b08ee10ba0f99d166e91f8617 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 16:14:27 +0200 Subject: [PATCH 43/89] Avoid double boxing of oneshot when calling `LKRoomPublishVideoTrack` Co-Authored-By: Nathan Sobo --- crates/live_kit/src/live_kit.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 0db2ac1e34..58a01d7f79 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -61,12 +61,7 @@ impl Room { pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future> { let (did_publish, tx, rx) = Self::build_done_callback(); unsafe { - LKRoomPublishVideoTrack( - self.0, - track.0, - did_publish, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) + LKRoomPublishVideoTrack(self.0, track.0, did_publish, tx); } async { rx.await.unwrap().context("error publishing video track") } } From 37ca5651ee63665426d9fb31d9581524f48754a6 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 16:20:37 +0200 Subject: [PATCH 44/89] Parameterize LiveKit URL Co-Authored-By: Nathan Sobo --- crates/capture/src/main.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 82a6934fa7..967fd6689c 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -30,6 +30,7 @@ fn main() { }], }]); + let live_kit_url = std::env::var("LIVE_KIT_URL").unwrap(); let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); @@ -45,9 +46,7 @@ fn main() { cx.foreground() .spawn(async move { println!("connecting..."); - room.connect("wss://zed.livekit.cloud", &token) - .await - .unwrap(); + room.connect(&live_kit_url, &token).await.unwrap(); let windows = live_kit::list_windows(); println!("connected! {:?}", windows); From 645338cff841d7c276d5a8c75ece2e3182a0fe88 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 7 Sep 2022 18:32:39 +0200 Subject: [PATCH 45/89] Register to publishing of remote tracks --- crates/capture/src/main.rs | 37 +++++--- .../Sources/LiveKitBridge/LiveKitBridge.swift | 47 +++++++--- crates/live_kit/src/live_kit.rs | 93 +++++++++++++++++-- 3 files changed, 143 insertions(+), 34 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 967fd6689c..f89bc4bf95 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,5 +1,7 @@ mod live_kit_token; +use std::time::Duration; + use gpui::{ actions, elements::{Canvas, *}, @@ -34,25 +36,38 @@ fn main() { let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); - let token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant", - ) - .unwrap(); - - let room = Room::new(); + let background = cx.background().clone(); cx.foreground() .spawn(async move { println!("connecting..."); - room.connect(&live_kit_url, &token).await.unwrap(); + let user1_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-1", + ) + .unwrap(); + let room1 = Room::new("user-1 room"); + room1.connect(&live_kit_url, &user1_token).await.unwrap(); + + let user2_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-2", + ) + .unwrap(); + let room2 = Room::new("user-2 room"); + room2.connect(&live_kit_url, &user2_token).await.unwrap(); + let windows = live_kit::list_windows(); println!("connected! {:?}", windows); let window_id = windows.iter().next().unwrap().id; let track = LocalVideoTrack::screen_share_for_window(window_id); - room.publish_video_track(&track).await.unwrap(); + room1.publish_video_track(&track).await.unwrap(); + + background.timer(Duration::from_secs(120)).await; }) .detach(); diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index a33aeeb6de..1802fd91fb 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,40 +1,61 @@ import Foundation import LiveKit +class LKRoomDelegate: RoomDelegate { + var data: UnsafeRawPointer + var onDidSubscribeToRemoteTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void + + init(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { + self.data = data + self.onDidSubscribeToRemoteTrack = onDidSubscribeToRemoteTrack + } + + func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { + self.onDidSubscribeToRemoteTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + } +} + @_cdecl("LKRelease") public func LKRelease(ptr: UnsafeRawPointer) { - let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue(); + let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue() +} + +@_cdecl("LKRoomDelegateCreate") +public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteTrack: onDidSubscribeToRemoteTrack) + return Unmanaged.passRetained(delegate).toOpaque() } @_cdecl("LKRoomCreate") -public func LKRoomCreate() -> UnsafeMutableRawPointer { - Unmanaged.passRetained(Room()).toOpaque() +public func LKRoomCreate(delegate: UnsafeRawPointer) -> UnsafeMutableRawPointer { + let delegate = Unmanaged.fromOpaque(delegate).takeUnretainedValue() + return Unmanaged.passRetained(Room(delegate: delegate)).toOpaque() } @_cdecl("LKRoomConnect") public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); + let room = Unmanaged.fromOpaque(room).takeUnretainedValue() room.connect(url as String, token as String).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?) }.catch { error in - callback(callback_data, error.localizedDescription as CFString); - }; + callback(callback_data, error.localizedDescription as CFString) + } } @_cdecl("LKRoomPublishVideoTrack") public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue(); - let track = Unmanaged.fromOpaque(track).takeUnretainedValue(); + let room = Unmanaged.fromOpaque(room).takeUnretainedValue() + let track = Unmanaged.fromOpaque(track).takeUnretainedValue() room.localParticipant?.publishVideoTrack(track: track).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?); + callback(callback_data, UnsafeRawPointer(nil) as! CFString?) }.catch { error in - callback(callback_data, error.localizedDescription as CFString); - }; + callback(callback_data, error.localizedDescription as CFString) + } } @_cdecl("LKCreateScreenShareTrackForWindow") public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutableRawPointer { - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)); + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)) return Unmanaged.passRetained(track).toOpaque() } diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 58a01d7f79..0c254f99b3 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -11,34 +11,56 @@ use core_graphics::window::{ kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, }; use futures::{channel::oneshot, Future}; -use std::ffi::c_void; +use std::{ + ffi::c_void, + sync::{Arc, Weak}, +}; extern "C" { fn LKRelease(object: *const c_void); - fn LKRoomCreate() -> *const c_void; + fn LKRoomDelegateCreate( + callback_data: *const c_void, + on_did_subscribe_to_remote_track: extern "C" fn( + callback_data: *mut c_void, + remote_track: *const c_void, + ), + ) -> *const c_void; + + fn LKRoomCreate(delegate: *const c_void) -> *const c_void; fn LKRoomConnect( room: *const c_void, url: CFStringRef, token: CFStringRef, - callback: extern "C" fn(*mut c_void, CFStringRef) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef), callback_data: *mut c_void, ); fn LKRoomPublishVideoTrack( room: *const c_void, track: *const c_void, - callback: extern "C" fn(*mut c_void, CFStringRef) -> (), + callback: extern "C" fn(*mut c_void, CFStringRef), callback_data: *mut c_void, ); fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } -pub struct Room(*const c_void); +pub struct Room { + debug_name: &'static str, + native_room: *const c_void, + _delegate: RoomDelegate, +} impl Room { - pub fn new() -> Self { - Self(unsafe { LKRoomCreate() }) + pub fn new(debug_name: &'static str) -> Arc { + Arc::new_cyclic(|weak_room| { + let delegate = RoomDelegate::new(weak_room.clone()); + Self { + debug_name, + native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, + _delegate: delegate, + } + }) } pub fn connect(&self, url: &str, token: &str) -> impl Future> { @@ -47,7 +69,7 @@ impl Room { let (did_connect, tx, rx) = Self::build_done_callback(); unsafe { LKRoomConnect( - self.0, + self.native_room, url.as_concrete_TypeRef(), token.as_concrete_TypeRef(), did_connect, @@ -61,11 +83,15 @@ impl Room { pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future> { let (did_publish, tx, rx) = Self::build_done_callback(); unsafe { - LKRoomPublishVideoTrack(self.0, track.0, did_publish, tx); + LKRoomPublishVideoTrack(self.native_room, track.0, did_publish, tx); } async { rx.await.unwrap().context("error publishing video track") } } + fn did_subscribe_to_remote_track(&self, track: RemoteVideoTrack) { + println!("{}: !!!!!!!!!!!!!!!!!!", self.debug_name); + } + fn build_done_callback() -> ( extern "C" fn(*mut c_void, CFStringRef), *mut c_void, @@ -91,7 +117,46 @@ impl Room { impl Drop for Room { fn drop(&mut self) { - unsafe { LKRelease(self.0) } + unsafe { LKRelease(self.native_room) } + } +} + +struct RoomDelegate { + native_delegate: *const c_void, + weak_room: *const Room, +} + +impl RoomDelegate { + fn new(weak_room: Weak) -> Self { + let weak_room = Weak::into_raw(weak_room); + let native_delegate = unsafe { + LKRoomDelegateCreate( + weak_room as *const c_void, + Self::on_did_subscribe_to_remote_track, + ) + }; + Self { + native_delegate, + weak_room, + } + } + + extern "C" fn on_did_subscribe_to_remote_track(room: *mut c_void, track: *const c_void) { + let room = unsafe { Weak::from_raw(room as *mut Room) }; + let track = unsafe { RemoteVideoTrack(track) }; + if let Some(room) = room.upgrade() { + room.did_subscribe_to_remote_track(track); + } + let _ = Weak::into_raw(room); + } +} + +impl Drop for RoomDelegate { + fn drop(&mut self) { + unsafe { + LKRelease(self.native_delegate); + let _ = Weak::from_raw(self.weak_room); + } } } @@ -109,6 +174,14 @@ impl Drop for LocalVideoTrack { } } +pub struct RemoteVideoTrack(*const c_void); + +impl Drop for RemoteVideoTrack { + fn drop(&mut self) { + unsafe { LKRelease(self.0) } + } +} + #[derive(Debug)] pub struct WindowInfo { pub id: u32, From d407f521db12751bce5575b788958eecbef329b4 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 12:23:36 +0200 Subject: [PATCH 46/89] WIP: render screen-sharing frames --- Cargo.lock | 2 + crates/capture/src/main.rs | 92 +++++++++++-------- crates/gpui/src/platform/mac/renderer.rs | 2 +- crates/live_kit/Cargo.toml | 3 + .../live_kit/LiveKitBridge/Package.resolved | 3 +- crates/live_kit/LiveKitBridge/Package.swift | 2 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 56 +++++++++-- crates/live_kit/src/live_kit.rs | 79 +++++++++++++--- 8 files changed, 178 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9dbdfe1fc6..7486a62082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2944,6 +2944,8 @@ dependencies = [ "core-foundation", "core-graphics", "futures", + "media", + "parking_lot 0.11.2", "serde", "serde_json", ] diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index f89bc4bf95..ee58aadf1a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,7 +1,6 @@ mod live_kit_token; -use std::time::Duration; - +use futures::StreamExt; use gpui::{ actions, elements::{Canvas, *}, @@ -13,6 +12,7 @@ use live_kit::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; use simplelog::SimpleLogger; +use std::sync::Arc; actions!(capture, [Quit]); @@ -36,47 +36,46 @@ fn main() { let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); - let background = cx.background().clone(); - cx.foreground() - .spawn(async move { - println!("connecting..."); - let user1_token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant-1", - ) + cx.spawn(|mut cx| async move { + let user1_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-1", + ) + .unwrap(); + let room1 = Room::new(); + room1.connect(&live_kit_url, &user1_token).await.unwrap(); + + let user2_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-2", + ) + .unwrap(); + let room2 = Room::new(); + room2.connect(&live_kit_url, &user2_token).await.unwrap(); + cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); + + let windows = live_kit::list_windows(); + let window = windows + .iter() + .find(|w| w.owner_name.as_deref() == Some("Safari")) .unwrap(); - let room1 = Room::new("user-1 room"); - room1.connect(&live_kit_url, &user1_token).await.unwrap(); + let track = LocalVideoTrack::screen_share_for_window(window.id); + room1.publish_video_track(&track).await.unwrap(); - let user2_token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant-2", - ) - .unwrap(); - let room2 = Room::new("user-2 room"); - room2.connect(&live_kit_url, &user2_token).await.unwrap(); - - let windows = live_kit::list_windows(); - println!("connected! {:?}", windows); - - let window_id = windows.iter().next().unwrap().id; - let track = LocalVideoTrack::screen_share_for_window(window_id); - room1.publish_video_track(&track).await.unwrap(); - - background.timer(Duration::from_secs(120)).await; - }) - .detach(); - - // cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); + std::mem::forget(track); + std::mem::forget(room1); + }) + .detach(); }); } struct ScreenCaptureView { image_buffer: Option, + _room: Arc, } impl gpui::Entity for ScreenCaptureView { @@ -84,8 +83,25 @@ impl gpui::Entity for ScreenCaptureView { } impl ScreenCaptureView { - pub fn new(_: &mut ViewContext) -> Self { - Self { image_buffer: None } + pub fn new(room: Arc, cx: &mut ViewContext) -> Self { + let mut remote_video_tracks = room.remote_video_tracks(); + cx.spawn_weak(|this, mut cx| async move { + if let Some(video_track) = remote_video_tracks.next().await { + video_track.add_renderer(move |frame| { + if let Some(this) = this.upgrade(&cx) { + this.update(&mut cx, |this, cx| { + this.image_buffer = Some(frame); + cx.notify(); + }); + } + }); + } + }) + .detach(); + Self { + image_buffer: None, + _room: room, + } } } diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index ad0915ed5b..21e67f3233 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -822,7 +822,7 @@ impl Renderer { { MTLPixelFormat::BGRA8Unorm } else { - panic!("unsupported pixel format") + MTLPixelFormat::R8Unorm }; let texture = self diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index d98b7eae88..e88d4f7b24 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -9,10 +9,13 @@ path = "src/live_kit.rs" doctest = false [dependencies] +media = { path = "../media" } + anyhow = "1.0.38" core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" +parking_lot = "0.11.1" [build-dependencies] serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved index df0b5d6243..e95cffe979 100644 --- a/crates/live_kit/LiveKitBridge/Package.resolved +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -5,8 +5,7 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift.git", "state" : { - "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", - "version" : "1.0.4" + "revision" : "5cc3c001779ab147199ce3ea0dce465b846368b4" } }, { diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift index 9e7039998d..fe71558804 100644 --- a/crates/live_kit/LiveKitBridge/Package.swift +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -15,7 +15,7 @@ let package = Package( targets: ["LiveKitBridge"]), ], dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", from: "1.0.0"), + .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "5cc3c001779ab147199ce3ea0dce465b846368b4"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 1802fd91fb..fb1eb9a79e 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,17 +1,49 @@ import Foundation import LiveKit +import WebRTC class LKRoomDelegate: RoomDelegate { var data: UnsafeRawPointer - var onDidSubscribeToRemoteTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void + var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void - init(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { + init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { self.data = data - self.onDidSubscribeToRemoteTrack = onDidSubscribeToRemoteTrack + self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack } func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { - self.onDidSubscribeToRemoteTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + if track.kind == .video { + self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + } + } +} + +class LKVideoRenderer: NSObject, VideoRenderer { + var data: UnsafeRawPointer + var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void + var onDrop: @convention(c) (UnsafeRawPointer) -> Void + var adaptiveStreamIsEnabled: Bool = false + var adaptiveStreamSize: CGSize = .zero + + init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { + self.data = data + self.onFrame = onFrame + self.onDrop = onDrop + } + + deinit { + self.onDrop(self.data) + } + + func setSize(_ size: CGSize) { + print("Called setSize", size); + } + + func renderFrame(_ frame: RTCVideoFrame?) { + let buffer = frame?.buffer as? RTCCVPixelBuffer + if let pixelBuffer = buffer?.pixelBuffer { + self.onFrame(self.data, pixelBuffer) + } } } @@ -21,8 +53,8 @@ public func LKRelease(ptr: UnsafeRawPointer) { } @_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteTrack: onDidSubscribeToRemoteTrack) +public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack) return Unmanaged.passRetained(delegate).toOpaque() } @@ -59,3 +91,15 @@ public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutable let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)) return Unmanaged.passRetained(track).toOpaque() } + +@_cdecl("LKVideoRendererCreate") +public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() +} + +@_cdecl("LKVideoTrackAddRenderer") +public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) { + let track = Unmanaged.fromOpaque(track).takeUnretainedValue() as! VideoTrack + let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() + track.add(videoRenderer: renderer) +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 0c254f99b3..59ce860a78 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -10,7 +10,12 @@ use core_graphics::window::{ kCGNullWindowID, kCGWindowListOptionExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, }; -use futures::{channel::oneshot, Future}; +use futures::{ + channel::{mpsc, oneshot}, + Future, +}; +use media::core_video::{CVImageBuffer, CVImageBufferRef}; +use parking_lot::Mutex; use std::{ ffi::c_void, sync::{Arc, Weak}, @@ -20,8 +25,8 @@ extern "C" { fn LKRelease(object: *const c_void); fn LKRoomDelegateCreate( - callback_data: *const c_void, - on_did_subscribe_to_remote_track: extern "C" fn( + callback_data: *mut c_void, + on_did_subscribe_to_remote_video_track: extern "C" fn( callback_data: *mut c_void, remote_track: *const c_void, ), @@ -42,22 +47,30 @@ extern "C" { callback_data: *mut c_void, ); + fn LKVideoRendererCreate( + callback_data: *mut c_void, + on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef), + on_drop: extern "C" fn(callback_data: *mut c_void), + ) -> *const c_void; + + fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } pub struct Room { - debug_name: &'static str, native_room: *const c_void, + remote_video_track_subscribers: Mutex>>>, _delegate: RoomDelegate, } impl Room { - pub fn new(debug_name: &'static str) -> Arc { + pub fn new() -> Arc { Arc::new_cyclic(|weak_room| { let delegate = RoomDelegate::new(weak_room.clone()); Self { - debug_name, native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, + remote_video_track_subscribers: Default::default(), _delegate: delegate, } }) @@ -88,8 +101,17 @@ impl Room { async { rx.await.unwrap().context("error publishing video track") } } - fn did_subscribe_to_remote_track(&self, track: RemoteVideoTrack) { - println!("{}: !!!!!!!!!!!!!!!!!!", self.debug_name); + pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver> { + let (tx, rx) = mpsc::unbounded(); + self.remote_video_track_subscribers.lock().push(tx); + rx + } + + fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { + let track = Arc::new(track); + self.remote_video_track_subscribers + .lock() + .retain(|tx| tx.unbounded_send(track.clone()).is_ok()); } fn build_done_callback() -> ( @@ -131,8 +153,8 @@ impl RoomDelegate { let weak_room = Weak::into_raw(weak_room); let native_delegate = unsafe { LKRoomDelegateCreate( - weak_room as *const c_void, - Self::on_did_subscribe_to_remote_track, + weak_room as *mut c_void, + Self::on_did_subscribe_to_remote_video_track, ) }; Self { @@ -141,11 +163,11 @@ impl RoomDelegate { } } - extern "C" fn on_did_subscribe_to_remote_track(room: *mut c_void, track: *const c_void) { + extern "C" fn on_did_subscribe_to_remote_video_track(room: *mut c_void, track: *const c_void) { let room = unsafe { Weak::from_raw(room as *mut Room) }; - let track = unsafe { RemoteVideoTrack(track) }; + let track = RemoteVideoTrack(track); if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_track(track); + room.did_subscribe_to_remote_video_track(track); } let _ = Weak::into_raw(room); } @@ -176,6 +198,37 @@ impl Drop for LocalVideoTrack { pub struct RemoteVideoTrack(*const c_void); +impl RemoteVideoTrack { + pub fn add_renderer(&self, callback: F) + where + F: 'static + FnMut(CVImageBuffer), + { + extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) + where + F: FnMut(CVImageBuffer), + { + unsafe { + let buffer = CVImageBuffer::wrap_under_get_rule(frame); + let callback = &mut *(callback_data as *mut F); + callback(buffer); + } + } + + extern "C" fn on_drop(callback_data: *mut c_void) { + unsafe { + let _ = Box::from_raw(callback_data as *mut F); + } + } + + let callback_data = Box::into_raw(Box::new(callback)); + unsafe { + let renderer = + LKVideoRendererCreate(callback_data as *mut c_void, on_frame::, on_drop::); + LKVideoTrackAddRenderer(self.0, renderer); + } + } +} + impl Drop for RemoteVideoTrack { fn drop(&mut self) { unsafe { LKRelease(self.0) } From 4e0380c9fb11d0dcf5d4d82b938d8d2eb5868924 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 14:51:35 +0200 Subject: [PATCH 47/89] Debounce frame assignment using a `watch` --- crates/capture/src/main.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ee58aadf1a..af2749ecd9 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -11,6 +11,7 @@ use gpui::{ use live_kit::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; +use postage::watch; use simplelog::SimpleLogger; use std::sync::Arc; @@ -65,9 +66,6 @@ fn main() { .unwrap(); let track = LocalVideoTrack::screen_share_for_window(window.id); room1.publish_video_track(&track).await.unwrap(); - - std::mem::forget(track); - std::mem::forget(room1); }) .detach(); }); @@ -87,17 +85,23 @@ impl ScreenCaptureView { let mut remote_video_tracks = room.remote_video_tracks(); cx.spawn_weak(|this, mut cx| async move { if let Some(video_track) = remote_video_tracks.next().await { - video_track.add_renderer(move |frame| { + let (mut frames_tx, mut frames_rx) = watch::channel_with(None); + video_track.add_renderer(move |frame| *frames_tx.borrow_mut() = Some(frame)); + + while let Some(frame) = frames_rx.next().await { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { - this.image_buffer = Some(frame); + this.image_buffer = frame; cx.notify(); }); + } else { + break; } - }); + } } }) .detach(); + Self { image_buffer: None, _room: room, From ca618b02b6ed2ca204b2c068008c352fc4f97859 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 15:39:15 +0200 Subject: [PATCH 48/89] Render surfaces correctly when encoded in `420YpCbCr8BiPlanarFullRange` --- crates/gpui/src/platform/mac/renderer.rs | 75 ++++--- .../gpui/src/platform/mac/shaders/shaders.h | 185 +++++++++--------- .../src/platform/mac/shaders/shaders.metal | 48 +++++ crates/media/src/media.rs | 15 +- 4 files changed, 202 insertions(+), 121 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 21e67f3233..dea9f0612e 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -28,6 +28,7 @@ pub struct Renderer { shadow_pipeline_state: metal::RenderPipelineState, sprite_pipeline_state: metal::RenderPipelineState, image_pipeline_state: metal::RenderPipelineState, + surface_pipeline_state: metal::RenderPipelineState, path_atlas_pipeline_state: metal::RenderPipelineState, underline_pipeline_state: metal::RenderPipelineState, unit_vertices: metal::Buffer, @@ -116,6 +117,14 @@ impl Renderer { "image_fragment", pixel_format, ); + let surface_pipeline_state = build_pipeline_state( + &device, + &library, + "surface", + "surface_vertex", + "surface_fragment", + pixel_format, + ); let path_atlas_pipeline_state = build_path_atlas_pipeline_state( &device, &library, @@ -141,6 +150,7 @@ impl Renderer { shadow_pipeline_state, sprite_pipeline_state, image_pipeline_state, + surface_pipeline_state, path_atlas_pipeline_state, underline_pipeline_state, unit_vertices, @@ -798,14 +808,14 @@ impl Renderer { return; } - command_encoder.set_render_pipeline_state(&self.image_pipeline_state); + command_encoder.set_render_pipeline_state(&self.surface_pipeline_state); command_encoder.set_vertex_buffer( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexVertices as u64, Some(&self.unit_vertices), 0, ); command_encoder.set_vertex_bytes( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexViewportSize as u64, mem::size_of::() as u64, [drawable_size.to_float2()].as_ptr() as *const c_void, ); @@ -817,64 +827,71 @@ impl Renderer { surface.image_buffer.height() as i32, ); let target_size = surface.bounds.size() * scale_factor; - let pixel_format = if surface.image_buffer.pixel_format_type() - == core_video::kCVPixelFormatType_32BGRA - { - MTLPixelFormat::BGRA8Unorm - } else { - MTLPixelFormat::R8Unorm - }; - let texture = self + assert_eq!( + surface.image_buffer.pixel_format_type(), + core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + ); + + let y_texture = self .cv_texture_cache .create_texture_from_image( surface.image_buffer.as_concrete_TypeRef(), ptr::null(), - pixel_format, - source_size.x() as usize, - source_size.y() as usize, + MTLPixelFormat::R8Unorm, + surface.image_buffer.plane_width(0), + surface.image_buffer.plane_height(0), 0, ) .unwrap(); + let cb_cr_texture = self + .cv_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + ptr::null(), + MTLPixelFormat::RG8Unorm, + surface.image_buffer.plane_width(1), + surface.image_buffer.plane_height(1), + 1, + ) + .unwrap(); align_offset(offset); - let next_offset = *offset + mem::size_of::(); + let next_offset = *offset + mem::size_of::(); assert!( next_offset <= INSTANCE_BUFFER_SIZE, "instance buffer exhausted" ); command_encoder.set_vertex_buffer( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexSurfaces as u64, Some(&self.instances), *offset as u64, ); command_encoder.set_vertex_bytes( - shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, + shaders::GPUISurfaceVertexInputIndex_GPUISurfaceVertexInputIndexAtlasSize as u64, mem::size_of::() as u64, [source_size.to_float2()].as_ptr() as *const c_void, ); command_encoder.set_fragment_texture( - shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, - Some(texture.as_texture_ref()), + shaders::GPUISurfaceFragmentInputIndex_GPUISurfaceFragmentInputIndexYAtlas as u64, + Some(y_texture.as_texture_ref()), + ); + command_encoder.set_fragment_texture( + shaders::GPUISurfaceFragmentInputIndex_GPUISurfaceFragmentInputIndexCbCrAtlas + as u64, + Some(cb_cr_texture.as_texture_ref()), ); unsafe { - let buffer_contents = - (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; + let buffer_contents = (self.instances.contents() as *mut u8).add(*offset) + as *mut shaders::GPUISurface; std::ptr::write( buffer_contents, - shaders::GPUIImage { + shaders::GPUISurface { origin: origin.to_float2(), target_size: target_size.to_float2(), source_size: source_size.to_float2(), - atlas_origin: Default::default(), - border_top: Default::default(), - border_right: Default::default(), - border_bottom: Default::default(), - border_left: Default::default(), - border_color: Default::default(), - corner_radius: Default::default(), }, ); } diff --git a/crates/gpui/src/platform/mac/shaders/shaders.h b/crates/gpui/src/platform/mac/shaders/shaders.h index 3f5096f37c..29be2c9e1e 100644 --- a/crates/gpui/src/platform/mac/shaders/shaders.h +++ b/crates/gpui/src/platform/mac/shaders/shaders.h @@ -1,122 +1,125 @@ #include -typedef struct -{ - vector_float2 viewport_size; +typedef struct { + vector_float2 viewport_size; } GPUIUniforms; -typedef enum -{ - GPUIQuadInputIndexVertices = 0, - GPUIQuadInputIndexQuads = 1, - GPUIQuadInputIndexUniforms = 2, +typedef enum { + GPUIQuadInputIndexVertices = 0, + GPUIQuadInputIndexQuads = 1, + GPUIQuadInputIndexUniforms = 2, } GPUIQuadInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 size; - vector_uchar4 background_color; - float border_top; - float border_right; - float border_bottom; - float border_left; - vector_uchar4 border_color; - float corner_radius; +typedef struct { + vector_float2 origin; + vector_float2 size; + vector_uchar4 background_color; + float border_top; + float border_right; + float border_bottom; + float border_left; + vector_uchar4 border_color; + float corner_radius; } GPUIQuad; -typedef enum -{ - GPUIShadowInputIndexVertices = 0, - GPUIShadowInputIndexShadows = 1, - GPUIShadowInputIndexUniforms = 2, +typedef enum { + GPUIShadowInputIndexVertices = 0, + GPUIShadowInputIndexShadows = 1, + GPUIShadowInputIndexUniforms = 2, } GPUIShadowInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 size; - float corner_radius; - float sigma; - vector_uchar4 color; +typedef struct { + vector_float2 origin; + vector_float2 size; + float corner_radius; + float sigma; + vector_uchar4 color; } GPUIShadow; -typedef enum -{ - GPUISpriteVertexInputIndexVertices = 0, - GPUISpriteVertexInputIndexSprites = 1, - GPUISpriteVertexInputIndexViewportSize = 2, - GPUISpriteVertexInputIndexAtlasSize = 3, +typedef enum { + GPUISpriteVertexInputIndexVertices = 0, + GPUISpriteVertexInputIndexSprites = 1, + GPUISpriteVertexInputIndexViewportSize = 2, + GPUISpriteVertexInputIndexAtlasSize = 3, } GPUISpriteVertexInputIndex; -typedef enum -{ - GPUISpriteFragmentInputIndexAtlas = 0, +typedef enum { + GPUISpriteFragmentInputIndexAtlas = 0, } GPUISpriteFragmentInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 target_size; - vector_float2 source_size; - vector_float2 atlas_origin; - vector_uchar4 color; - uint8_t compute_winding; +typedef struct { + vector_float2 origin; + vector_float2 target_size; + vector_float2 source_size; + vector_float2 atlas_origin; + vector_uchar4 color; + uint8_t compute_winding; } GPUISprite; -typedef enum -{ - GPUIPathAtlasVertexInputIndexVertices = 0, - GPUIPathAtlasVertexInputIndexAtlasSize = 1, +typedef enum { + GPUIPathAtlasVertexInputIndexVertices = 0, + GPUIPathAtlasVertexInputIndexAtlasSize = 1, } GPUIPathAtlasVertexInputIndex; -typedef struct -{ - vector_float2 xy_position; - vector_float2 st_position; - vector_float2 clip_rect_origin; - vector_float2 clip_rect_size; +typedef struct { + vector_float2 xy_position; + vector_float2 st_position; + vector_float2 clip_rect_origin; + vector_float2 clip_rect_size; } GPUIPathVertex; -typedef enum -{ - GPUIImageVertexInputIndexVertices = 0, - GPUIImageVertexInputIndexImages = 1, - GPUIImageVertexInputIndexViewportSize = 2, - GPUIImageVertexInputIndexAtlasSize = 3, +typedef enum { + GPUIImageVertexInputIndexVertices = 0, + GPUIImageVertexInputIndexImages = 1, + GPUIImageVertexInputIndexViewportSize = 2, + GPUIImageVertexInputIndexAtlasSize = 3, } GPUIImageVertexInputIndex; -typedef enum -{ - GPUIImageFragmentInputIndexAtlas = 0, +typedef enum { + GPUIImageFragmentInputIndexAtlas = 0, } GPUIImageFragmentInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 target_size; - vector_float2 source_size; - vector_float2 atlas_origin; - float border_top; - float border_right; - float border_bottom; - float border_left; - vector_uchar4 border_color; - float corner_radius; +typedef struct { + vector_float2 origin; + vector_float2 target_size; + vector_float2 source_size; + vector_float2 atlas_origin; + float border_top; + float border_right; + float border_bottom; + float border_left; + vector_uchar4 border_color; + float corner_radius; } GPUIImage; -typedef enum -{ - GPUIUnderlineInputIndexVertices = 0, - GPUIUnderlineInputIndexUnderlines = 1, - GPUIUnderlineInputIndexUniforms = 2, +typedef enum { + GPUISurfaceVertexInputIndexVertices = 0, + GPUISurfaceVertexInputIndexSurfaces = 1, + GPUISurfaceVertexInputIndexViewportSize = 2, + GPUISurfaceVertexInputIndexAtlasSize = 3, +} GPUISurfaceVertexInputIndex; + +typedef enum { + GPUISurfaceFragmentInputIndexYAtlas = 0, + GPUISurfaceFragmentInputIndexCbCrAtlas = 1, +} GPUISurfaceFragmentInputIndex; + +typedef struct { + vector_float2 origin; + vector_float2 target_size; + vector_float2 source_size; +} GPUISurface; + +typedef enum { + GPUIUnderlineInputIndexVertices = 0, + GPUIUnderlineInputIndexUnderlines = 1, + GPUIUnderlineInputIndexUniforms = 2, } GPUIUnderlineInputIndex; -typedef struct -{ - vector_float2 origin; - vector_float2 size; - float thickness; - vector_uchar4 color; - uint8_t squiggly; +typedef struct { + vector_float2 origin; + vector_float2 size; + float thickness; + vector_uchar4 color; + uint8_t squiggly; } GPUIUnderline; diff --git a/crates/gpui/src/platform/mac/shaders/shaders.metal b/crates/gpui/src/platform/mac/shaders/shaders.metal index 2d79e69d56..795026e747 100644 --- a/crates/gpui/src/platform/mac/shaders/shaders.metal +++ b/crates/gpui/src/platform/mac/shaders/shaders.metal @@ -263,6 +263,54 @@ fragment float4 image_fragment( return quad_sdf(input); } +vertex QuadFragmentInput surface_vertex( + uint unit_vertex_id [[vertex_id]], + uint image_id [[instance_id]], + constant float2 *unit_vertices [[buffer(GPUISurfaceVertexInputIndexVertices)]], + constant GPUISurface *images [[buffer(GPUISurfaceVertexInputIndexSurfaces)]], + constant float2 *viewport_size [[buffer(GPUISurfaceVertexInputIndexViewportSize)]], + constant float2 *atlas_size [[buffer(GPUISurfaceVertexInputIndexAtlasSize)]] +) { + float2 unit_vertex = unit_vertices[unit_vertex_id]; + GPUISurface image = images[image_id]; + float2 position = unit_vertex * image.target_size + image.origin; + float4 device_position = to_device_position(position, *viewport_size); + float2 atlas_position = (unit_vertex * image.source_size) / *atlas_size; + + return QuadFragmentInput { + device_position, + atlas_position, + image.origin, + image.target_size, + float4(0.), + 0., + 0., + 0., + 0., + float4(0.), + 0., + }; +} + +fragment float4 surface_fragment( + QuadFragmentInput input [[stage_in]], + texture2d y_atlas [[ texture(GPUISurfaceFragmentInputIndexYAtlas) ]], + texture2d cb_cr_atlas [[ texture(GPUISurfaceFragmentInputIndexCbCrAtlas) ]] +) { + constexpr sampler atlas_sampler(mag_filter::linear, min_filter::linear); + const float4x4 ycbcrToRGBTransform = float4x4( + float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f), + float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f), + float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f), + float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f) + ); + float4 ycbcr = float4(y_atlas.sample(atlas_sampler, input.atlas_position).r, + cb_cr_atlas.sample(atlas_sampler, input.atlas_position).rg, 1.0); + + input.background_color = ycbcrToRGBTransform * ycbcr; + return quad_sdf(input); +} + struct PathAtlasVertexOutput { float4 position [[position]]; float2 st_position; diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index ebe3ef7f4d..fe69e684e7 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -31,7 +31,10 @@ pub mod core_video { #![allow(non_snake_case)] use super::*; - pub use crate::bindings::kCVPixelFormatType_32BGRA; + pub use crate::bindings::{ + kCVPixelFormatType_32BGRA, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, + }; use crate::bindings::{kCVReturnSuccess, CVReturn, OSType}; use anyhow::{anyhow, Result}; use core_foundation::{ @@ -68,6 +71,14 @@ pub mod core_video { unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } } + pub fn plane_width(&self, plane: usize) -> usize { + unsafe { CVPixelBufferGetWidthOfPlane(self.as_concrete_TypeRef(), plane) } + } + + pub fn plane_height(&self, plane: usize) -> usize { + unsafe { CVPixelBufferGetHeightOfPlane(self.as_concrete_TypeRef(), plane) } + } + pub fn pixel_format_type(&self) -> OSType { unsafe { CVPixelBufferGetPixelFormatType(self.as_concrete_TypeRef()) } } @@ -79,6 +90,8 @@ pub mod core_video { fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetWidthOfPlane(buffer: CVImageBufferRef, plane: usize) -> usize; + fn CVPixelBufferGetHeightOfPlane(buffer: CVImageBufferRef, plane: usize) -> usize; fn CVPixelBufferGetPixelFormatType(buffer: CVImageBufferRef) -> OSType; } From 21c91a29e7c18b88c2715b2984f3a60e34ff126c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 22 Aug 2022 18:17:14 +0200 Subject: [PATCH 49/89] Add the ability to hide the titlebar when creating windows Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform.rs | 15 ++++++++--- crates/gpui/src/platform/mac/window.rs | 36 +++++++++++++++++--------- crates/zed/src/zed.rs | 10 ++++--- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index 38e8348ac3..e31cc46213 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -136,8 +136,13 @@ pub trait WindowContext { #[derive(Debug)] pub struct WindowOptions<'a> { pub bounds: WindowBounds, + pub titlebar: Option>, +} + +#[derive(Debug)] +pub struct TitlebarOptions<'a> { pub title: Option<&'a str>, - pub titlebar_appears_transparent: bool, + pub appears_transparent: bool, pub traffic_light_position: Option, } @@ -246,9 +251,11 @@ impl<'a> Default for WindowOptions<'a> { fn default() -> Self { Self { bounds: WindowBounds::Maximized, - title: Default::default(), - titlebar_appears_transparent: Default::default(), - traffic_light_position: Default::default(), + titlebar: Some(TitlebarOptions { + title: Default::default(), + appears_transparent: Default::default(), + traffic_light_position: Default::default(), + }), } } } diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 9b52041d80..5ceb8c96be 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -339,13 +339,19 @@ impl Window { WindowBounds::Fixed(rect) => rect, } .to_ns_rect(); - let mut style_mask = NSWindowStyleMask::NSClosableWindowMask - | NSWindowStyleMask::NSMiniaturizableWindowMask - | NSWindowStyleMask::NSResizableWindowMask - | NSWindowStyleMask::NSTitledWindowMask; - if options.titlebar_appears_transparent { - style_mask |= NSWindowStyleMask::NSFullSizeContentViewWindowMask; + let mut style_mask; + if let Some(titlebar) = options.titlebar.as_ref() { + style_mask = NSWindowStyleMask::NSClosableWindowMask + | NSWindowStyleMask::NSMiniaturizableWindowMask + | NSWindowStyleMask::NSResizableWindowMask + | NSWindowStyleMask::NSTitledWindowMask; + + if titlebar.appears_transparent { + style_mask |= NSWindowStyleMask::NSFullSizeContentViewWindowMask; + } + } else { + style_mask = NSWindowStyleMask::empty(); } let native_window: id = msg_send![WINDOW_CLASS, alloc]; @@ -409,7 +415,10 @@ impl Window { command_queue: device.new_command_queue(), last_fresh_keydown: None, layer, - traffic_light_position: options.traffic_light_position, + traffic_light_position: options + .titlebar + .as_ref() + .and_then(|titlebar| titlebar.traffic_light_position), previous_modifiers_changed_event: None, ime_state: ImeState::None, ime_text: None, @@ -425,12 +434,15 @@ impl Window { Rc::into_raw(window.0.clone()) as *const c_void, ); - if let Some(title) = options.title.as_ref() { - native_window.setTitle_(NSString::alloc(nil).init_str(title)); - } - if options.titlebar_appears_transparent { - native_window.setTitlebarAppearsTransparent_(YES); + if let Some(titlebar) = options.titlebar { + if let Some(title) = titlebar.title { + native_window.setTitle_(NSString::alloc(nil).init_str(title)); + } + if titlebar.appears_transparent { + native_window.setTitlebarAppearsTransparent_(YES); + } } + native_window.setAcceptsMouseMovedEvents_(YES); native_view.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index a8f10a1579..59e9f8de9d 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -20,7 +20,7 @@ use gpui::{ geometry::vector::vec2f, impl_actions, platform::{WindowBounds, WindowOptions}, - AssetSource, AsyncAppContext, ViewContext, + AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, }; use language::Rope; pub use lsp; @@ -330,9 +330,11 @@ pub fn initialize_workspace( pub fn build_window_options() -> WindowOptions<'static> { WindowOptions { bounds: WindowBounds::Maximized, - title: None, - titlebar_appears_transparent: true, - traffic_light_position: Some(vec2f(8., 8.)), + titlebar: Some(TitlebarOptions { + title: None, + appears_transparent: true, + traffic_light_position: Some(vec2f(8., 8.)), + }), } } From c03300df29250045175c7427efe40c3f611514d9 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 22 Aug 2022 18:17:48 +0200 Subject: [PATCH 50/89] WIP: Start on `App::add_status_bar_item` Co-Authored-By: Nathan Sobo --- crates/gpui/src/app.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index e9091d74c8..9598540166 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1926,6 +1926,29 @@ impl MutableAppContext { }) } + // pub fn add_status_bar_item( + // &mut self, + // build_item: F1, + // build_menu: F2, + // menu_bounds: Vector2F, + // ) where + // I: View, + // M: View, + // F1: FnOnce(&mut ViewContext) -> I, + // F2: FnOnce(&mut ViewContext) -> M, + // { + // self.add_window( + // WindowOptions { + // bounds: menu_bounds, + // titlebar: None, + // title: None, + // titlebar_appears_transparent: true, + // traffic_light_position: (), + // }, + // build_root_view, + // ) + // } + pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle where T: View, From 2b9fe0a2e6d32f025a093ef142caabcf2baeb117 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 8 Sep 2022 19:07:11 +0200 Subject: [PATCH 51/89] WIP --- crates/capture/src/main.rs | 16 ++++++-- crates/gpui/src/app.rs | 23 ++--------- crates/gpui/src/platform.rs | 13 +++++-- crates/gpui/src/platform/mac.rs | 1 + crates/gpui/src/platform/mac/platform.rs | 42 ++++++++++++--------- crates/gpui/src/platform/mac/status_item.rs | 25 ++++++++++++ crates/gpui/src/platform/test.rs | 18 +++++---- crates/zed/src/main.rs | 2 + 8 files changed, 89 insertions(+), 51 deletions(-) create mode 100644 crates/gpui/src/platform/mac/status_item.rs diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index af2749ecd9..c34f451e41 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -116,15 +116,25 @@ impl gpui::View for ScreenCaptureView { fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { let image_buffer = self.image_buffer.clone(); - Canvas::new(move |bounds, _, cx| { + let canvas = Canvas::new(move |bounds, _, cx| { if let Some(image_buffer) = image_buffer.clone() { cx.scene.push_surface(Surface { bounds, image_buffer, }); } - }) - .boxed() + }); + + if let Some(image_buffer) = self.image_buffer.as_ref() { + canvas + .constrained() + .with_width(image_buffer.width() as f32) + .with_height(image_buffer.height() as f32) + .aligned() + .boxed() + } else { + canvas.boxed() + } } } diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 9598540166..4a749b551f 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1926,27 +1926,12 @@ impl MutableAppContext { }) } - // pub fn add_status_bar_item( - // &mut self, - // build_item: F1, - // build_menu: F2, - // menu_bounds: Vector2F, - // ) where + // pub fn add_status_bar_item(&mut self, build_item: F) + // where // I: View, - // M: View, - // F1: FnOnce(&mut ViewContext) -> I, - // F2: FnOnce(&mut ViewContext) -> M, + // F: FnOnce(&mut ViewContext) -> I, // { - // self.add_window( - // WindowOptions { - // bounds: menu_bounds, - // titlebar: None, - // title: None, - // titlebar_appears_transparent: true, - // traffic_light_position: (), - // }, - // build_root_view, - // ) + // mem::forget(self.platform.add_status_item()); // } pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index e31cc46213..ec11ee196a 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -39,6 +39,11 @@ pub trait Platform: Send + Sync { fn fonts(&self) -> Arc; fn activate(&self, ignoring_other_apps: bool); + fn hide(&self); + fn hide_other_apps(&self); + fn unhide_other_apps(&self); + fn quit(&self); + fn open_window( &self, id: usize, @@ -46,10 +51,8 @@ pub trait Platform: Send + Sync { executor: Rc, ) -> Box; fn key_window_id(&self) -> Option; - fn hide(&self); - fn hide_other_apps(&self); - fn unhide_other_apps(&self); - fn quit(&self); + + fn add_status_item(&self) -> Box; fn write_to_clipboard(&self, item: ClipboardItem); fn read_from_clipboard(&self) -> Option; @@ -133,6 +136,8 @@ pub trait WindowContext { fn present_scene(&mut self, scene: Scene); } +pub trait StatusItem {} + #[derive(Debug)] pub struct WindowOptions<'a> { pub bounds: WindowBounds, diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index c1a4d45561..bd3c4eaf20 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -7,6 +7,7 @@ mod image_cache; mod platform; mod renderer; mod sprite_cache; +mod status_item; mod window; use cocoa::base::{BOOL, NO, YES}; diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index 113653478a..cda6521a8a 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -1,4 +1,6 @@ -use super::{event::key_to_native, BoolExt as _, Dispatcher, FontSystem, Window}; +use super::{ + event::key_to_native, status_item::StatusItem, BoolExt as _, Dispatcher, FontSystem, Window, +}; use crate::{ executor, keymap, platform::{self, CursorStyle}, @@ -439,23 +441,6 @@ impl platform::Platform for MacPlatform { } } - fn open_window( - &self, - id: usize, - options: platform::WindowOptions, - executor: Rc, - ) -> Box { - Box::new(Window::open(id, options, executor, self.fonts())) - } - - fn key_window_id(&self) -> Option { - Window::key_window_id() - } - - fn fonts(&self) -> Arc { - self.fonts.clone() - } - fn hide(&self) { unsafe { let app = NSApplication::sharedApplication(nil); @@ -497,6 +482,27 @@ impl platform::Platform for MacPlatform { } } + fn open_window( + &self, + id: usize, + options: platform::WindowOptions, + executor: Rc, + ) -> Box { + Box::new(Window::open(id, options, executor, self.fonts())) + } + + fn key_window_id(&self) -> Option { + Window::key_window_id() + } + + fn add_status_item(&self) -> Box { + Box::new(StatusItem::add()) + } + + fn fonts(&self) -> Arc { + self.fonts.clone() + } + fn write_to_clipboard(&self, item: ClipboardItem) { unsafe { self.pasteboard.clearContents(); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs new file mode 100644 index 0000000000..6b32d5cb40 --- /dev/null +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -0,0 +1,25 @@ +use cocoa::{ + appkit::{NSSquareStatusItemLength, NSStatusBar}, + base::{id, nil}, +}; +use core_foundation::base::TCFType; +use core_graphics::color::CGColor; +use objc::{msg_send, sel, sel_impl}; + +pub struct StatusItem(id); + +impl StatusItem { + pub fn add() -> Self { + unsafe { + let status_bar = NSStatusBar::systemStatusBar(nil); + let native_item: id = + msg_send![status_bar, statusItemWithLength: NSSquareStatusItemLength]; + let button: id = msg_send![native_item, button]; + let layer: id = msg_send![button, layer]; + let _: () = msg_send![layer, setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef()]; + StatusItem(native_item) + } + } +} + +impl crate::StatusItem for StatusItem {} diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 519ba931d1..7fe7aca669 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -120,6 +120,14 @@ impl super::Platform for Platform { fn activate(&self, _ignoring_other_apps: bool) {} + fn hide(&self) {} + + fn hide_other_apps(&self) {} + + fn unhide_other_apps(&self) {} + + fn quit(&self) {} + fn open_window( &self, _: usize, @@ -136,13 +144,9 @@ impl super::Platform for Platform { None } - fn hide(&self) {} - - fn hide_other_apps(&self) {} - - fn unhide_other_apps(&self) {} - - fn quit(&self) {} + fn add_status_item(&self) -> Box { + todo!() + } fn write_to_clipboard(&self, item: ClipboardItem) { *self.current_clipboard_item.lock() = Some(item); diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index fc3616d592..fafc954d2c 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -87,6 +87,8 @@ fn main() { }); app.run(move |cx| { + std::mem::forget(cx.platform().add_status_item()); + let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); From 1c810d7e8d2737be1abea3f797d74e0a1ce24a82 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 9 Sep 2022 15:07:41 +0200 Subject: [PATCH 52/89] WIP: Show status bar item with a backing metal layer Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform/mac/status_item.rs | 54 +++++++++++++++++---- crates/zed/src/main.rs | 1 - 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 6b32d5cb40..29bae98210 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,22 +1,58 @@ use cocoa::{ - appkit::{NSSquareStatusItemLength, NSStatusBar}, - base::{id, nil}, + appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView}, + base::{id, nil, NO, YES}, + foundation::NSRect, + quartzcore::AutoresizingMask, }; use core_foundation::base::TCFType; use core_graphics::color::CGColor; -use objc::{msg_send, sel, sel_impl}; +use foreign_types::ForeignType; +use objc::{class, msg_send, rc::StrongPtr, sel, sel_impl}; -pub struct StatusItem(id); +pub struct StatusItem(StrongPtr); impl StatusItem { pub fn add() -> Self { + const PIXEL_FORMAT: metal::MTLPixelFormat = metal::MTLPixelFormat::BGRA8Unorm; + unsafe { let status_bar = NSStatusBar::systemStatusBar(nil); - let native_item: id = - msg_send![status_bar, statusItemWithLength: NSSquareStatusItemLength]; - let button: id = msg_send![native_item, button]; - let layer: id = msg_send![button, layer]; - let _: () = msg_send![layer, setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef()]; + let native_item = + StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); + native_item.button().setWantsLayer(true); + + let device: metal::Device = if let Some(device) = metal::Device::system_default() { + device + } else { + log::error!("unable to access a compatible graphics device"); + std::process::exit(1); + }; + + let layer: id = msg_send![class!(CAMetalLayer), layer]; + let _: () = msg_send![layer, setDevice: device.as_ptr()]; + let _: () = msg_send![layer, setPixelFormat: PIXEL_FORMAT]; + let _: () = msg_send![layer, setAllowsNextDrawableTimeout: NO]; + let _: () = msg_send![layer, setNeedsDisplayOnBoundsChange: YES]; + let _: () = msg_send![layer, setPresentsWithTransaction: YES]; + let _: () = msg_send![ + layer, + setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE + | AutoresizingMask::HEIGHT_SIZABLE + ]; + let _: () = msg_send![ + layer, + setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef() + ]; + + let _: () = msg_send![native_item.button(), setLayer: layer]; + let native_item_window: id = msg_send![native_item.button(), window]; + + dbg!(native_item_window.frame().as_CGRect()); + // let rect_in_window: NSRect = msg_send![native_item.button(), convertRect: native_item.button().bounds() toView: nil]; + // let screen_rect: NSRect = + // msg_send![native_item_window, convertRectToScreen: rect_in_window]; + // dbg!(screen_rect.as_CGRect()); + StatusItem(native_item) } } diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index fafc954d2c..43a403ab7f 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -88,7 +88,6 @@ fn main() { app.run(move |cx| { std::mem::forget(cx.platform().add_status_item()); - let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); From e803dd9f72be0055c337e082bd500d9881a4f68c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 9 Sep 2022 15:24:28 +0200 Subject: [PATCH 53/89] Remove `platform::WindowContext` trait Co-Authored-By: Nathan Sobo --- crates/gpui/src/platform.rs | 4 +-- crates/gpui/src/platform/mac/status_item.rs | 1 - crates/gpui/src/platform/mac/window.rs | 6 +--- crates/gpui/src/platform/test.rs | 34 ++++++++++----------- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index ec11ee196a..b2e1995920 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -110,7 +110,7 @@ pub trait InputHandler { fn rect_for_range(&self, range_utf16: Range) -> Option; } -pub trait Window: WindowContext { +pub trait Window { fn as_any_mut(&mut self) -> &mut dyn Any; fn on_event(&mut self, callback: Box bool>); fn on_active_status_change(&mut self, callback: Box); @@ -127,9 +127,7 @@ pub trait Window: WindowContext { fn minimize(&self); fn zoom(&self); fn toggle_full_screen(&self); -} -pub trait WindowContext { fn size(&self) -> Vector2F; fn scale_factor(&self) -> f32; fn titlebar_height(&self) -> f32; diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 29bae98210..a672e1e323 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,7 +1,6 @@ use cocoa::{ appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView}, base::{id, nil, NO, YES}, - foundation::NSRect, quartzcore::AutoresizingMask, }; use core_foundation::base::TCFType; diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 5ceb8c96be..ed1231588c 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -6,7 +6,7 @@ use crate::{ vector::{vec2f, Vector2F}, }, keymap::Keystroke, - platform::{self, Event, WindowBounds, WindowContext}, + platform::{self, Event, WindowBounds}, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, MouseMovedEvent, Scene, }; @@ -649,9 +649,7 @@ impl platform::Window for Window { }) .detach(); } -} -impl platform::WindowContext for Window { fn size(&self) -> Vector2F { self.0.as_ref().borrow().size() } @@ -713,9 +711,7 @@ impl WindowState { } } } -} -impl platform::WindowContext for WindowState { fn size(&self) -> Vector2F { let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_window.contentView()) }.size; diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 7fe7aca669..f64307debf 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -228,24 +228,6 @@ impl super::Dispatcher for Dispatcher { } } -impl super::WindowContext for Window { - fn size(&self) -> Vector2F { - self.size - } - - fn scale_factor(&self) -> f32 { - self.scale_factor - } - - fn titlebar_height(&self) -> f32 { - 24. - } - - fn present_scene(&mut self, scene: crate::Scene) { - self.current_scene = Some(scene); - } -} - impl super::Window for Window { fn as_any_mut(&mut self) -> &mut dyn Any { self @@ -300,6 +282,22 @@ impl super::Window for Window { fn zoom(&self) {} fn toggle_full_screen(&self) {} + + fn size(&self) -> Vector2F { + self.size + } + + fn scale_factor(&self) -> f32 { + self.scale_factor + } + + fn titlebar_height(&self) -> f32 { + 24. + } + + fn present_scene(&mut self, scene: crate::Scene) { + self.current_scene = Some(scene); + } } pub fn platform() -> Platform { From f50c6af00176abd573a965358c25f34b0e313e60 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 12 Sep 2022 10:07:13 +0200 Subject: [PATCH 54/89] Encapsulate metal layer into `Renderer` --- crates/gpui/src/platform/mac/renderer.rs | 92 ++++++++++++++++-------- crates/gpui/src/platform/mac/window.rs | 71 +++++------------- 2 files changed, 79 insertions(+), 84 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index dea9f0612e..f9cc6e730d 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -8,12 +8,17 @@ use crate::{ platform, scene::{Glyph, Icon, Image, ImageGlyph, Layer, Quad, Scene, Shadow, Underline}, }; -use cocoa::foundation::NSUInteger; +use cocoa::{ + base::{NO, YES}, + foundation::{NSRect, NSUInteger}, + quartzcore::AutoresizingMask, +}; use core_foundation::base::TCFType; use foreign_types::ForeignTypeRef; use log::warn; use media::core_video::{self, CVMetalTextureCache}; -use metal::{MTLPixelFormat, MTLResourceOptions, NSRange}; +use metal::{CGFloat, CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; +use objc::{self, msg_send, sel, sel_impl}; use shaders::ToFloat2 as _; use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, ptr, sync::Arc, vec}; @@ -21,6 +26,8 @@ const SHADERS_METALLIB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/shader const INSTANCE_BUFFER_SIZE: usize = 8192 * 1024; // This is an arbitrary decision. There's probably a more optimal value. pub struct Renderer { + layer: metal::MetalLayer, + command_queue: CommandQueue, sprite_cache: SpriteCache, image_cache: ImageCache, path_atlases: AtlasAllocator, @@ -48,12 +55,30 @@ pub struct Surface { } impl Renderer { - pub fn new( - device: metal::Device, - pixel_format: metal::MTLPixelFormat, - scale_factor: f32, - fonts: Arc, - ) -> Self { + pub fn new(fonts: Arc) -> Self { + const PIXEL_FORMAT: MTLPixelFormat = MTLPixelFormat::BGRA8Unorm; + + let device: metal::Device = if let Some(device) = metal::Device::system_default() { + device + } else { + log::error!("unable to access a compatible graphics device"); + std::process::exit(1); + }; + + let layer = metal::MetalLayer::new(); + layer.set_device(&device); + layer.set_pixel_format(PIXEL_FORMAT); + layer.set_presents_with_transaction(true); + unsafe { + let _: () = msg_send![&*layer, setAllowsNextDrawableTimeout: NO]; + let _: () = msg_send![&*layer, setNeedsDisplayOnBoundsChange: YES]; + let _: () = msg_send![ + &*layer, + setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE + | AutoresizingMask::HEIGHT_SIZABLE + ]; + } + let library = device .new_library_with_data(SHADERS_METALLIB) .expect("error building metal library"); @@ -76,13 +101,8 @@ impl Renderer { MTLResourceOptions::StorageModeManaged, ); - let sprite_cache = SpriteCache::new( - device.clone(), - vec2i(1024, 768), - scale_factor, - fonts.clone(), - ); - let image_cache = ImageCache::new(device.clone(), vec2i(1024, 768), scale_factor, fonts); + let sprite_cache = SpriteCache::new(device.clone(), vec2i(1024, 768), 1., fonts.clone()); + let image_cache = ImageCache::new(device.clone(), vec2i(1024, 768), 1., fonts); let path_atlases = AtlasAllocator::new(device.clone(), build_path_atlas_texture_descriptor()); let quad_pipeline_state = build_pipeline_state( @@ -91,7 +111,7 @@ impl Renderer { "quad", "quad_vertex", "quad_fragment", - pixel_format, + PIXEL_FORMAT, ); let shadow_pipeline_state = build_pipeline_state( &device, @@ -99,7 +119,7 @@ impl Renderer { "shadow", "shadow_vertex", "shadow_fragment", - pixel_format, + PIXEL_FORMAT, ); let sprite_pipeline_state = build_pipeline_state( &device, @@ -107,7 +127,7 @@ impl Renderer { "sprite", "sprite_vertex", "sprite_fragment", - pixel_format, + PIXEL_FORMAT, ); let image_pipeline_state = build_pipeline_state( &device, @@ -115,7 +135,7 @@ impl Renderer { "image", "image_vertex", "image_fragment", - pixel_format, + PIXEL_FORMAT, ); let surface_pipeline_state = build_pipeline_state( &device, @@ -123,7 +143,7 @@ impl Renderer { "surface", "surface_vertex", "surface_fragment", - pixel_format, + PIXEL_FORMAT, ); let path_atlas_pipeline_state = build_path_atlas_pipeline_state( &device, @@ -139,10 +159,12 @@ impl Renderer { "underline", "underline_vertex", "underline_fragment", - pixel_format, + PIXEL_FORMAT, ); let cv_texture_cache = CVMetalTextureCache::new(device.as_ptr()).unwrap(); Self { + layer, + command_queue: device.new_command_queue(), sprite_cache, image_cache, path_atlases, @@ -159,13 +181,21 @@ impl Renderer { } } - pub fn render( - &mut self, - scene: &Scene, - drawable_size: Vector2F, - command_buffer: &metal::CommandBufferRef, - output: &metal::TextureRef, - ) { + pub fn layer(&self) -> &metal::MetalLayerRef { + &*self.layer + } + + pub fn render(&mut self, scene: &Scene) { + let layer = self.layer.clone(); + let drawable = layer.next_drawable().unwrap(); + let command_queue = self.command_queue.clone(); + let command_buffer = command_queue.new_command_buffer(); + + let frame: NSRect = unsafe { msg_send![self.layer(), frame] }; + let scale_factor: CGFloat = unsafe { msg_send![self.layer(), contentsScale] }; + let drawable_size = + vec2f(frame.size.width as f32, frame.size.height as f32) * scale_factor as f32; + self.sprite_cache.set_scale_factor(scene.scale_factor()); self.image_cache.set_scale_factor(scene.scale_factor()); @@ -178,13 +208,17 @@ impl Renderer { &mut offset, drawable_size, command_buffer, - output, + drawable.texture(), ); self.instances.did_modify_range(NSRange { location: 0, length: offset as NSUInteger, }); self.image_cache.finish_frame(); + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drawable.present(); } fn render_path_atlases( diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index ed1231588c..822e1700f1 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -20,11 +20,10 @@ use cocoa::{ foundation::{ NSAutoreleasePool, NSInteger, NSNotFound, NSPoint, NSRect, NSSize, NSString, NSUInteger, }, - quartzcore::AutoresizingMask, }; use core_graphics::display::CGRect; use ctor::ctor; -use foreign_types::ForeignType as _; +use foreign_types::ForeignTypeRef; use objc::{ class, declare::ClassDecl, @@ -306,9 +305,7 @@ struct WindowState { executor: Rc, scene_to_render: Option, renderer: Renderer, - command_queue: metal::CommandQueue, last_fresh_keydown: Option, - layer: id, traffic_light_position: Option, previous_modifiers_changed_event: Option, //State tracking what the IME did after the last request @@ -329,8 +326,6 @@ impl Window { executor: Rc, fonts: Arc, ) -> Self { - const PIXEL_FORMAT: metal::MTLPixelFormat = metal::MTLPixelFormat::BGRA8Unorm; - unsafe { let pool = NSAutoreleasePool::new(nil); @@ -368,25 +363,6 @@ impl Window { native_window.setFrame_display_(screen.visibleFrame(), YES); } - let device: metal::Device = if let Some(device) = metal::Device::system_default() { - device - } else { - log::error!("unable to access a compatible graphics device"); - std::process::exit(1); - }; - - let layer: id = msg_send![class!(CAMetalLayer), layer]; - let _: () = msg_send![layer, setDevice: device.as_ptr()]; - let _: () = msg_send![layer, setPixelFormat: PIXEL_FORMAT]; - let _: () = msg_send![layer, setAllowsNextDrawableTimeout: NO]; - let _: () = msg_send![layer, setNeedsDisplayOnBoundsChange: YES]; - let _: () = msg_send![layer, setPresentsWithTransaction: YES]; - let _: () = msg_send![ - layer, - setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE - | AutoresizingMask::HEIGHT_SIZABLE - ]; - let native_view: id = msg_send![VIEW_CLASS, alloc]; let native_view = NSView::init(native_view); assert!(!native_view.is_null()); @@ -406,15 +382,8 @@ impl Window { synthetic_drag_counter: 0, executor, scene_to_render: Default::default(), - renderer: Renderer::new( - device.clone(), - PIXEL_FORMAT, - get_scale_factor(native_window), - fonts, - ), - command_queue: device.new_command_queue(), + renderer: Renderer::new(fonts), last_fresh_keydown: None, - layer, traffic_light_position: options .titlebar .as_ref() @@ -1057,7 +1026,7 @@ extern "C" fn close_window(this: &Object, _: Sel) { extern "C" fn make_backing_layer(this: &Object, _: Sel) -> id { let window_state = unsafe { get_window_state(this) }; let window_state = window_state.as_ref().borrow(); - window_state.layer + window_state.renderer.layer().as_ptr() as id } extern "C" fn view_did_change_backing_properties(this: &Object, _: Sel) { @@ -1072,8 +1041,14 @@ extern "C" fn view_did_change_backing_properties(this: &Object, _: Sel) { height: size.y() as f64 * scale_factor, }; - let _: () = msg_send![window_state_borrow.layer, setContentsScale: scale_factor]; - let _: () = msg_send![window_state_borrow.layer, setDrawableSize: drawable_size]; + let _: () = msg_send![ + window_state_borrow.renderer.layer(), + setContentsScale: scale_factor + ]; + let _: () = msg_send![ + window_state_borrow.renderer.layer(), + setDrawableSize: drawable_size + ]; } if let Some(mut callback) = window_state_borrow.resize_callback.take() { @@ -1102,7 +1077,10 @@ extern "C" fn set_frame_size(this: &Object, _: Sel, size: NSSize) { }; unsafe { - let _: () = msg_send![window_state_borrow.layer, setDrawableSize: drawable_size]; + let _: () = msg_send![ + window_state_borrow.renderer.layer(), + setDrawableSize: drawable_size + ]; } drop(window_state_borrow); @@ -1118,25 +1096,8 @@ extern "C" fn display_layer(this: &Object, _: Sel, _: id) { unsafe { let window_state = get_window_state(this); let mut window_state = window_state.as_ref().borrow_mut(); - if let Some(scene) = window_state.scene_to_render.take() { - let drawable: &metal::MetalDrawableRef = msg_send![window_state.layer, nextDrawable]; - let command_queue = window_state.command_queue.clone(); - let command_buffer = command_queue.new_command_buffer(); - - let size = window_state.size(); - let scale_factor = window_state.scale_factor(); - - window_state.renderer.render( - &scene, - size * scale_factor, - command_buffer, - drawable.texture(), - ); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - drawable.present(); + window_state.renderer.render(&scene); }; } } From 6578af6f3b26ef01fd558e203c256a516c772c6f Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 12 Sep 2022 12:03:03 +0200 Subject: [PATCH 55/89] WIP: Start rendering GPUI views to macOS status bar --- crates/gpui/src/app.rs | 202 ++++++++++++-------- crates/gpui/src/platform.rs | 4 +- crates/gpui/src/platform/mac/platform.rs | 4 +- crates/gpui/src/platform/mac/status_item.rs | 153 ++++++++++----- crates/gpui/src/platform/test.rs | 4 +- 5 files changed, 235 insertions(+), 132 deletions(-) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 4a749b551f..e7f0e260e3 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1920,19 +1920,134 @@ impl MutableAppContext { }, ); root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); - this.open_platform_window(window_id, window_options); + + let mut window = + this.cx + .platform + .open_window(window_id, window_options, this.foreground.clone()); + let presenter = Rc::new(RefCell::new( + this.build_presenter(window_id, window.titlebar_height()), + )); + + { + let mut app = this.upgrade(); + let presenter = Rc::downgrade(&presenter); + window.on_event(Box::new(move |event| { + app.update(|cx| { + if let Some(presenter) = presenter.upgrade() { + if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { + if cx.dispatch_keystroke(window_id, keystroke) { + return true; + } + } + + presenter.borrow_mut().dispatch_event(event, false, cx) + } else { + false + } + }) + })); + } + + { + let mut app = this.upgrade(); + window.on_active_status_change(Box::new(move |is_active| { + app.update(|cx| cx.window_changed_active_status(window_id, is_active)) + })); + } + + { + let mut app = this.upgrade(); + window.on_resize(Box::new(move || { + app.update(|cx| cx.window_was_resized(window_id)) + })); + } + + { + let mut app = this.upgrade(); + window.on_fullscreen(Box::new(move |is_fullscreen| { + app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) + })); + } + + { + let mut app = this.upgrade(); + window.on_close(Box::new(move || { + app.update(|cx| cx.remove_window(window_id)); + })); + } + + window.set_input_handler(Box::new(WindowInputHandler { + app: this.upgrade().0, + window_id, + })); + + let scene = presenter.borrow_mut().build_scene( + window.size(), + window.scale_factor(), + false, + this, + ); + window.present_scene(scene); + this.presenters_and_platform_windows + .insert(window_id, (presenter.clone(), window)); (window_id, root_view) }) } - // pub fn add_status_bar_item(&mut self, build_item: F) - // where - // I: View, - // F: FnOnce(&mut ViewContext) -> I, - // { - // mem::forget(self.platform.add_status_item()); - // } + pub fn add_status_bar_item(&mut self, build_root_view: F) -> (usize, ViewHandle) + where + T: View, + F: FnOnce(&mut ViewContext) -> T, + { + self.update(|this| { + let window_id = post_inc(&mut this.next_window_id); + let root_view = this + .build_and_insert_view(window_id, ParentId::Root, |cx| Some(build_root_view(cx))) + .unwrap(); + this.cx.windows.insert( + window_id, + Window { + root_view: root_view.clone().into(), + focused_view_id: Some(root_view.id()), + is_active: false, + invalidation: None, + is_fullscreen: false, + }, + ); + root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); + + let mut status_item = this.cx.platform.add_status_item(); + let presenter = Rc::new(RefCell::new(this.build_presenter(window_id, 0.))); + + { + let mut app = this.upgrade(); + let presenter = Rc::downgrade(&presenter); + status_item.on_event(Box::new(move |event| { + app.update(|cx| { + if let Some(presenter) = presenter.upgrade() { + presenter.borrow_mut().dispatch_event(event, cx) + } else { + false + } + }) + })); + } + + let scene = presenter.borrow_mut().build_scene( + status_item.size(), + status_item.scale_factor(), + false, + this, + ); + status_item.present_scene(scene); + this.presenters_and_platform_windows + .insert(window_id, (presenter.clone(), status_item)); + + (window_id, root_view) + }) + } pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle where @@ -1956,77 +2071,6 @@ impl MutableAppContext { self.flush_effects(); } - fn open_platform_window(&mut self, window_id: usize, window_options: WindowOptions) { - let mut window = - self.cx - .platform - .open_window(window_id, window_options, self.foreground.clone()); - let presenter = Rc::new(RefCell::new( - self.build_presenter(window_id, window.titlebar_height()), - )); - - { - let mut app = self.upgrade(); - let presenter = Rc::downgrade(&presenter); - window.on_event(Box::new(move |event| { - app.update(|cx| { - if let Some(presenter) = presenter.upgrade() { - if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { - if cx.dispatch_keystroke(window_id, keystroke) { - return true; - } - } - - presenter.borrow_mut().dispatch_event(event, false, cx) - } else { - false - } - }) - })); - } - - { - let mut app = self.upgrade(); - window.on_active_status_change(Box::new(move |is_active| { - app.update(|cx| cx.window_changed_active_status(window_id, is_active)) - })); - } - - { - let mut app = self.upgrade(); - window.on_resize(Box::new(move || { - app.update(|cx| cx.window_was_resized(window_id)) - })); - } - - { - let mut app = self.upgrade(); - window.on_fullscreen(Box::new(move |is_fullscreen| { - app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) - })); - } - - { - let mut app = self.upgrade(); - window.on_close(Box::new(move || { - app.update(|cx| cx.remove_window(window_id)); - })); - } - - window.set_input_handler(Box::new(WindowInputHandler { - app: self.upgrade().0, - window_id, - })); - - let scene = - presenter - .borrow_mut() - .build_scene(window.size(), window.scale_factor(), false, self); - window.present_scene(scene); - self.presenters_and_platform_windows - .insert(window_id, (presenter.clone(), window)); - } - pub fn build_presenter(&mut self, window_id: usize, titlebar_height: f32) -> Presenter { Presenter::new( window_id, diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index b2e1995920..b739ace947 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -52,7 +52,7 @@ pub trait Platform: Send + Sync { ) -> Box; fn key_window_id(&self) -> Option; - fn add_status_item(&self) -> Box; + fn add_status_item(&self) -> Box; fn write_to_clipboard(&self, item: ClipboardItem); fn read_from_clipboard(&self) -> Option; @@ -134,8 +134,6 @@ pub trait Window { fn present_scene(&mut self, scene: Scene); } -pub trait StatusItem {} - #[derive(Debug)] pub struct WindowOptions<'a> { pub bounds: WindowBounds, diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index cda6521a8a..d1a3b7a797 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -495,8 +495,8 @@ impl platform::Platform for MacPlatform { Window::key_window_id() } - fn add_status_item(&self) -> Box { - Box::new(StatusItem::add()) + fn add_status_item(&self) -> Box { + Box::new(StatusItem::add(self.fonts())) } fn fonts(&self) -> Arc { diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index a672e1e323..6a4d754966 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,60 +1,121 @@ -use cocoa::{ - appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView}, - base::{id, nil, NO, YES}, - quartzcore::AutoresizingMask, +use crate::{ + geometry::vector::{vec2f, Vector2F}, + platform::{self, mac::renderer::Renderer}, + Event, FontSystem, Scene, }; -use core_foundation::base::TCFType; -use core_graphics::color::CGColor; -use foreign_types::ForeignType; -use objc::{class, msg_send, rc::StrongPtr, sel, sel_impl}; +use cocoa::{ + appkit::{ + NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSViewHeightSizable, + NSViewWidthSizable, NSWindow, + }, + base::{id, nil, YES}, + foundation::NSSize, +}; +use foreign_types::ForeignTypeRef; +use objc::{msg_send, rc::StrongPtr, sel, sel_impl}; +use std::{cell::RefCell, rc::Rc, sync::Arc}; -pub struct StatusItem(StrongPtr); +pub struct StatusItem(Rc>); + +struct StatusItemState { + native_item: StrongPtr, + renderer: Renderer, + event_callback: Option bool>>, +} impl StatusItem { - pub fn add() -> Self { - const PIXEL_FORMAT: metal::MTLPixelFormat = metal::MTLPixelFormat::BGRA8Unorm; - + pub fn add(fonts: Arc) -> Self { unsafe { + let renderer = Renderer::new(fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); - native_item.button().setWantsLayer(true); - let device: metal::Device = if let Some(device) = metal::Device::system_default() { - device - } else { - log::error!("unable to access a compatible graphics device"); - std::process::exit(1); - }; + let button = native_item.button(); + button.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); + button.setWantsBestResolutionOpenGLSurface_(YES); + button.setLayer(renderer.layer().as_ptr() as id); - let layer: id = msg_send![class!(CAMetalLayer), layer]; - let _: () = msg_send![layer, setDevice: device.as_ptr()]; - let _: () = msg_send![layer, setPixelFormat: PIXEL_FORMAT]; - let _: () = msg_send![layer, setAllowsNextDrawableTimeout: NO]; - let _: () = msg_send![layer, setNeedsDisplayOnBoundsChange: YES]; - let _: () = msg_send![layer, setPresentsWithTransaction: YES]; - let _: () = msg_send![ - layer, - setAutoresizingMask: AutoresizingMask::WIDTH_SIZABLE - | AutoresizingMask::HEIGHT_SIZABLE - ]; - let _: () = msg_send![ - layer, - setBackgroundColor: CGColor::rgb(1., 0., 0., 1.).as_concrete_TypeRef() - ]; - - let _: () = msg_send![native_item.button(), setLayer: layer]; - let native_item_window: id = msg_send![native_item.button(), window]; - - dbg!(native_item_window.frame().as_CGRect()); - // let rect_in_window: NSRect = msg_send![native_item.button(), convertRect: native_item.button().bounds() toView: nil]; - // let screen_rect: NSRect = - // msg_send![native_item_window, convertRectToScreen: rect_in_window]; - // dbg!(screen_rect.as_CGRect()); - - StatusItem(native_item) + Self(Rc::new(RefCell::new(StatusItemState { + native_item, + renderer, + event_callback: None, + }))) } } } -impl crate::StatusItem for StatusItem {} +impl platform::Window for StatusItem { + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + + fn on_event(&mut self, callback: Box bool>) { + self.0.borrow_mut().event_callback = Some(callback); + } + + fn on_active_status_change(&mut self, _: Box) {} + + fn on_resize(&mut self, _: Box) {} + + fn on_fullscreen(&mut self, _: Box) {} + + fn on_should_close(&mut self, _: Box bool>) {} + + fn on_close(&mut self, _: Box) {} + + fn set_input_handler(&mut self, _: Box) {} + + fn prompt( + &self, + _: crate::PromptLevel, + _: &str, + _: &[&str], + ) -> postage::oneshot::Receiver { + panic!() + } + + fn activate(&self) {} + + fn set_title(&mut self, _: &str) {} + + fn set_edited(&mut self, _: bool) {} + + fn show_character_palette(&self) {} + + fn minimize(&self) {} + + fn zoom(&self) {} + + fn toggle_full_screen(&self) {} + + fn size(&self) -> Vector2F { + self.0.borrow().size() + } + + fn scale_factor(&self) -> f32 { + self.0.borrow().scale_factor() + } + + fn titlebar_height(&self) -> f32 { + 0. + } + + fn present_scene(&mut self, scene: Scene) { + self.0.borrow_mut().renderer.render(&scene); + } +} + +impl StatusItemState { + fn size(&self) -> Vector2F { + let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_item.button()) }.size; + vec2f(width as f32, height as f32) + } + + fn scale_factor(&self) -> f32 { + unsafe { + let window: id = msg_send![self.native_item.button(), window]; + window.screen().backingScaleFactor() as f32 + } + } +} diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index f64307debf..0ce44f0817 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -144,8 +144,8 @@ impl super::Platform for Platform { None } - fn add_status_item(&self) -> Box { - todo!() + fn add_status_item(&self) -> Box { + Box::new(Window::new(vec2f(24., 24.))) } fn write_to_clipboard(&self, item: ClipboardItem) { From 2acd215bb887945554bf5a9feaad9dceacab9a77 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 10:07:33 +0200 Subject: [PATCH 56/89] Wire up event handling for status items --- crates/gpui/src/platform/mac/status_item.rs | 161 +++++++++++++++++--- 1 file changed, 137 insertions(+), 24 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 6a4d754966..81a8ad036f 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -5,15 +5,53 @@ use crate::{ }; use cocoa::{ appkit::{ - NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSViewHeightSizable, - NSViewWidthSizable, NSWindow, + NSApplication, NSButton, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, + NSViewHeightSizable, NSViewWidthSizable, NSWindow, }, base::{id, nil, YES}, - foundation::NSSize, + foundation::{NSSize, NSUInteger}, }; +use ctor::ctor; use foreign_types::ForeignTypeRef; -use objc::{msg_send, rc::StrongPtr, sel, sel_impl}; -use std::{cell::RefCell, rc::Rc, sync::Arc}; +use objc::{ + class, + declare::ClassDecl, + msg_send, + rc::StrongPtr, + runtime::{Class, Object, Sel}, + sel, sel_impl, +}; +use std::{ + cell::RefCell, + ffi::c_void, + ptr, + rc::{Rc, Weak}, + sync::Arc, +}; + +static mut HANDLER_CLASS: *const Class = ptr::null(); +const STATE_IVAR: &str = "state"; + +#[allow(non_upper_case_globals)] +const NSEventMaskAny: NSUInteger = NSUInteger::MAX; + +#[ctor] +unsafe fn build_classes() { + HANDLER_CLASS = { + let mut decl = ClassDecl::new("GPUIStatusItemEventHandler", class!(NSObject)).unwrap(); + decl.add_ivar::<*mut c_void>(STATE_IVAR); + decl.add_method( + sel!(dealloc), + dealloc_handler as extern "C" fn(&Object, Sel), + ); + decl.add_method( + sel!(handleEvent), + handle_event as extern "C" fn(&Object, Sel), + ); + + decl.register() + }; +} pub struct StatusItem(Rc>); @@ -21,6 +59,7 @@ struct StatusItemState { native_item: StrongPtr, renderer: Renderer, event_callback: Option bool>>, + _event_handler: StrongPtr, } impl StatusItem { @@ -36,11 +75,22 @@ impl StatusItem { button.setWantsBestResolutionOpenGLSurface_(YES); button.setLayer(renderer.layer().as_ptr() as id); - Self(Rc::new(RefCell::new(StatusItemState { - native_item, - renderer, - event_callback: None, - }))) + Self(Rc::new_cyclic(|state| { + let event_handler = StrongPtr::new(msg_send![HANDLER_CLASS, alloc]); + let _: () = msg_send![*event_handler, init]; + (**event_handler) + .set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); + button.setTarget_(*event_handler); + button.setAction_(sel!(handleEvent)); + let _: () = msg_send![button, sendActionOn: NSEventMaskAny]; + + RefCell::new(StatusItemState { + native_item, + renderer, + event_callback: None, + _event_handler: event_handler, + }) + })) } } } @@ -54,17 +104,29 @@ impl platform::Window for StatusItem { self.0.borrow_mut().event_callback = Some(callback); } - fn on_active_status_change(&mut self, _: Box) {} + fn on_active_status_change(&mut self, _: Box) { + unimplemented!() + } - fn on_resize(&mut self, _: Box) {} + fn on_resize(&mut self, _: Box) { + unimplemented!() + } - fn on_fullscreen(&mut self, _: Box) {} + fn on_fullscreen(&mut self, _: Box) { + unimplemented!() + } - fn on_should_close(&mut self, _: Box bool>) {} + fn on_should_close(&mut self, _: Box bool>) { + unimplemented!() + } - fn on_close(&mut self, _: Box) {} + fn on_close(&mut self, _: Box) { + unimplemented!() + } - fn set_input_handler(&mut self, _: Box) {} + fn set_input_handler(&mut self, _: Box) { + unimplemented!() + } fn prompt( &self, @@ -72,22 +134,36 @@ impl platform::Window for StatusItem { _: &str, _: &[&str], ) -> postage::oneshot::Receiver { - panic!() + unimplemented!() } - fn activate(&self) {} + fn activate(&self) { + unimplemented!() + } - fn set_title(&mut self, _: &str) {} + fn set_title(&mut self, _: &str) { + unimplemented!() + } - fn set_edited(&mut self, _: bool) {} + fn set_edited(&mut self, _: bool) { + unimplemented!() + } - fn show_character_palette(&self) {} + fn show_character_palette(&self) { + unimplemented!() + } - fn minimize(&self) {} + fn minimize(&self) { + unimplemented!() + } - fn zoom(&self) {} + fn zoom(&self) { + unimplemented!() + } - fn toggle_full_screen(&self) {} + fn toggle_full_screen(&self) { + unimplemented!() + } fn size(&self) -> Vector2F { self.0.borrow().size() @@ -119,3 +195,40 @@ impl StatusItemState { } } } + +extern "C" fn dealloc_handler(this: &Object, _: Sel) { + unsafe { + drop_state(this); + let _: () = msg_send![super(this, class!(NSObject)), dealloc]; + } +} + +extern "C" fn handle_event(this: &Object, _: Sel) { + unsafe { + if let Some(state) = get_state(this).upgrade() { + let mut state_borrow = state.as_ref().borrow_mut(); + let app = NSApplication::sharedApplication(nil); + let native_event: id = msg_send![app, currentEvent]; + if let Some(event) = Event::from_native(native_event, Some(state_borrow.size().y())) { + if let Some(mut callback) = state_borrow.event_callback.take() { + drop(state_borrow); + callback(event); + state.borrow_mut().event_callback = Some(callback); + } + } + } + } +} + +unsafe fn get_state(object: &Object) -> Weak> { + let raw: *mut c_void = *object.get_ivar(STATE_IVAR); + let weak1 = Weak::from_raw(raw as *mut RefCell); + let weak2 = weak1.clone(); + let _ = Weak::into_raw(weak1); + weak2 +} + +unsafe fn drop_state(object: &Object) { + let raw: *const c_void = *object.get_ivar(STATE_IVAR); + Weak::from_raw(raw as *const RefCell); +} From a9c2881831b7c924689539af46cefe6fd7fcf9eb Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 12:08:56 +0200 Subject: [PATCH 57/89] Set contents scale and drawable size when creating status metal layer --- crates/gpui/src/platform/mac/renderer.rs | 8 ++------ crates/gpui/src/platform/mac/status_item.rs | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index f9cc6e730d..dc2d8a9d50 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -187,15 +187,11 @@ impl Renderer { pub fn render(&mut self, scene: &Scene) { let layer = self.layer.clone(); + let drawable_size = layer.drawable_size(); let drawable = layer.next_drawable().unwrap(); let command_queue = self.command_queue.clone(); let command_buffer = command_queue.new_command_buffer(); - let frame: NSRect = unsafe { msg_send![self.layer(), frame] }; - let scale_factor: CGFloat = unsafe { msg_send![self.layer(), contentsScale] }; - let drawable_size = - vec2f(frame.size.width as f32, frame.size.height as f32) * scale_factor as f32; - self.sprite_cache.set_scale_factor(scene.scale_factor()); self.image_cache.set_scale_factor(scene.scale_factor()); @@ -206,7 +202,7 @@ impl Renderer { scene, path_sprites, &mut offset, - drawable_size, + vec2f(drawable_size.width as f32, drawable_size.height as f32), command_buffer, drawable.texture(), ); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 81a8ad036f..bd8c83154b 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,7 +1,7 @@ use crate::{ geometry::vector::{vec2f, Vector2F}, platform::{self, mac::renderer::Renderer}, - Event, FontSystem, Scene, + Event, FontSystem, Scene, Window, }; use cocoa::{ appkit::{ @@ -75,7 +75,7 @@ impl StatusItem { button.setWantsBestResolutionOpenGLSurface_(YES); button.setLayer(renderer.layer().as_ptr() as id); - Self(Rc::new_cyclic(|state| { + let item = Self(Rc::new_cyclic(|state| { let event_handler = StrongPtr::new(msg_send![HANDLER_CLASS, alloc]); let _: () = msg_send![*event_handler, init]; (**event_handler) @@ -90,7 +90,18 @@ impl StatusItem { event_callback: None, _event_handler: event_handler, }) - })) + })); + + { + let item = item.0.borrow(); + let layer = item.renderer.layer(); + let scale_factor = item.scale_factor(); + let size = item.size() * scale_factor; + layer.set_contents_scale(scale_factor.into()); + layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); + } + + item } } } From 11d47f5c728628820ea95b4795eab20b5b94cbdc Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:05:50 +0200 Subject: [PATCH 58/89] Use a transparent layer for status bar This allows the compositor to blend the GPUI view with the background. --- crates/gpui/src/platform/mac/renderer.rs | 10 ++++++---- crates/gpui/src/platform/mac/status_item.rs | 2 +- crates/gpui/src/platform/mac/window.rs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index dc2d8a9d50..ea094e998c 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -10,14 +10,14 @@ use crate::{ }; use cocoa::{ base::{NO, YES}, - foundation::{NSRect, NSUInteger}, + foundation::NSUInteger, quartzcore::AutoresizingMask, }; use core_foundation::base::TCFType; use foreign_types::ForeignTypeRef; use log::warn; use media::core_video::{self, CVMetalTextureCache}; -use metal::{CGFloat, CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; +use metal::{CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; use objc::{self, msg_send, sel, sel_impl}; use shaders::ToFloat2 as _; use std::{collections::HashMap, ffi::c_void, iter::Peekable, mem, ptr, sync::Arc, vec}; @@ -55,7 +55,7 @@ pub struct Surface { } impl Renderer { - pub fn new(fonts: Arc) -> Self { + pub fn new(is_opaque: bool, fonts: Arc) -> Self { const PIXEL_FORMAT: MTLPixelFormat = MTLPixelFormat::BGRA8Unorm; let device: metal::Device = if let Some(device) = metal::Device::system_default() { @@ -69,6 +69,7 @@ impl Renderer { layer.set_device(&device); layer.set_pixel_format(PIXEL_FORMAT); layer.set_presents_with_transaction(true); + layer.set_opaque(is_opaque); unsafe { let _: () = msg_send![&*layer, setAllowsNextDrawableTimeout: NO]; let _: () = msg_send![&*layer, setNeedsDisplayOnBoundsChange: YES]; @@ -363,7 +364,8 @@ impl Renderer { color_attachment.set_texture(Some(output)); color_attachment.set_load_action(metal::MTLLoadAction::Clear); color_attachment.set_store_action(metal::MTLStoreAction::Store); - color_attachment.set_clear_color(metal::MTLClearColor::new(0., 0., 0., 1.)); + let alpha = if self.layer.is_opaque() { 1. } else { 0. }; + color_attachment.set_clear_color(metal::MTLClearColor::new(0., 0., 0., alpha)); let command_encoder = command_buffer.new_render_command_encoder(render_pass_descriptor); command_encoder.set_viewport(metal::MTLViewport { diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index bd8c83154b..9d98c4a1f7 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -65,7 +65,7 @@ struct StatusItemState { impl StatusItem { pub fn add(fonts: Arc) -> Self { unsafe { - let renderer = Renderer::new(fonts); + let renderer = Renderer::new(false, fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 822e1700f1..4abee8b961 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -382,7 +382,7 @@ impl Window { synthetic_drag_counter: 0, executor, scene_to_render: Default::default(), - renderer: Renderer::new(fonts), + renderer: Renderer::new(true, fonts), last_fresh_keydown: None, traffic_light_position: options .titlebar From a34eaa36061d030353502e8048ca401067238d0c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:49:49 +0200 Subject: [PATCH 59/89] Pass `reused` parameter to `Presenter::dispatch_event` for status items --- crates/gpui/src/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index e7f0e260e3..a4d067fb4f 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -2027,7 +2027,7 @@ impl MutableAppContext { status_item.on_event(Box::new(move |event| { app.update(|cx| { if let Some(presenter) = presenter.upgrade() { - presenter.borrow_mut().dispatch_event(event, cx) + presenter.borrow_mut().dispatch_event(event, false, cx) } else { false } From b3dd09a0f2df44ded01ff30f83e1ac2f0c365abc Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:50:18 +0200 Subject: [PATCH 60/89] :art: --- crates/gpui/src/platform/mac/status_item.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 9d98c4a1f7..9c08917927 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -5,8 +5,8 @@ use crate::{ }; use cocoa::{ appkit::{ - NSApplication, NSButton, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, - NSViewHeightSizable, NSViewWidthSizable, NSWindow, + NSApplication, NSButton, NSEventMask, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, + NSView, NSViewHeightSizable, NSViewWidthSizable, NSWindow, }, base::{id, nil, YES}, foundation::{NSSize, NSUInteger}, @@ -32,9 +32,6 @@ use std::{ static mut HANDLER_CLASS: *const Class = ptr::null(); const STATE_IVAR: &str = "state"; -#[allow(non_upper_case_globals)] -const NSEventMaskAny: NSUInteger = NSUInteger::MAX; - #[ctor] unsafe fn build_classes() { HANDLER_CLASS = { @@ -82,7 +79,7 @@ impl StatusItem { .set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); button.setTarget_(*event_handler); button.setAction_(sel!(handleEvent)); - let _: () = msg_send![button, sendActionOn: NSEventMaskAny]; + let _: () = msg_send![button, sendActionOn: NSEventMask::NSAnyEventMask]; RefCell::new(StatusItemState { native_item, From a102b3ba4b1acb3d38f48611401e1fd77fc07670 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 14:51:00 +0200 Subject: [PATCH 61/89] Start on a real status bar item implementation --- Cargo.lock | 25 +++++++++++++ assets/icons/zed_32.svg | 22 ++++++++++++ crates/contacts_status_item/Cargo.toml | 32 +++++++++++++++++ .../src/contacts_status_item.rs | 36 +++++++++++++++++++ crates/zed/Cargo.toml | 1 + crates/zed/src/main.rs | 4 ++- 6 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 assets/icons/zed_32.svg create mode 100644 crates/contacts_status_item/Cargo.toml create mode 100644 crates/contacts_status_item/src/contacts_status_item.rs diff --git a/Cargo.lock b/Cargo.lock index 7486a62082..05701b7c56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1126,6 +1126,30 @@ dependencies = [ "workspace", ] +[[package]] +name = "contacts_status_item" +version = "0.1.0" +dependencies = [ + "anyhow", + "client", + "collections", + "editor", + "futures", + "fuzzy", + "gpui", + "language", + "log", + "menu", + "picker", + "postage", + "project", + "serde", + "settings", + "theme", + "util", + "workspace", +] + [[package]] name = "context_menu" version = "0.1.0" @@ -7122,6 +7146,7 @@ dependencies = [ "collections", "command_palette", "contacts_panel", + "contacts_status_item", "context_menu", "ctor", "diagnostics", diff --git a/assets/icons/zed_32.svg b/assets/icons/zed_32.svg new file mode 100644 index 0000000000..f8c0c537c3 --- /dev/null +++ b/assets/icons/zed_32.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/contacts_status_item/Cargo.toml b/crates/contacts_status_item/Cargo.toml new file mode 100644 index 0000000000..df115a3842 --- /dev/null +++ b/crates/contacts_status_item/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "contacts_status_item" +version = "0.1.0" +edition = "2021" + +[lib] +path = "src/contacts_status_item.rs" +doctest = false + +[dependencies] +client = { path = "../client" } +collections = { path = "../collections" } +editor = { path = "../editor" } +fuzzy = { path = "../fuzzy" } +gpui = { path = "../gpui" } +menu = { path = "../menu" } +picker = { path = "../picker" } +project = { path = "../project" } +settings = { path = "../settings" } +theme = { path = "../theme" } +util = { path = "../util" } +workspace = { path = "../workspace" } +anyhow = "1.0" +futures = "0.3" +log = "0.4" +postage = { version = "0.4.1", features = ["futures-traits"] } +serde = { version = "1.0", features = ["derive", "rc"] } + +[dev-dependencies] +language = { path = "../language", features = ["test-support"] } +project = { path = "../project", features = ["test-support"] } +workspace = { path = "../workspace", features = ["test-support"] } diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs new file mode 100644 index 0000000000..928eecb8fe --- /dev/null +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -0,0 +1,36 @@ +use gpui::{color::Color, elements::*, Entity, RenderContext, View}; + +pub struct ContactsStatusItem; + +impl Entity for ContactsStatusItem { + type Event = (); +} + +impl View for ContactsStatusItem { + fn ui_name() -> &'static str { + "ContactsStatusItem" + } + + fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + MouseEventHandler::new::(0, cx, |state, cx| { + Svg::new("icons/zed_32.svg") + .with_color(if state.clicked.is_some() { + Color::red() + } else { + Color::blue() + }) + .boxed() + }) + .on_down(gpui::MouseButton::Left, |_, cx| {}) + .on_up(gpui::MouseButton::Left, |_, cx| {}) + .contained() + .with_background_color(Color::green()) + .boxed() + } +} + +impl ContactsStatusItem { + pub fn new() -> Self { + Self + } +} diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.toml index 6d2a617d4c..446139eaaf 100644 --- a/crates/zed/Cargo.toml +++ b/crates/zed/Cargo.toml @@ -27,6 +27,7 @@ context_menu = { path = "../context_menu" } client = { path = "../client" } clock = { path = "../clock" } contacts_panel = { path = "../contacts_panel" } +contacts_status_item = { path = "../contacts_status_item" } diagnostics = { path = "../diagnostics" } editor = { path = "../editor" } file_finder = { path = "../file_finder" } diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index 43a403ab7f..287ac721fd 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -14,6 +14,7 @@ use client::{ http::{self, HttpClient}, UserStore, ZED_SECRET_CLIENT_TOKEN, }; +use contacts_status_item::ContactsStatusItem; use fs::OpenOptions; use futures::{ channel::{mpsc, oneshot}, @@ -87,7 +88,8 @@ fn main() { }); app.run(move |cx| { - std::mem::forget(cx.platform().add_status_item()); + cx.add_status_bar_item(|_| ContactsStatusItem::new()); + let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); From 97ccb16c97bd9da0fdb2e04b44e9a914adc967b2 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 16:20:34 +0200 Subject: [PATCH 62/89] Rework status bar item to use a custom view --- .../src/contacts_status_item.rs | 7 +- crates/gpui/src/platform/mac/status_item.rs | 102 ++++++++++++------ 2 files changed, 75 insertions(+), 34 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 928eecb8fe..5905c9ad12 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -21,8 +21,11 @@ impl View for ContactsStatusItem { }) .boxed() }) - .on_down(gpui::MouseButton::Left, |_, cx| {}) - .on_up(gpui::MouseButton::Left, |_, cx| {}) + .on_down(gpui::MouseButton::Left, |_, _| {}) + .on_down_out(gpui::MouseButton::Left, |_, _| {}) + .on_up(gpui::MouseButton::Left, |_, _| {}) + .on_up_out(gpui::MouseButton::Left, |_, _| {}) + .aligned() .contained() .with_background_color(Color::green()) .boxed() diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 9c08917927..c369ab4245 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,15 +1,12 @@ use crate::{ geometry::vector::{vec2f, Vector2F}, platform::{self, mac::renderer::Renderer}, - Event, FontSystem, Scene, Window, + Event, FontSystem, Scene, }; use cocoa::{ - appkit::{ - NSApplication, NSButton, NSEventMask, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, - NSView, NSViewHeightSizable, NSViewWidthSizable, NSWindow, - }, + appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, - foundation::{NSSize, NSUInteger}, + foundation::{NSAutoreleasePool, NSPoint, NSRect, NSSize}, }; use ctor::ctor; use foreign_types::ForeignTypeRef; @@ -29,21 +26,56 @@ use std::{ sync::Arc, }; -static mut HANDLER_CLASS: *const Class = ptr::null(); +static mut VIEW_CLASS: *const Class = ptr::null(); const STATE_IVAR: &str = "state"; #[ctor] unsafe fn build_classes() { - HANDLER_CLASS = { - let mut decl = ClassDecl::new("GPUIStatusItemEventHandler", class!(NSObject)).unwrap(); + VIEW_CLASS = { + let mut decl = ClassDecl::new("GPUIStatusItemView", class!(NSView)).unwrap(); decl.add_ivar::<*mut c_void>(STATE_IVAR); + + decl.add_method(sel!(dealloc), dealloc_view as extern "C" fn(&Object, Sel)); + decl.add_method( - sel!(dealloc), - dealloc_handler as extern "C" fn(&Object, Sel), + sel!(mouseDown:), + handle_view_event as extern "C" fn(&Object, Sel, id), ); decl.add_method( - sel!(handleEvent), - handle_event as extern "C" fn(&Object, Sel), + sel!(mouseUp:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(rightMouseDown:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(rightMouseUp:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(otherMouseDown:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(otherMouseUp:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(mouseMoved:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(mouseDragged:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(scrollWheel:), + handle_view_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(flagsChanged:), + handle_view_event as extern "C" fn(&Object, Sel, id), ); decl.register() @@ -56,36 +88,39 @@ struct StatusItemState { native_item: StrongPtr, renderer: Renderer, event_callback: Option bool>>, - _event_handler: StrongPtr, } impl StatusItem { pub fn add(fonts: Arc) -> Self { unsafe { + let pool = NSAutoreleasePool::new(nil); + let renderer = Renderer::new(false, fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = StrongPtr::retain(status_bar.statusItemWithLength_(NSSquareStatusItemLength)); let button = native_item.button(); - button.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); - button.setWantsBestResolutionOpenGLSurface_(YES); - button.setLayer(renderer.layer().as_ptr() as id); + let _: () = msg_send![button, setHidden: YES]; let item = Self(Rc::new_cyclic(|state| { - let event_handler = StrongPtr::new(msg_send![HANDLER_CLASS, alloc]); - let _: () = msg_send![*event_handler, init]; - (**event_handler) - .set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); - button.setTarget_(*event_handler); - button.setAction_(sel!(handleEvent)); - let _: () = msg_send![button, sendActionOn: NSEventMask::NSAnyEventMask]; + let parent_view = button.superview().superview(); + + let view: id = msg_send![VIEW_CLASS, alloc]; + NSView::initWithFrame_( + view, + NSRect::new(NSPoint::new(0., 0.), NSView::frame(parent_view).size), + ); + view.setWantsBestResolutionOpenGLSurface_(YES); + view.setLayer(renderer.layer().as_ptr() as id); + view.setWantsLayer(true); + (*view).set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); + parent_view.addSubview_(view.autorelease()); RefCell::new(StatusItemState { native_item, renderer, event_callback: None, - _event_handler: event_handler, }) })); @@ -98,6 +133,8 @@ impl StatusItem { layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); } + pool.drain(); + item } } @@ -192,8 +229,11 @@ impl platform::Window for StatusItem { impl StatusItemState { fn size(&self) -> Vector2F { - let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_item.button()) }.size; - vec2f(width as f32, height as f32) + unsafe { + let NSSize { width, height, .. } = + NSWindow::frame(self.native_item.button().superview().superview()).size; + vec2f(width as f32, height as f32) + } } fn scale_factor(&self) -> f32 { @@ -204,19 +244,17 @@ impl StatusItemState { } } -extern "C" fn dealloc_handler(this: &Object, _: Sel) { +extern "C" fn dealloc_view(this: &Object, _: Sel) { unsafe { drop_state(this); - let _: () = msg_send![super(this, class!(NSObject)), dealloc]; + let _: () = msg_send![super(this, class!(NSView)), dealloc]; } } -extern "C" fn handle_event(this: &Object, _: Sel) { +extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { unsafe { if let Some(state) = get_state(this).upgrade() { let mut state_borrow = state.as_ref().borrow_mut(); - let app = NSApplication::sharedApplication(nil); - let native_event: id = msg_send![app, currentEvent]; if let Some(event) = Event::from_native(native_event, Some(state_borrow.size().y())) { if let Some(mut callback) = state_borrow.event_callback.take() { drop(state_borrow); From 0f9ff575689276de5b7b4cb07c0ce776b5d89a6c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 13 Sep 2022 17:00:11 +0200 Subject: [PATCH 63/89] Show the correct icon in status bar --- assets/icons/zed_22.svg | 4 ++++ assets/icons/zed_32.svg | 22 ------------------- .../src/contacts_status_item.rs | 22 +++---------------- 3 files changed, 7 insertions(+), 41 deletions(-) create mode 100644 assets/icons/zed_22.svg delete mode 100644 assets/icons/zed_32.svg diff --git a/assets/icons/zed_22.svg b/assets/icons/zed_22.svg new file mode 100644 index 0000000000..68e7dc8e57 --- /dev/null +++ b/assets/icons/zed_22.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/assets/icons/zed_32.svg b/assets/icons/zed_32.svg deleted file mode 100644 index f8c0c537c3..0000000000 --- a/assets/icons/zed_32.svg +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 5905c9ad12..c7cd9c3a4a 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -1,4 +1,4 @@ -use gpui::{color::Color, elements::*, Entity, RenderContext, View}; +use gpui::{elements::*, Entity, RenderContext, View}; pub struct ContactsStatusItem; @@ -11,24 +11,8 @@ impl View for ContactsStatusItem { "ContactsStatusItem" } - fn render(&mut self, cx: &mut RenderContext) -> ElementBox { - MouseEventHandler::new::(0, cx, |state, cx| { - Svg::new("icons/zed_32.svg") - .with_color(if state.clicked.is_some() { - Color::red() - } else { - Color::blue() - }) - .boxed() - }) - .on_down(gpui::MouseButton::Left, |_, _| {}) - .on_down_out(gpui::MouseButton::Left, |_, _| {}) - .on_up(gpui::MouseButton::Left, |_, _| {}) - .on_up_out(gpui::MouseButton::Left, |_, _| {}) - .aligned() - .contained() - .with_background_color(Color::green()) - .boxed() + fn render(&mut self, _: &mut RenderContext) -> ElementBox { + Svg::new("icons/zed_22.svg").aligned().boxed() } } From f67e2bea29f286d41b36ffccfde0fa9cbfb04bfa Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 11:47:43 +0200 Subject: [PATCH 64/89] Refresh windows when OS appearance changes --- .../src/contacts_status_item.rs | 13 +- crates/diagnostics/src/diagnostics.rs | 2 +- crates/editor/src/element.rs | 4 +- crates/gpui/src/app.rs | 47 +++++- crates/gpui/src/elements/list.rs | 4 +- crates/gpui/src/elements/text.rs | 2 +- crates/gpui/src/platform.rs | 28 +++- crates/gpui/src/platform/mac.rs | 1 + crates/gpui/src/platform/mac/appearance.rs | 37 +++++ crates/gpui/src/platform/mac/platform.rs | 5 + crates/gpui/src/platform/mac/status_item.rs | 146 ++++++++++++++---- crates/gpui/src/platform/mac/window.rs | 47 +++++- crates/gpui/src/platform/test.rs | 6 + crates/gpui/src/presenter.rs | 29 +++- 14 files changed, 304 insertions(+), 67 deletions(-) create mode 100644 crates/gpui/src/platform/mac/appearance.rs diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index c7cd9c3a4a..f3363f18d3 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -1,4 +1,4 @@ -use gpui::{elements::*, Entity, RenderContext, View}; +use gpui::{color::Color, elements::*, Appearance, Entity, RenderContext, View}; pub struct ContactsStatusItem; @@ -11,8 +11,15 @@ impl View for ContactsStatusItem { "ContactsStatusItem" } - fn render(&mut self, _: &mut RenderContext) -> ElementBox { - Svg::new("icons/zed_22.svg").aligned().boxed() + fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + let color = match cx.appearance { + Appearance::Light | Appearance::VibrantLight => Color::black(), + Appearance::Dark | Appearance::VibrantDark => Color::white(), + }; + Svg::new("icons/zed_22.svg") + .with_color(color) + .aligned() + .boxed() } } diff --git a/crates/diagnostics/src/diagnostics.rs b/crates/diagnostics/src/diagnostics.rs index 271843dc69..25db51e5e8 100644 --- a/crates/diagnostics/src/diagnostics.rs +++ b/crates/diagnostics/src/diagnostics.rs @@ -1149,7 +1149,7 @@ mod tests { editor: &ViewHandle, cx: &mut MutableAppContext, ) -> Vec<(u32, String)> { - let mut presenter = cx.build_presenter(editor.id(), 0.); + let mut presenter = cx.build_presenter(editor.id(), 0., Default::default()); let mut cx = presenter.build_layout_context(Default::default(), false, cx); cx.render(editor, |editor, cx| { let snapshot = editor.snapshot(cx); diff --git a/crates/editor/src/element.rs b/crates/editor/src/element.rs index d9d0c3990c..5146be120a 100644 --- a/crates/editor/src/element.rs +++ b/crates/editor/src/element.rs @@ -2044,7 +2044,7 @@ mod tests { let layouts = editor.update(cx, |editor, cx| { let snapshot = editor.snapshot(cx); - let mut presenter = cx.build_presenter(window_id, 30.); + let mut presenter = cx.build_presenter(window_id, 30., Default::default()); let layout_cx = presenter.build_layout_context(Vector2F::zero(), false, cx); element.layout_line_numbers(0..6, &Default::default(), &snapshot, &layout_cx) }); @@ -2083,7 +2083,7 @@ mod tests { ); let mut scene = Scene::new(1.0); - let mut presenter = cx.build_presenter(window_id, 30.); + let mut presenter = cx.build_presenter(window_id, 30., Default::default()); let mut layout_cx = presenter.build_layout_context(Vector2F::zero(), false, cx); let (size, mut state) = element.layout( SizeConstraint::new(vec2f(500., 500.), vec2f(500., 500.)), diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index a4d067fb4f..0baff67e94 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -9,8 +9,8 @@ use crate::{ platform::{self, KeyDownEvent, Platform, PromptLevel, WindowOptions}, presenter::Presenter, util::post_inc, - AssetCache, AssetSource, ClipboardItem, FontCache, InputHandler, MouseButton, MouseRegionId, - PathPromptOptions, TextLayoutCache, + Appearance, AssetCache, AssetSource, ClipboardItem, FontCache, InputHandler, MouseButton, + MouseRegionId, PathPromptOptions, TextLayoutCache, }; pub use action::*; use anyhow::{anyhow, Context, Result}; @@ -579,6 +579,7 @@ impl TestAppContext { hovered_region_ids: Default::default(), clicked_region_ids: None, refreshing: false, + appearance: Appearance::Light, }; f(view, &mut render_cx) }) @@ -1260,6 +1261,7 @@ impl MutableAppContext { &mut self, window_id: usize, titlebar_height: f32, + appearance: Appearance, ) -> HashMap { self.start_frame(); #[allow(clippy::needless_collect)] @@ -1287,6 +1289,7 @@ impl MutableAppContext { hovered_region_ids: Default::default(), clicked_region_ids: None, refreshing: false, + appearance, }) .unwrap(), ) @@ -1925,9 +1928,11 @@ impl MutableAppContext { this.cx .platform .open_window(window_id, window_options, this.foreground.clone()); - let presenter = Rc::new(RefCell::new( - this.build_presenter(window_id, window.titlebar_height()), - )); + let presenter = Rc::new(RefCell::new(this.build_presenter( + window_id, + window.titlebar_height(), + window.appearance(), + ))); { let mut app = this.upgrade(); @@ -1977,6 +1982,12 @@ impl MutableAppContext { })); } + { + let mut app = this.upgrade(); + window + .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); + } + window.set_input_handler(Box::new(WindowInputHandler { app: this.upgrade().0, window_id, @@ -2019,7 +2030,11 @@ impl MutableAppContext { root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); let mut status_item = this.cx.platform.add_status_item(); - let presenter = Rc::new(RefCell::new(this.build_presenter(window_id, 0.))); + let presenter = Rc::new(RefCell::new(this.build_presenter( + window_id, + 0., + status_item.appearance(), + ))); { let mut app = this.upgrade(); @@ -2035,6 +2050,12 @@ impl MutableAppContext { })); } + { + let mut app = this.upgrade(); + status_item + .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); + } + let scene = presenter.borrow_mut().build_scene( status_item.size(), status_item.scale_factor(), @@ -2071,10 +2092,16 @@ impl MutableAppContext { self.flush_effects(); } - pub fn build_presenter(&mut self, window_id: usize, titlebar_height: f32) -> Presenter { + pub fn build_presenter( + &mut self, + window_id: usize, + titlebar_height: f32, + appearance: Appearance, + ) -> Presenter { Presenter::new( window_id, titlebar_height, + appearance, self.cx.font_cache.clone(), TextLayoutCache::new(self.cx.platform.fonts()), self.assets.clone(), @@ -2412,7 +2439,7 @@ impl MutableAppContext { { { let mut presenter = presenter.borrow_mut(); - presenter.invalidate(&mut invalidation, self); + presenter.invalidate(&mut invalidation, window.appearance(), self); let scene = presenter.build_scene(window.size(), window.scale_factor(), false, self); window.present_scene(scene); @@ -2476,6 +2503,7 @@ impl MutableAppContext { let mut presenter = presenter.borrow_mut(); presenter.refresh( invalidation.as_mut().unwrap_or(&mut Default::default()), + window.appearance(), self, ); let scene = presenter.build_scene(window.size(), window.scale_factor(), true, self); @@ -4082,6 +4110,7 @@ pub struct RenderParams { pub hovered_region_ids: HashSet, pub clicked_region_ids: Option<(Vec, MouseButton)>, pub refreshing: bool, + pub appearance: Appearance, } pub struct RenderContext<'a, T: View> { @@ -4092,6 +4121,7 @@ pub struct RenderContext<'a, T: View> { pub(crate) clicked_region_ids: Option<(Vec, MouseButton)>, pub app: &'a mut MutableAppContext, pub titlebar_height: f32, + pub appearance: Appearance, pub refreshing: bool, } @@ -4112,6 +4142,7 @@ impl<'a, V: View> RenderContext<'a, V> { hovered_region_ids: params.hovered_region_ids.clone(), clicked_region_ids: params.clicked_region_ids.clone(), refreshing: params.refreshing, + appearance: params.appearance, } } diff --git a/crates/gpui/src/elements/list.rs b/crates/gpui/src/elements/list.rs index df480005b5..57436c256b 100644 --- a/crates/gpui/src/elements/list.rs +++ b/crates/gpui/src/elements/list.rs @@ -659,7 +659,7 @@ mod tests { #[crate::test(self)] fn test_layout(cx: &mut crate::MutableAppContext) { - let mut presenter = cx.build_presenter(0, 0.); + let mut presenter = cx.build_presenter(0, 0., Default::default()); let (_, view) = cx.add_window(Default::default(), |_| TestView); let constraint = SizeConstraint::new(vec2f(0., 0.), vec2f(100., 40.)); @@ -759,7 +759,7 @@ mod tests { .unwrap_or(10); let (_, view) = cx.add_window(Default::default(), |_| TestView); - let mut presenter = cx.build_presenter(0, 0.); + let mut presenter = cx.build_presenter(0, 0., Default::default()); let mut next_id = 0; let elements = Rc::new(RefCell::new( (0..rng.gen_range(0..=20)) diff --git a/crates/gpui/src/elements/text.rs b/crates/gpui/src/elements/text.rs index 92e38894c1..45dc2dbae3 100644 --- a/crates/gpui/src/elements/text.rs +++ b/crates/gpui/src/elements/text.rs @@ -291,7 +291,7 @@ mod tests { #[crate::test(self)] fn test_soft_wrapping_with_carriage_returns(cx: &mut MutableAppContext) { let (window_id, _) = cx.add_window(Default::default(), |_| TestView); - let mut presenter = cx.build_presenter(window_id, Default::default()); + let mut presenter = cx.build_presenter(window_id, Default::default(), Default::default()); fonts::with_font_cache(cx.font_cache().clone(), || { let mut text = Text::new("Hello\r\n".into(), Default::default()).with_soft_wrap(true); let (_, state) = text.layout( diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index b739ace947..d2ae7d8d17 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -132,6 +132,8 @@ pub trait Window { fn scale_factor(&self) -> f32; fn titlebar_height(&self) -> f32; fn present_scene(&mut self, scene: Scene); + fn appearance(&self) -> Appearance; + fn on_appearance_changed(&mut self, callback: Box); } #[derive(Debug)] @@ -147,6 +149,20 @@ pub struct TitlebarOptions<'a> { pub traffic_light_position: Option, } +#[derive(Copy, Clone, Debug)] +pub enum Appearance { + Light, + VibrantLight, + Dark, + VibrantDark, +} + +impl Default for Appearance { + fn default() -> Self { + Self::Light + } +} + #[derive(Debug)] pub enum WindowBounds { Maximized, @@ -173,6 +189,12 @@ pub enum CursorStyle { IBeam, } +impl Default for CursorStyle { + fn default() -> Self { + Self::Arrow + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct AppVersion { major: usize, @@ -180,12 +202,6 @@ pub struct AppVersion { patch: usize, } -impl Default for CursorStyle { - fn default() -> Self { - Self::Arrow - } -} - impl FromStr for AppVersion { type Err = anyhow::Error; diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index bd3c4eaf20..90b378e4a6 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -1,3 +1,4 @@ +mod appearance; mod atlas; mod dispatcher; mod event; diff --git a/crates/gpui/src/platform/mac/appearance.rs b/crates/gpui/src/platform/mac/appearance.rs new file mode 100644 index 0000000000..c0b8258a0e --- /dev/null +++ b/crates/gpui/src/platform/mac/appearance.rs @@ -0,0 +1,37 @@ +use std::ffi::CStr; + +use cocoa::{ + appkit::{NSAppearanceNameVibrantDark, NSAppearanceNameVibrantLight}, + base::id, + foundation::NSString, +}; +use objc::{msg_send, sel, sel_impl}; + +use crate::Appearance; + +impl Appearance { + pub unsafe fn from_native(appearance: id) -> Self { + let name: id = msg_send![appearance, name]; + if name == NSAppearanceNameVibrantLight { + Self::VibrantLight + } else if name == NSAppearanceNameVibrantDark { + Self::VibrantDark + } else if name == NSAppearanceNameAqua { + Self::Light + } else if name == NSAppearanceNameDarkAqua { + Self::Dark + } else { + println!( + "unknown appearance: {:?}", + CStr::from_ptr(name.UTF8String()) + ); + Self::Light + } + } +} + +#[link(name = "AppKit", kind = "framework")] +extern "C" { + pub static NSAppearanceNameAqua: id; + pub static NSAppearanceNameDarkAqua: id; +} diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index d1a3b7a797..29e6b58e59 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -52,6 +52,11 @@ use time::UtcOffset; #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; +#[allow(non_upper_case_globals)] +pub const NSViewLayerContentsRedrawDuringViewResize: NSInteger = 2; +#[allow(non_upper_case_globals)] +pub const NSKeyValueObservingOptionNew: NSInteger = 1; + const MAC_PLATFORM_IVAR: &str = "platform"; static mut APP_CLASS: *const Class = ptr::null(); static mut APP_DELEGATE_CLASS: *const Class = ptr::null(); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index c369ab4245..a6570e4890 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,12 +1,18 @@ use crate::{ geometry::vector::{vec2f, Vector2F}, - platform::{self, mac::renderer::Renderer}, + platform::{ + self, + mac::{ + platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, + renderer::Renderer, + }, + }, Event, FontSystem, Scene, }; use cocoa::{ appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, - foundation::{NSAutoreleasePool, NSPoint, NSRect, NSSize}, + foundation::{NSPoint, NSRect, NSSize, NSString}, }; use ctor::ctor; use foreign_types::ForeignTypeRef; @@ -15,7 +21,7 @@ use objc::{ declare::ClassDecl, msg_send, rc::StrongPtr, - runtime::{Class, Object, Sel}, + runtime::{Class, Object, Protocol, Sel}, sel, sel_impl, }; use std::{ @@ -77,6 +83,20 @@ unsafe fn build_classes() { sel!(flagsChanged:), handle_view_event as extern "C" fn(&Object, Sel, id), ); + decl.add_method( + sel!(makeBackingLayer), + make_backing_layer as extern "C" fn(&Object, Sel) -> id, + ); + decl.add_method( + sel!(observeValueForKeyPath:ofObject:change:context:), + appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + ); + + decl.add_protocol(Protocol::get("CALayerDelegate").unwrap()); + decl.add_method( + sel!(displayLayer:), + display_layer as extern "C" fn(&Object, Sel, id), + ); decl.register() }; @@ -86,15 +106,16 @@ pub struct StatusItem(Rc>); struct StatusItemState { native_item: StrongPtr, + native_view: StrongPtr, renderer: Renderer, + scene: Option, event_callback: Option bool>>, + appearance_changed_callback: Option>, } impl StatusItem { pub fn add(fonts: Arc) -> Self { unsafe { - let pool = NSAutoreleasePool::new(nil); - let renderer = Renderer::new(false, fonts); let status_bar = NSStatusBar::systemStatusBar(nil); let native_item = @@ -103,39 +124,51 @@ impl StatusItem { let button = native_item.button(); let _: () = msg_send![button, setHidden: YES]; - let item = Self(Rc::new_cyclic(|state| { - let parent_view = button.superview().superview(); - - let view: id = msg_send![VIEW_CLASS, alloc]; - NSView::initWithFrame_( - view, - NSRect::new(NSPoint::new(0., 0.), NSView::frame(parent_view).size), - ); - view.setWantsBestResolutionOpenGLSurface_(YES); - view.setLayer(renderer.layer().as_ptr() as id); - view.setWantsLayer(true); - (*view).set_ivar(STATE_IVAR, Weak::into_raw(state.clone()) as *const c_void); - parent_view.addSubview_(view.autorelease()); - - RefCell::new(StatusItemState { - native_item, - renderer, - event_callback: None, - }) + let native_view = msg_send![VIEW_CLASS, alloc]; + let state = Rc::new(RefCell::new(StatusItemState { + native_item, + native_view: StrongPtr::new(native_view), + renderer, + scene: None, + event_callback: None, + appearance_changed_callback: None, })); + let parent_view = button.superview().superview(); + NSView::initWithFrame_( + native_view, + NSRect::new(NSPoint::new(0., 0.), NSView::frame(parent_view).size), + ); + (*native_view).set_ivar( + STATE_IVAR, + Weak::into_raw(Rc::downgrade(&state)) as *const c_void, + ); + native_view.setWantsBestResolutionOpenGLSurface_(YES); + native_view.setWantsLayer(true); + let _: () = msg_send![ + native_view, + setLayerContentsRedrawPolicy: NSViewLayerContentsRedrawDuringViewResize + ]; + let _: () = msg_send![ + button, + addObserver: native_view + forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") + options: NSKeyValueObservingOptionNew + context: nil + ]; + + parent_view.addSubview_(native_view); + { - let item = item.0.borrow(); - let layer = item.renderer.layer(); - let scale_factor = item.scale_factor(); - let size = item.size() * scale_factor; + let state = state.borrow(); + let layer = state.renderer.layer(); + let scale_factor = state.scale_factor(); + let size = state.size() * scale_factor; layer.set_contents_scale(scale_factor.into()); layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); } - pool.drain(); - - item + Self(state) } } } @@ -149,6 +182,10 @@ impl platform::Window for StatusItem { self.0.borrow_mut().event_callback = Some(callback); } + fn on_appearance_changed(&mut self, callback: Box) { + self.0.borrow_mut().appearance_changed_callback = Some(callback); + } + fn on_active_status_change(&mut self, _: Box) { unimplemented!() } @@ -223,7 +260,18 @@ impl platform::Window for StatusItem { } fn present_scene(&mut self, scene: Scene) { - self.0.borrow_mut().renderer.render(&scene); + self.0.borrow_mut().scene = Some(scene); + unsafe { + let _: () = msg_send![*self.0.borrow().native_view, setNeedsDisplay: YES]; + } + } + + fn appearance(&self) -> crate::Appearance { + unsafe { + let appearance: id = + msg_send![self.0.borrow().native_item.button(), effectiveAppearance]; + crate::Appearance::from_native(appearance) + } } } @@ -247,6 +295,7 @@ impl StatusItemState { extern "C" fn dealloc_view(this: &Object, _: Sel) { unsafe { drop_state(this); + let _: () = msg_send![super(this, class!(NSView)), dealloc]; } } @@ -266,6 +315,39 @@ extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { } } +extern "C" fn make_backing_layer(this: &Object, _: Sel) -> id { + if let Some(state) = unsafe { get_state(this).upgrade() } { + let state = state.borrow(); + state.renderer.layer().as_ptr() as id + } else { + nil + } +} + +extern "C" fn display_layer(this: &Object, _: Sel, _: id) { + unsafe { + if let Some(state) = get_state(this).upgrade() { + let mut state = state.borrow_mut(); + if let Some(scene) = state.scene.take() { + state.renderer.render(&scene); + } + } + } +} + +extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { + unsafe { + if let Some(state) = get_state(this).upgrade() { + let mut state_borrow = state.as_ref().borrow_mut(); + if let Some(mut callback) = state_borrow.appearance_changed_callback.take() { + drop(state_borrow); + callback(); + state.borrow_mut().appearance_changed_callback = Some(callback); + } + } + } +} + unsafe fn get_state(object: &Object) -> Weak> { let raw: *mut c_void = *object.get_ivar(STATE_IVAR); let weak1 = Weak::from_raw(raw as *mut RefCell); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 4abee8b961..5cb45719eb 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1,4 +1,3 @@ -use super::{geometry::RectFExt, renderer::Renderer}; use crate::{ executor, geometry::{ @@ -6,7 +5,12 @@ use crate::{ vector::{vec2f, Vector2F}, }, keymap::Keystroke, - platform::{self, Event, WindowBounds}, + mac::platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, + platform::{ + self, + mac::{geometry::RectFExt, renderer::Renderer}, + Event, WindowBounds, + }, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, MouseMovedEvent, Scene, }; @@ -102,9 +106,6 @@ unsafe impl objc::Encode for NSRange { } } -#[allow(non_upper_case_globals)] -const NSViewLayerContentsRedrawDuringViewResize: NSInteger = 2; - #[ctor] unsafe fn build_classes() { WINDOW_CLASS = { @@ -264,6 +265,10 @@ unsafe fn build_classes() { attributed_substring_for_proposed_range as extern "C" fn(&Object, Sel, NSRange, *mut c_void) -> id, ); + decl.add_method( + sel!(observeValueForKeyPath:ofObject:change:context:), + appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + ); // Suppress beep on keystrokes with modifier keys. decl.add_method( @@ -298,6 +303,7 @@ struct WindowState { fullscreen_callback: Option>, should_close_callback: Option bool>>, close_callback: Option>, + appearance_changed_callback: Option>, input_handler: Option>, pending_key_down: Option<(KeyDownEvent, Option)>, performed_key_equivalent: bool, @@ -376,6 +382,7 @@ impl Window { close_callback: None, activate_callback: None, fullscreen_callback: None, + appearance_changed_callback: None, input_handler: None, pending_key_down: None, performed_key_equivalent: false, @@ -433,6 +440,13 @@ impl Window { native_window.center(); native_window.makeKeyAndOrderFront_(nil); + let _: () = msg_send![ + native_window, + addObserver: native_view + forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") + options: NSKeyValueObservingOptionNew + context: nil + ]; window.0.borrow().move_traffic_light(); pool.drain(); @@ -634,6 +648,17 @@ impl platform::Window for Window { fn titlebar_height(&self) -> f32 { self.0.as_ref().borrow().titlebar_height() } + + fn appearance(&self) -> crate::Appearance { + unsafe { + let appearance: id = msg_send![self.0.borrow().native_window, effectiveAppearance]; + crate::Appearance::from_native(appearance) + } + } + + fn on_appearance_changed(&mut self, callback: Box) { + self.0.borrow_mut().appearance_changed_callback = Some(callback); + } } impl WindowState { @@ -1270,6 +1295,18 @@ extern "C" fn do_command_by_selector(this: &Object, _: Sel, _: Sel) { } } +extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { + unsafe { + let state = get_window_state(this); + let mut state_borrow = state.as_ref().borrow_mut(); + if let Some(mut callback) = state_borrow.appearance_changed_callback.take() { + drop(state_borrow); + callback(); + state.borrow_mut().appearance_changed_callback = Some(callback); + } + } +} + async fn synthetic_drag( window_state: Weak>, drag_id: usize, diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 0ce44f0817..9a4f3638be 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -298,6 +298,12 @@ impl super::Window for Window { fn present_scene(&mut self, scene: crate::Scene) { self.current_scene = Some(scene); } + + fn appearance(&self) -> crate::Appearance { + crate::Appearance::Light + } + + fn on_appearance_changed(&mut self, _: Box) {} } pub fn platform() -> Platform { diff --git a/crates/gpui/src/presenter.rs b/crates/gpui/src/presenter.rs index 859ba46375..2943fda647 100644 --- a/crates/gpui/src/presenter.rs +++ b/crates/gpui/src/presenter.rs @@ -11,10 +11,10 @@ use crate::{ HoverRegionEvent, MouseRegionEvent, MoveRegionEvent, UpOutRegionEvent, UpRegionEvent, }, text_layout::TextLayoutCache, - Action, AnyModelHandle, AnyViewHandle, AnyWeakModelHandle, AssetCache, ElementBox, Entity, - FontSystem, ModelHandle, MouseButton, MouseMovedEvent, MouseRegion, MouseRegionId, ParentId, - ReadModel, ReadView, RenderContext, RenderParams, Scene, UpgradeModelHandle, UpgradeViewHandle, - View, ViewHandle, WeakModelHandle, WeakViewHandle, + Action, AnyModelHandle, AnyViewHandle, AnyWeakModelHandle, Appearance, AssetCache, ElementBox, + Entity, FontSystem, ModelHandle, MouseButton, MouseMovedEvent, MouseRegion, MouseRegionId, + ParentId, ReadModel, ReadView, RenderContext, RenderParams, Scene, UpgradeModelHandle, + UpgradeViewHandle, View, ViewHandle, WeakModelHandle, WeakViewHandle, }; use collections::{HashMap, HashSet}; use pathfinder_geometry::vector::{vec2f, Vector2F}; @@ -40,12 +40,14 @@ pub struct Presenter { clicked_button: Option, mouse_position: Vector2F, titlebar_height: f32, + appearance: Appearance, } impl Presenter { pub fn new( window_id: usize, titlebar_height: f32, + appearance: Appearance, font_cache: Arc, text_layout_cache: TextLayoutCache, asset_cache: Arc, @@ -53,7 +55,7 @@ impl Presenter { ) -> Self { Self { window_id, - rendered_views: cx.render_views(window_id, titlebar_height), + rendered_views: cx.render_views(window_id, titlebar_height, appearance), cursor_regions: Default::default(), mouse_regions: Default::default(), font_cache, @@ -65,15 +67,18 @@ impl Presenter { clicked_button: None, mouse_position: vec2f(0., 0.), titlebar_height, + appearance, } } pub fn invalidate( &mut self, invalidation: &mut WindowInvalidation, + appearance: Appearance, cx: &mut MutableAppContext, ) { cx.start_frame(); + self.appearance = appearance; for view_id in &invalidation.removed { invalidation.updated.remove(view_id); self.rendered_views.remove(view_id); @@ -96,14 +101,20 @@ impl Presenter { ) }), refreshing: false, + appearance, }) .unwrap(), ); } } - pub fn refresh(&mut self, invalidation: &mut WindowInvalidation, cx: &mut MutableAppContext) { - self.invalidate(invalidation, cx); + pub fn refresh( + &mut self, + invalidation: &mut WindowInvalidation, + appearance: Appearance, + cx: &mut MutableAppContext, + ) { + self.invalidate(invalidation, appearance, cx); for (view_id, view) in &mut self.rendered_views { if !invalidation.updated.contains(view_id) { *view = cx @@ -122,6 +133,7 @@ impl Presenter { ) }), refreshing: true, + appearance, }) .unwrap(); } @@ -194,6 +206,7 @@ impl Presenter { ) }), titlebar_height: self.titlebar_height, + appearance: self.appearance, window_size, app: cx, } @@ -545,6 +558,7 @@ pub struct LayoutContext<'a> { pub refreshing: bool, pub window_size: Vector2F, titlebar_height: f32, + appearance: Appearance, hovered_region_ids: HashSet, clicked_region_ids: Option<(Vec, MouseButton)>, } @@ -619,6 +633,7 @@ impl<'a> LayoutContext<'a> { hovered_region_ids: self.hovered_region_ids.clone(), clicked_region_ids: self.clicked_region_ids.clone(), refreshing: self.refreshing, + appearance: self.appearance, }; f(view, &mut render_cx) }) From 9b8492a3ba7c5befc22b08040528f0cdeb83a2a0 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 11:58:05 +0200 Subject: [PATCH 65/89] Extract a common `App::register_platform_window` --- crates/gpui/src/app.rs | 193 +++++++++----------- crates/gpui/src/platform/mac/status_item.rs | 24 +-- 2 files changed, 88 insertions(+), 129 deletions(-) diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 0baff67e94..c8e8efec79 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1924,84 +1924,11 @@ impl MutableAppContext { ); root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); - let mut window = + let window = this.cx .platform .open_window(window_id, window_options, this.foreground.clone()); - let presenter = Rc::new(RefCell::new(this.build_presenter( - window_id, - window.titlebar_height(), - window.appearance(), - ))); - - { - let mut app = this.upgrade(); - let presenter = Rc::downgrade(&presenter); - window.on_event(Box::new(move |event| { - app.update(|cx| { - if let Some(presenter) = presenter.upgrade() { - if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { - if cx.dispatch_keystroke(window_id, keystroke) { - return true; - } - } - - presenter.borrow_mut().dispatch_event(event, false, cx) - } else { - false - } - }) - })); - } - - { - let mut app = this.upgrade(); - window.on_active_status_change(Box::new(move |is_active| { - app.update(|cx| cx.window_changed_active_status(window_id, is_active)) - })); - } - - { - let mut app = this.upgrade(); - window.on_resize(Box::new(move || { - app.update(|cx| cx.window_was_resized(window_id)) - })); - } - - { - let mut app = this.upgrade(); - window.on_fullscreen(Box::new(move |is_fullscreen| { - app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) - })); - } - - { - let mut app = this.upgrade(); - window.on_close(Box::new(move || { - app.update(|cx| cx.remove_window(window_id)); - })); - } - - { - let mut app = this.upgrade(); - window - .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); - } - - window.set_input_handler(Box::new(WindowInputHandler { - app: this.upgrade().0, - window_id, - })); - - let scene = presenter.borrow_mut().build_scene( - window.size(), - window.scale_factor(), - false, - this, - ); - window.present_scene(scene); - this.presenters_and_platform_windows - .insert(window_id, (presenter.clone(), window)); + this.register_platform_window(window_id, window); (window_id, root_view) }) @@ -2029,47 +1956,91 @@ impl MutableAppContext { ); root_view.update(this, |view, cx| view.on_focus_in(cx.handle().into(), cx)); - let mut status_item = this.cx.platform.add_status_item(); - let presenter = Rc::new(RefCell::new(this.build_presenter( - window_id, - 0., - status_item.appearance(), - ))); - - { - let mut app = this.upgrade(); - let presenter = Rc::downgrade(&presenter); - status_item.on_event(Box::new(move |event| { - app.update(|cx| { - if let Some(presenter) = presenter.upgrade() { - presenter.borrow_mut().dispatch_event(event, false, cx) - } else { - false - } - }) - })); - } - - { - let mut app = this.upgrade(); - status_item - .on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); - } - - let scene = presenter.borrow_mut().build_scene( - status_item.size(), - status_item.scale_factor(), - false, - this, - ); - status_item.present_scene(scene); - this.presenters_and_platform_windows - .insert(window_id, (presenter.clone(), status_item)); + let status_item = this.cx.platform.add_status_item(); + this.register_platform_window(window_id, status_item); (window_id, root_view) }) } + fn register_platform_window( + &mut self, + window_id: usize, + mut window: Box, + ) { + let presenter = Rc::new(RefCell::new(self.build_presenter( + window_id, + window.titlebar_height(), + window.appearance(), + ))); + + { + let mut app = self.upgrade(); + let presenter = Rc::downgrade(&presenter); + window.on_event(Box::new(move |event| { + app.update(|cx| { + if let Some(presenter) = presenter.upgrade() { + if let Event::KeyDown(KeyDownEvent { keystroke, .. }) = &event { + if cx.dispatch_keystroke(window_id, keystroke) { + return true; + } + } + + presenter.borrow_mut().dispatch_event(event, false, cx) + } else { + false + } + }) + })); + } + + { + let mut app = self.upgrade(); + window.on_active_status_change(Box::new(move |is_active| { + app.update(|cx| cx.window_changed_active_status(window_id, is_active)) + })); + } + + { + let mut app = self.upgrade(); + window.on_resize(Box::new(move || { + app.update(|cx| cx.window_was_resized(window_id)) + })); + } + + { + let mut app = self.upgrade(); + window.on_fullscreen(Box::new(move |is_fullscreen| { + app.update(|cx| cx.window_was_fullscreen_changed(window_id, is_fullscreen)) + })); + } + + { + let mut app = self.upgrade(); + window.on_close(Box::new(move || { + app.update(|cx| cx.remove_window(window_id)); + })); + } + + { + let mut app = self.upgrade(); + window.on_appearance_changed(Box::new(move || app.update(|cx| cx.refresh_windows()))); + } + + window.set_input_handler(Box::new(WindowInputHandler { + app: self.upgrade().0, + window_id, + })); + + let scene = + presenter + .borrow_mut() + .build_scene(window.size(), window.scale_factor(), false, self); + window.present_scene(scene); + self.presenters_and_platform_windows + .insert(window_id, (presenter.clone(), window)); + } + pub fn replace_root_view(&mut self, window_id: usize, build_root_view: F) -> ViewHandle where T: View, diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index a6570e4890..b99b707bff 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -186,29 +186,17 @@ impl platform::Window for StatusItem { self.0.borrow_mut().appearance_changed_callback = Some(callback); } - fn on_active_status_change(&mut self, _: Box) { - unimplemented!() - } + fn on_active_status_change(&mut self, _: Box) {} - fn on_resize(&mut self, _: Box) { - unimplemented!() - } + fn on_resize(&mut self, _: Box) {} - fn on_fullscreen(&mut self, _: Box) { - unimplemented!() - } + fn on_fullscreen(&mut self, _: Box) {} - fn on_should_close(&mut self, _: Box bool>) { - unimplemented!() - } + fn on_should_close(&mut self, _: Box bool>) {} - fn on_close(&mut self, _: Box) { - unimplemented!() - } + fn on_close(&mut self, _: Box) {} - fn set_input_handler(&mut self, _: Box) { - unimplemented!() - } + fn set_input_handler(&mut self, _: Box) {} fn prompt( &self, From d10f6f60adad0b64f511531f3d5bd4ea0b6cf0b8 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 15:43:51 +0200 Subject: [PATCH 66/89] Toggle contacts popover when clicking on status bar icon --- .../src/contacts_popover.rs | 26 +++++++ .../src/contacts_status_item.rs | 63 ++++++++++++++-- crates/gpui/src/app.rs | 29 ++++++-- crates/gpui/src/platform.rs | 5 +- crates/gpui/src/platform/mac/status_item.rs | 45 +++++++++--- crates/gpui/src/platform/mac/window.rs | 71 ++++++++++++++----- crates/gpui/src/platform/test.rs | 11 ++- crates/zed/src/main.rs | 1 + crates/zed/src/zed.rs | 1 + 9 files changed, 208 insertions(+), 44 deletions(-) create mode 100644 crates/contacts_status_item/src/contacts_popover.rs diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs new file mode 100644 index 0000000000..0365a2ee2a --- /dev/null +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -0,0 +1,26 @@ +use gpui::{color::Color, elements::*, Entity, RenderContext, View}; + +pub struct ContactsPopover; + +impl Entity for ContactsPopover { + type Event = (); +} + +impl View for ContactsPopover { + fn ui_name() -> &'static str { + "ContactsPopover" + } + + fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + Empty::new() + .contained() + .with_background_color(Color::red()) + .boxed() + } +} + +impl ContactsPopover { + pub fn new() -> Self { + Self + } +} diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index f3363f18d3..70e8083a76 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -1,6 +1,24 @@ -use gpui::{color::Color, elements::*, Appearance, Entity, RenderContext, View}; +mod contacts_popover; -pub struct ContactsStatusItem; +use contacts_popover::ContactsPopover; +use gpui::{ + actions, + color::Color, + elements::*, + geometry::{rect::RectF, vector::vec2f}, + Appearance, Entity, MouseButton, MutableAppContext, RenderContext, View, ViewContext, + ViewHandle, +}; + +actions!(contacts_status_item, [ToggleContactsPopover]); + +pub fn init(cx: &mut MutableAppContext) { + cx.add_action(ContactsStatusItem::toggle_contacts_popover); +} + +pub struct ContactsStatusItem { + popover: Option>, +} impl Entity for ContactsStatusItem { type Event = (); @@ -16,15 +34,46 @@ impl View for ContactsStatusItem { Appearance::Light | Appearance::VibrantLight => Color::black(), Appearance::Dark | Appearance::VibrantDark => Color::white(), }; - Svg::new("icons/zed_22.svg") - .with_color(color) - .aligned() - .boxed() + MouseEventHandler::new::(0, cx, |_, _| { + Svg::new("icons/zed_22.svg") + .with_color(color) + .aligned() + .boxed() + }) + .on_click(MouseButton::Left, |_, cx| { + cx.dispatch_action(ToggleContactsPopover); + }) + .boxed() } } impl ContactsStatusItem { pub fn new() -> Self { - Self + Self { popover: None } + } + + fn toggle_contacts_popover(&mut self, _: &ToggleContactsPopover, cx: &mut ViewContext) { + match self.popover.take() { + Some(popover) => { + cx.remove_window(popover.window_id()); + } + None => { + let window_bounds = cx.window_bounds(); + let size = vec2f(360., 460.); + let origin = window_bounds.lower_left() + + vec2f(window_bounds.width() / 2. - size.x() / 2., 0.); + self.popover = Some( + cx.add_window( + gpui::WindowOptions { + bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), + titlebar: None, + center: false, + }, + |_| ContactsPopover::new(), + ) + .1, + ); + } + } } } diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index c8e8efec79..3ea88995f1 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -1244,6 +1244,10 @@ impl MutableAppContext { .map_or(false, |window| window.is_fullscreen) } + pub fn window_bounds(&self, window_id: usize) -> RectF { + self.presenters_and_platform_windows[&window_id].1.bounds() + } + pub fn render_view(&mut self, params: RenderParams) -> Result { let window_id = params.window_id; let view_id = params.view_id; @@ -2032,10 +2036,12 @@ impl MutableAppContext { window_id, })); - let scene = - presenter - .borrow_mut() - .build_scene(window.size(), window.scale_factor(), false, self); + let scene = presenter.borrow_mut().build_scene( + window.content_size(), + window.scale_factor(), + false, + self, + ); window.present_scene(scene); self.presenters_and_platform_windows .insert(window_id, (presenter.clone(), window)); @@ -2411,8 +2417,12 @@ impl MutableAppContext { { let mut presenter = presenter.borrow_mut(); presenter.invalidate(&mut invalidation, window.appearance(), self); - let scene = - presenter.build_scene(window.size(), window.scale_factor(), false, self); + let scene = presenter.build_scene( + window.content_size(), + window.scale_factor(), + false, + self, + ); window.present_scene(scene); } self.presenters_and_platform_windows @@ -2477,7 +2487,8 @@ impl MutableAppContext { window.appearance(), self, ); - let scene = presenter.build_scene(window.size(), window.scale_factor(), true, self); + let scene = + presenter.build_scene(window.content_size(), window.scale_factor(), true, self); window.present_scene(scene); } self.presenters_and_platform_windows = presenters; @@ -3749,6 +3760,10 @@ impl<'a, T: View> ViewContext<'a, T> { self.app.toggle_window_full_screen(self.window_id) } + pub fn window_bounds(&self) -> RectF { + self.app.window_bounds(self.window_id) + } + pub fn prompt( &self, level: PromptLevel, diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index d2ae7d8d17..f2fd09d30b 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -128,7 +128,8 @@ pub trait Window { fn zoom(&self); fn toggle_full_screen(&self); - fn size(&self) -> Vector2F; + fn bounds(&self) -> RectF; + fn content_size(&self) -> Vector2F; fn scale_factor(&self) -> f32; fn titlebar_height(&self) -> f32; fn present_scene(&mut self, scene: Scene); @@ -140,6 +141,7 @@ pub trait Window { pub struct WindowOptions<'a> { pub bounds: WindowBounds, pub titlebar: Option>, + pub center: bool, } #[derive(Debug)] @@ -273,6 +275,7 @@ impl<'a> Default for WindowOptions<'a> { appears_transparent: Default::default(), traffic_light_position: Default::default(), }), + center: false, } } } diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index b99b707bff..1ed39894a1 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -1,5 +1,8 @@ use crate::{ - geometry::vector::{vec2f, Vector2F}, + geometry::{ + rect::RectF, + vector::{vec2f, Vector2F}, + }, platform::{ self, mac::{ @@ -10,7 +13,7 @@ use crate::{ Event, FontSystem, Scene, }; use cocoa::{ - appkit::{NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, + appkit::{NSScreen, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, foundation::{NSPoint, NSRect, NSSize, NSString}, }; @@ -163,7 +166,7 @@ impl StatusItem { let state = state.borrow(); let layer = state.renderer.layer(); let scale_factor = state.scale_factor(); - let size = state.size() * scale_factor; + let size = state.content_size() * scale_factor; layer.set_contents_scale(scale_factor.into()); layer.set_drawable_size(metal::CGSize::new(size.x().into(), size.y().into())); } @@ -235,8 +238,12 @@ impl platform::Window for StatusItem { unimplemented!() } - fn size(&self) -> Vector2F { - self.0.borrow().size() + fn bounds(&self) -> RectF { + self.0.borrow().bounds() + } + + fn content_size(&self) -> Vector2F { + self.0.borrow().content_size() } fn scale_factor(&self) -> f32 { @@ -264,10 +271,28 @@ impl platform::Window for StatusItem { } impl StatusItemState { - fn size(&self) -> Vector2F { + fn bounds(&self) -> RectF { + unsafe { + let window: id = msg_send![self.native_item.button(), window]; + let screen_frame = window.screen().visibleFrame(); + let window_frame = NSWindow::frame(window); + let origin = vec2f( + window_frame.origin.x as f32, + (window_frame.origin.y - screen_frame.size.height - window_frame.size.height) + as f32, + ); + let size = vec2f( + window_frame.size.width as f32, + window_frame.size.height as f32, + ); + RectF::new(origin, size) + } + } + + fn content_size(&self) -> Vector2F { unsafe { let NSSize { width, height, .. } = - NSWindow::frame(self.native_item.button().superview().superview()).size; + NSView::frame(self.native_item.button().superview().superview()).size; vec2f(width as f32, height as f32) } } @@ -275,7 +300,7 @@ impl StatusItemState { fn scale_factor(&self) -> f32 { unsafe { let window: id = msg_send![self.native_item.button(), window]; - window.screen().backingScaleFactor() as f32 + NSScreen::backingScaleFactor(window.screen()) as f32 } } } @@ -292,7 +317,9 @@ extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { unsafe { if let Some(state) = get_state(this).upgrade() { let mut state_borrow = state.as_ref().borrow_mut(); - if let Some(event) = Event::from_native(native_event, Some(state_borrow.size().y())) { + if let Some(event) = + Event::from_native(native_event, Some(state_borrow.content_size().y())) + { if let Some(mut callback) = state_borrow.event_callback.take() { drop(state_borrow); callback(event); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 5cb45719eb..00688f1ea5 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -335,12 +335,6 @@ impl Window { unsafe { let pool = NSAutoreleasePool::new(nil); - let frame = match options.bounds { - WindowBounds::Maximized => RectF::new(Default::default(), vec2f(1024., 768.)), - WindowBounds::Fixed(rect) => rect, - } - .to_ns_rect(); - let mut style_mask; if let Some(titlebar) = options.titlebar.as_ref() { style_mask = NSWindowStyleMask::NSClosableWindowMask @@ -357,16 +351,31 @@ impl Window { let native_window: id = msg_send![WINDOW_CLASS, alloc]; let native_window = native_window.initWithContentRect_styleMask_backing_defer_( - frame, + RectF::new(Default::default(), vec2f(1024., 768.)).to_ns_rect(), style_mask, NSBackingStoreBuffered, NO, ); assert!(!native_window.is_null()); - if matches!(options.bounds, WindowBounds::Maximized) { - let screen = native_window.screen(); - native_window.setFrame_display_(screen.visibleFrame(), YES); + let screen = native_window.screen(); + match options.bounds { + WindowBounds::Maximized => { + native_window.setFrame_display_(screen.visibleFrame(), YES); + } + WindowBounds::Fixed(top_left_bounds) => { + let frame = screen.visibleFrame(); + let bottom_left_bounds = RectF::new( + vec2f( + top_left_bounds.origin_x(), + frame.size.height as f32 + - top_left_bounds.origin_y() + - top_left_bounds.height(), + ), + top_left_bounds.size(), + ); + native_window.setFrame_display_(bottom_left_bounds.to_ns_rect(), YES); + } } let native_view: id = msg_send![VIEW_CLASS, alloc]; @@ -438,7 +447,10 @@ impl Window { native_window.setContentView_(native_view.autorelease()); native_window.makeFirstResponder_(native_view); - native_window.center(); + if options.center { + native_window.center(); + } + native_window.makeKeyAndOrderFront_(nil); let _: () = msg_send![ native_window, @@ -633,8 +645,12 @@ impl platform::Window for Window { .detach(); } - fn size(&self) -> Vector2F { - self.0.as_ref().borrow().size() + fn bounds(&self) -> RectF { + self.0.as_ref().borrow().bounds() + } + + fn content_size(&self) -> Vector2F { + self.0.as_ref().borrow().content_size() } fn scale_factor(&self) -> f32 { @@ -706,7 +722,24 @@ impl WindowState { } } - fn size(&self) -> Vector2F { + fn bounds(&self) -> RectF { + unsafe { + let screen_frame = self.native_window.screen().visibleFrame(); + let window_frame = NSWindow::frame(self.native_window); + let origin = vec2f( + window_frame.origin.x as f32, + (window_frame.origin.y - screen_frame.size.height - window_frame.size.height) + as f32, + ); + let size = vec2f( + window_frame.size.width as f32, + window_frame.size.height as f32, + ); + RectF::new(origin, size) + } + } + + fn content_size(&self) -> Vector2F { let NSSize { width, height, .. } = unsafe { NSView::frame(self.native_window.contentView()) }.size; vec2f(width as f32, height as f32) @@ -783,7 +816,8 @@ extern "C" fn handle_key_event(this: &Object, native_event: id, key_equivalent: let mut window_state_borrow = window_state.as_ref().borrow_mut(); - let event = unsafe { Event::from_native(native_event, Some(window_state_borrow.size().y())) }; + let event = + unsafe { Event::from_native(native_event, Some(window_state_borrow.content_size().y())) }; if let Some(event) = event { if key_equivalent { @@ -875,7 +909,8 @@ extern "C" fn handle_view_event(this: &Object, _: Sel, native_event: id) { let weak_window_state = Rc::downgrade(&window_state); let mut window_state_borrow = window_state.as_ref().borrow_mut(); - let event = unsafe { Event::from_native(native_event, Some(window_state_borrow.size().y())) }; + let event = + unsafe { Event::from_native(native_event, Some(window_state_borrow.content_size().y())) }; if let Some(event) = event { match &event { Event::MouseMoved( @@ -1060,7 +1095,7 @@ extern "C" fn view_did_change_backing_properties(this: &Object, _: Sel) { unsafe { let scale_factor = window_state_borrow.scale_factor() as f64; - let size = window_state_borrow.size(); + let size = window_state_borrow.content_size(); let drawable_size: NSSize = NSSize { width: size.x() as f64 * scale_factor, height: size.y() as f64 * scale_factor, @@ -1087,7 +1122,7 @@ extern "C" fn set_frame_size(this: &Object, _: Sel, size: NSSize) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.as_ref().borrow(); - if window_state_borrow.size() == vec2f(size.width as f32, size.height as f32) { + if window_state_borrow.content_size() == vec2f(size.width as f32, size.height as f32) { return; } diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index 9a4f3638be..9a458a1dd9 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -1,6 +1,9 @@ use super::{AppVersion, CursorStyle, WindowBounds}; use crate::{ - geometry::vector::{vec2f, Vector2F}, + geometry::{ + rect::RectF, + vector::{vec2f, Vector2F}, + }, keymap, Action, ClipboardItem, }; use anyhow::{anyhow, Result}; @@ -283,7 +286,11 @@ impl super::Window for Window { fn toggle_full_screen(&self) {} - fn size(&self) -> Vector2F { + fn bounds(&self) -> RectF { + RectF::new(Default::default(), self.size) + } + + fn content_size(&self) -> Vector2F { self.size } diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index 287ac721fd..868ad15d67 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -105,6 +105,7 @@ fn main() { watch_settings_file(default_settings, settings_file, themes.clone(), cx); watch_keymap_file(keymap_file, cx); + contacts_status_item::init(cx); context_menu::init(cx); project::Project::init(&client); client::Channel::init(&client); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index 59e9f8de9d..a14e85e93b 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -335,6 +335,7 @@ pub fn build_window_options() -> WindowOptions<'static> { appears_transparent: true, traffic_light_position: Some(vec2f(8., 8.)), }), + center: false, } } From c1f448d8a88aef17163d05bce4fc092de103ea57 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 15:49:08 +0200 Subject: [PATCH 67/89] Use `viewDidChangeEffectiveAppearance` to detect appearance changes --- crates/gpui/src/platform/mac/platform.rs | 2 -- crates/gpui/src/platform/mac/status_item.rs | 20 +++++--------------- crates/gpui/src/platform/mac/window.rs | 15 ++++----------- 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index 29e6b58e59..9169e72d06 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -54,8 +54,6 @@ const NSUTF8StringEncoding: NSUInteger = 4; #[allow(non_upper_case_globals)] pub const NSViewLayerContentsRedrawDuringViewResize: NSInteger = 2; -#[allow(non_upper_case_globals)] -pub const NSKeyValueObservingOptionNew: NSInteger = 1; const MAC_PLATFORM_IVAR: &str = "platform"; static mut APP_CLASS: *const Class = ptr::null(); diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 1ed39894a1..404b5e2ef4 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -5,17 +5,14 @@ use crate::{ }, platform::{ self, - mac::{ - platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, - renderer::Renderer, - }, + mac::{platform::NSViewLayerContentsRedrawDuringViewResize, renderer::Renderer}, }, Event, FontSystem, Scene, }; use cocoa::{ appkit::{NSScreen, NSSquareStatusItemLength, NSStatusBar, NSStatusItem, NSView, NSWindow}, base::{id, nil, YES}, - foundation::{NSPoint, NSRect, NSSize, NSString}, + foundation::{NSPoint, NSRect, NSSize}, }; use ctor::ctor; use foreign_types::ForeignTypeRef; @@ -91,8 +88,8 @@ unsafe fn build_classes() { make_backing_layer as extern "C" fn(&Object, Sel) -> id, ); decl.add_method( - sel!(observeValueForKeyPath:ofObject:change:context:), - appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + sel!(viewDidChangeEffectiveAppearance), + view_did_change_effective_appearance as extern "C" fn(&Object, Sel), ); decl.add_protocol(Protocol::get("CALayerDelegate").unwrap()); @@ -152,13 +149,6 @@ impl StatusItem { native_view, setLayerContentsRedrawPolicy: NSViewLayerContentsRedrawDuringViewResize ]; - let _: () = msg_send![ - button, - addObserver: native_view - forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") - options: NSKeyValueObservingOptionNew - context: nil - ]; parent_view.addSubview_(native_view); @@ -350,7 +340,7 @@ extern "C" fn display_layer(this: &Object, _: Sel, _: id) { } } -extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { +extern "C" fn view_did_change_effective_appearance(this: &Object, _: Sel) { unsafe { if let Some(state) = get_state(this).upgrade() { let mut state_borrow = state.as_ref().borrow_mut(); diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 00688f1ea5..2bdd5a720c 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -5,7 +5,7 @@ use crate::{ vector::{vec2f, Vector2F}, }, keymap::Keystroke, - mac::platform::{NSKeyValueObservingOptionNew, NSViewLayerContentsRedrawDuringViewResize}, + mac::platform::NSViewLayerContentsRedrawDuringViewResize, platform::{ self, mac::{geometry::RectFExt, renderer::Renderer}, @@ -266,8 +266,8 @@ unsafe fn build_classes() { as extern "C" fn(&Object, Sel, NSRange, *mut c_void) -> id, ); decl.add_method( - sel!(observeValueForKeyPath:ofObject:change:context:), - appearance_changed as extern "C" fn(&Object, Sel, id, id, id, id), + sel!(viewDidChangeEffectiveAppearance), + view_did_change_effective_appearance as extern "C" fn(&Object, Sel), ); // Suppress beep on keystrokes with modifier keys. @@ -452,13 +452,6 @@ impl Window { } native_window.makeKeyAndOrderFront_(nil); - let _: () = msg_send![ - native_window, - addObserver: native_view - forKeyPath: NSString::alloc(nil).init_str("effectiveAppearance") - options: NSKeyValueObservingOptionNew - context: nil - ]; window.0.borrow().move_traffic_light(); pool.drain(); @@ -1330,7 +1323,7 @@ extern "C" fn do_command_by_selector(this: &Object, _: Sel, _: Sel) { } } -extern "C" fn appearance_changed(this: &Object, _: Sel, _: id, _: id, _: id, _: id) { +extern "C" fn view_did_change_effective_appearance(this: &Object, _: Sel) { unsafe { let state = get_window_state(this); let mut state_borrow = state.as_ref().borrow_mut(); From 1c9c7ef7ae75e8051f76e124d9d9396d3d904e67 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 14 Sep 2022 17:09:07 +0200 Subject: [PATCH 68/89] Add the ability to specify a level when creating windows This lets some windows stay on top of others, independently of whether the application is in the foreground. --- .../src/contacts_status_item.rs | 19 +++++++++---------- crates/gpui/src/platform.rs | 8 ++++++++ crates/gpui/src/platform/mac/window.rs | 13 ++++++++++++- crates/zed/src/zed.rs | 3 ++- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 70e8083a76..33ca61ce6c 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -62,17 +62,16 @@ impl ContactsStatusItem { let size = vec2f(360., 460.); let origin = window_bounds.lower_left() + vec2f(window_bounds.width() / 2. - size.x() / 2., 0.); - self.popover = Some( - cx.add_window( - gpui::WindowOptions { - bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), - titlebar: None, - center: false, - }, - |_| ContactsPopover::new(), - ) - .1, + let (_, popover) = cx.add_window( + gpui::WindowOptions { + bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), + titlebar: None, + center: false, + level: gpui::WindowLevel::PopUp, + }, + |_| ContactsPopover::new(), ); + self.popover = Some(popover); } } } diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index f2fd09d30b..87c44ae390 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -142,6 +142,7 @@ pub struct WindowOptions<'a> { pub bounds: WindowBounds, pub titlebar: Option>, pub center: bool, + pub level: WindowLevel, } #[derive(Debug)] @@ -165,6 +166,12 @@ impl Default for Appearance { } } +#[derive(Copy, Clone, Debug)] +pub enum WindowLevel { + Normal, + PopUp, +} + #[derive(Debug)] pub enum WindowBounds { Maximized, @@ -276,6 +283,7 @@ impl<'a> Default for WindowOptions<'a> { traffic_light_position: Default::default(), }), center: false, + level: WindowLevel::Normal, } } } diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 2bdd5a720c..90dabfff4e 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -12,7 +12,7 @@ use crate::{ Event, WindowBounds, }, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, - MouseMovedEvent, Scene, + MouseMovedEvent, Scene, WindowLevel, }; use block::ConcreteBlock; use cocoa::{ @@ -56,6 +56,12 @@ const WINDOW_STATE_IVAR: &str = "windowState"; static mut WINDOW_CLASS: *const Class = ptr::null(); static mut VIEW_CLASS: *const Class = ptr::null(); +#[allow(non_upper_case_globals)] +const NSNormalWindowLevel: NSInteger = 0; + +#[allow(non_upper_case_globals)] +const NSPopUpWindowLevel: NSInteger = 101; + #[repr(C)] #[derive(Copy, Clone, Debug)] struct NSRange { @@ -452,6 +458,11 @@ impl Window { } native_window.makeKeyAndOrderFront_(nil); + let native_level = match options.level { + WindowLevel::Normal => NSNormalWindowLevel, + WindowLevel::PopUp => NSPopUpWindowLevel, + }; + native_window.setLevel_(native_level); window.0.borrow().move_traffic_light(); pool.drain(); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index a14e85e93b..077282f526 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -20,7 +20,7 @@ use gpui::{ geometry::vector::vec2f, impl_actions, platform::{WindowBounds, WindowOptions}, - AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, + AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, WindowLevel, }; use language::Rope; pub use lsp; @@ -336,6 +336,7 @@ pub fn build_window_options() -> WindowOptions<'static> { traffic_light_position: Some(vec2f(8., 8.)), }), center: false, + level: WindowLevel::Normal, } } From 44553875d0ec0e5445b428e5c0339054410b8800 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 11:44:51 +0200 Subject: [PATCH 69/89] Allow contacts popover to be activated even if app isn't foregrounded --- .../src/contacts_popover.rs | 18 ++- .../src/contacts_status_item.rs | 22 ++- crates/gpui/src/platform.rs | 8 +- crates/gpui/src/platform/mac/window.rs | 135 ++++++++++-------- crates/zed/src/zed.rs | 5 +- 5 files changed, 118 insertions(+), 70 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs index 0365a2ee2a..7637b51cc9 100644 --- a/crates/contacts_status_item/src/contacts_popover.rs +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -1,9 +1,13 @@ -use gpui::{color::Color, elements::*, Entity, RenderContext, View}; +use gpui::{color::Color, elements::*, Entity, RenderContext, View, ViewContext}; + +pub enum Event { + Deactivated, +} pub struct ContactsPopover; impl Entity for ContactsPopover { - type Event = (); + type Event = Event; } impl View for ContactsPopover { @@ -20,7 +24,15 @@ impl View for ContactsPopover { } impl ContactsPopover { - pub fn new() -> Self { + pub fn new(cx: &mut ViewContext) -> Self { + cx.observe_window_activation(Self::window_activation_changed) + .detach(); Self } + + fn window_activation_changed(&mut self, is_active: bool, cx: &mut ViewContext) { + if !is_active { + cx.emit(Event::Deactivated); + } + } } diff --git a/crates/contacts_status_item/src/contacts_status_item.rs b/crates/contacts_status_item/src/contacts_status_item.rs index 33ca61ce6c..83cdce7e8e 100644 --- a/crates/contacts_status_item/src/contacts_status_item.rs +++ b/crates/contacts_status_item/src/contacts_status_item.rs @@ -7,7 +7,7 @@ use gpui::{ elements::*, geometry::{rect::RectF, vector::vec2f}, Appearance, Entity, MouseButton, MutableAppContext, RenderContext, View, ViewContext, - ViewHandle, + ViewHandle, WindowKind, }; actions!(contacts_status_item, [ToggleContactsPopover]); @@ -67,12 +67,28 @@ impl ContactsStatusItem { bounds: gpui::WindowBounds::Fixed(RectF::new(origin, size)), titlebar: None, center: false, - level: gpui::WindowLevel::PopUp, + kind: WindowKind::PopUp, + is_movable: false, }, - |_| ContactsPopover::new(), + |cx| ContactsPopover::new(cx), ); + cx.subscribe(&popover, Self::on_popover_event).detach(); self.popover = Some(popover); } } } + + fn on_popover_event( + &mut self, + popover: ViewHandle, + event: &contacts_popover::Event, + cx: &mut ViewContext, + ) { + match event { + contacts_popover::Event::Deactivated => { + self.popover.take(); + cx.remove_window(popover.window_id()); + } + } + } } diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index 87c44ae390..89e55c9f6f 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -142,7 +142,8 @@ pub struct WindowOptions<'a> { pub bounds: WindowBounds, pub titlebar: Option>, pub center: bool, - pub level: WindowLevel, + pub kind: WindowKind, + pub is_movable: bool, } #[derive(Debug)] @@ -167,7 +168,7 @@ impl Default for Appearance { } #[derive(Copy, Clone, Debug)] -pub enum WindowLevel { +pub enum WindowKind { Normal, PopUp, } @@ -283,7 +284,8 @@ impl<'a> Default for WindowOptions<'a> { traffic_light_position: Default::default(), }), center: false, - level: WindowLevel::Normal, + kind: WindowKind::Normal, + is_movable: true, } } } diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 90dabfff4e..f21428a1a8 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -12,7 +12,7 @@ use crate::{ Event, WindowBounds, }, InputHandler, KeyDownEvent, ModifiersChangedEvent, MouseButton, MouseButtonEvent, - MouseMovedEvent, Scene, WindowLevel, + MouseMovedEvent, Scene, WindowKind, }; use block::ConcreteBlock; use cocoa::{ @@ -54,6 +54,7 @@ use std::{ const WINDOW_STATE_IVAR: &str = "windowState"; static mut WINDOW_CLASS: *const Class = ptr::null(); +static mut PANEL_CLASS: *const Class = ptr::null(); static mut VIEW_CLASS: *const Class = ptr::null(); #[allow(non_upper_case_globals)] @@ -114,50 +115,8 @@ unsafe impl objc::Encode for NSRange { #[ctor] unsafe fn build_classes() { - WINDOW_CLASS = { - let mut decl = ClassDecl::new("GPUIWindow", class!(NSWindow)).unwrap(); - decl.add_ivar::<*mut c_void>(WINDOW_STATE_IVAR); - decl.add_method(sel!(dealloc), dealloc_window as extern "C" fn(&Object, Sel)); - decl.add_method( - sel!(canBecomeMainWindow), - yes as extern "C" fn(&Object, Sel) -> BOOL, - ); - decl.add_method( - sel!(canBecomeKeyWindow), - yes as extern "C" fn(&Object, Sel) -> BOOL, - ); - decl.add_method( - sel!(sendEvent:), - send_event as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowDidResize:), - window_did_resize as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowWillEnterFullScreen:), - window_will_enter_fullscreen as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowWillExitFullScreen:), - window_will_exit_fullscreen as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowDidBecomeKey:), - window_did_change_key_status as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowDidResignKey:), - window_did_change_key_status as extern "C" fn(&Object, Sel, id), - ); - decl.add_method( - sel!(windowShouldClose:), - window_should_close as extern "C" fn(&Object, Sel, id) -> BOOL, - ); - decl.add_method(sel!(close), close_window as extern "C" fn(&Object, Sel)); - decl.register() - }; - + WINDOW_CLASS = build_window_class("GPUIWindow", class!(NSWindow)); + PANEL_CLASS = build_window_class("GPUIPanel", class!(NSPanel)); VIEW_CLASS = { let mut decl = ClassDecl::new("GPUIView", class!(NSView)).unwrap(); decl.add_ivar::<*mut c_void>(WINDOW_STATE_IVAR); @@ -286,6 +245,50 @@ unsafe fn build_classes() { }; } +unsafe fn build_window_class(name: &'static str, superclass: &Class) -> *const Class { + let mut decl = ClassDecl::new(name, superclass).unwrap(); + decl.add_ivar::<*mut c_void>(WINDOW_STATE_IVAR); + decl.add_method(sel!(dealloc), dealloc_window as extern "C" fn(&Object, Sel)); + decl.add_method( + sel!(canBecomeMainWindow), + yes as extern "C" fn(&Object, Sel) -> BOOL, + ); + decl.add_method( + sel!(canBecomeKeyWindow), + yes as extern "C" fn(&Object, Sel) -> BOOL, + ); + decl.add_method( + sel!(sendEvent:), + send_event as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowDidResize:), + window_did_resize as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowWillEnterFullScreen:), + window_will_enter_fullscreen as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowWillExitFullScreen:), + window_will_exit_fullscreen as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowDidBecomeKey:), + window_did_change_key_status as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowDidResignKey:), + window_did_change_key_status as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(windowShouldClose:), + window_should_close as extern "C" fn(&Object, Sel, id) -> BOOL, + ); + decl.add_method(sel!(close), close_window as extern "C" fn(&Object, Sel)); + decl.register() +} + pub struct Window(Rc>); ///Used to track what the IME does when we send it a keystroke. @@ -352,10 +355,21 @@ impl Window { style_mask |= NSWindowStyleMask::NSFullSizeContentViewWindowMask; } } else { - style_mask = NSWindowStyleMask::empty(); + style_mask = NSWindowStyleMask::NSTitledWindowMask + | NSWindowStyleMask::NSFullSizeContentViewWindowMask; } - let native_window: id = msg_send![WINDOW_CLASS, alloc]; + let native_window: id = match options.kind { + WindowKind::Normal => msg_send![WINDOW_CLASS, alloc], + WindowKind::PopUp => { + #[allow(non_upper_case_globals)] + const NSWindowStyleMaskNonactivatingPanel: NSWindowStyleMask = + unsafe { NSWindowStyleMask::from_bits_unchecked(1 << 7) }; + + style_mask |= NSWindowStyleMaskNonactivatingPanel; + msg_send![PANEL_CLASS, alloc] + } + }; let native_window = native_window.initWithContentRect_styleMask_backing_defer_( RectF::new(Default::default(), vec2f(1024., 768.)).to_ns_rect(), style_mask, @@ -425,13 +439,17 @@ impl Window { Rc::into_raw(window.0.clone()) as *const c_void, ); - if let Some(titlebar) = options.titlebar { - if let Some(title) = titlebar.title { - native_window.setTitle_(NSString::alloc(nil).init_str(title)); - } - if titlebar.appears_transparent { - native_window.setTitlebarAppearsTransparent_(YES); - } + if let Some(title) = options.titlebar.as_ref().and_then(|t| t.title) { + native_window.setTitle_(NSString::alloc(nil).init_str(title)); + } + + native_window.setMovable_(options.is_movable); + + if options + .titlebar + .map_or(true, |titlebar| titlebar.appears_transparent) + { + native_window.setTitlebarAppearsTransparent_(YES); } native_window.setAcceptsMouseMovedEvents_(YES); @@ -458,11 +476,10 @@ impl Window { } native_window.makeKeyAndOrderFront_(nil); - let native_level = match options.level { - WindowLevel::Normal => NSNormalWindowLevel, - WindowLevel::PopUp => NSPopUpWindowLevel, - }; - native_window.setLevel_(native_level); + match options.kind { + WindowKind::Normal => native_window.setLevel_(NSNormalWindowLevel), + WindowKind::PopUp => native_window.setLevel_(NSPopUpWindowLevel), + } window.0.borrow().move_traffic_light(); pool.drain(); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index 077282f526..bee77d2d03 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -20,7 +20,7 @@ use gpui::{ geometry::vector::vec2f, impl_actions, platform::{WindowBounds, WindowOptions}, - AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, WindowLevel, + AssetSource, AsyncAppContext, TitlebarOptions, ViewContext, WindowKind, }; use language::Rope; pub use lsp; @@ -336,7 +336,8 @@ pub fn build_window_options() -> WindowOptions<'static> { traffic_light_position: Some(vec2f(8., 8.)), }), center: false, - level: WindowLevel::Normal, + kind: WindowKind::Normal, + is_movable: true, } } From 0c422fadb8315cab5d16c7a846c1b9925a7b60eb Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 11:57:22 +0200 Subject: [PATCH 70/89] Style contacts popover background based on theme --- crates/contacts_status_item/src/contacts_popover.rs | 6 ++++-- crates/theme/src/theme.rs | 6 ++++++ styles/src/styleTree/app.ts | 2 ++ styles/src/styleTree/contactsPopover.ts | 8 ++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 styles/src/styleTree/contactsPopover.ts diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs index 7637b51cc9..cd0e361f5b 100644 --- a/crates/contacts_status_item/src/contacts_popover.rs +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -1,4 +1,5 @@ -use gpui::{color::Color, elements::*, Entity, RenderContext, View, ViewContext}; +use gpui::{elements::*, Entity, RenderContext, View, ViewContext}; +use settings::Settings; pub enum Event { Deactivated, @@ -16,9 +17,10 @@ impl View for ContactsPopover { } fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + let theme = &cx.global::().theme.contacts_popover; Empty::new() .contained() - .with_background_color(Color::red()) + .with_background_color(theme.background) .boxed() } } diff --git a/crates/theme/src/theme.rs b/crates/theme/src/theme.rs index 9d90b44402..244a2e6aec 100644 --- a/crates/theme/src/theme.rs +++ b/crates/theme/src/theme.rs @@ -19,6 +19,7 @@ pub struct Theme { pub workspace: Workspace, pub context_menu: ContextMenu, pub chat_panel: ChatPanel, + pub contacts_popover: ContactsPopover, pub contacts_panel: ContactsPanel, pub contact_finder: ContactFinder, pub project_panel: ProjectPanel, @@ -301,6 +302,11 @@ pub struct CommandPalette { pub keystroke_spacing: f32, } +#[derive(Deserialize, Default)] +pub struct ContactsPopover { + pub background: Color, +} + #[derive(Deserialize, Default)] pub struct ContactsPanel { #[serde(flatten)] diff --git a/styles/src/styleTree/app.ts b/styles/src/styleTree/app.ts index 1345d4e855..a3ab4b654c 100644 --- a/styles/src/styleTree/app.ts +++ b/styles/src/styleTree/app.ts @@ -3,6 +3,7 @@ import chatPanel from "./chatPanel"; import { text } from "./components"; import contactFinder from "./contactFinder"; import contactsPanel from "./contactsPanel"; +import contactsPopover from "./contactsPopover"; import commandPalette from "./commandPalette"; import editor from "./editor"; import projectPanel from "./projectPanel"; @@ -34,6 +35,7 @@ export default function app(theme: Theme): Object { commandPalette: commandPalette(theme), projectPanel: projectPanel(theme), chatPanel: chatPanel(theme), + contactsPopover: contactsPopover(theme), contactsPanel: contactsPanel(theme), contactFinder: contactFinder(theme), search: search(theme), diff --git a/styles/src/styleTree/contactsPopover.ts b/styles/src/styleTree/contactsPopover.ts new file mode 100644 index 0000000000..d9f7900cce --- /dev/null +++ b/styles/src/styleTree/contactsPopover.ts @@ -0,0 +1,8 @@ +import Theme from "../themes/common/theme"; +import { backgroundColor } from "./components"; + +export default function workspace(theme: Theme) { + return { + background: backgroundColor(theme, 300), + } +} \ No newline at end of file From 147268157cf703f91eb703e96c3d2c6887293714 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 12:15:40 +0200 Subject: [PATCH 71/89] Animate popup window as if it were an `NSMenu` --- crates/gpui/src/platform/mac/window.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index f21428a1a8..107870d9e6 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -475,11 +475,20 @@ impl Window { native_window.center(); } - native_window.makeKeyAndOrderFront_(nil); match options.kind { WindowKind::Normal => native_window.setLevel_(NSNormalWindowLevel), - WindowKind::PopUp => native_window.setLevel_(NSPopUpWindowLevel), + WindowKind::PopUp => { + #[allow(non_upper_case_globals)] + const NSWindowAnimationBehaviorUtilityWindow: NSInteger = 4; + + native_window.setLevel_(NSPopUpWindowLevel); + let _: () = msg_send![ + native_window, + setAnimationBehavior: NSWindowAnimationBehaviorUtilityWindow + ]; + } } + native_window.makeKeyAndOrderFront_(nil); window.0.borrow().move_traffic_light(); pool.drain(); From 462e5852c2d581651264356de75235832c48426f Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 15:44:54 +0200 Subject: [PATCH 72/89] Use an "always active" tracking area to detect `mouseMoved` events This ensures that we can still receive mouse moved events (e.g. for setting the cursor style) for panels that float above other windows even if the application isn't active. --- crates/gpui/src/platform/mac/window.rs | 30 ++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 107870d9e6..a8e9853be6 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -57,11 +57,21 @@ static mut WINDOW_CLASS: *const Class = ptr::null(); static mut PANEL_CLASS: *const Class = ptr::null(); static mut VIEW_CLASS: *const Class = ptr::null(); +#[allow(non_upper_case_globals)] +const NSWindowStyleMaskNonactivatingPanel: NSWindowStyleMask = + unsafe { NSWindowStyleMask::from_bits_unchecked(1 << 7) }; #[allow(non_upper_case_globals)] const NSNormalWindowLevel: NSInteger = 0; - #[allow(non_upper_case_globals)] const NSPopUpWindowLevel: NSInteger = 101; +#[allow(non_upper_case_globals)] +const NSTrackingMouseMoved: NSUInteger = 0x02; +#[allow(non_upper_case_globals)] +const NSTrackingActiveAlways: NSUInteger = 0x80; +#[allow(non_upper_case_globals)] +const NSTrackingInVisibleRect: NSUInteger = 0x200; +#[allow(non_upper_case_globals)] +const NSWindowAnimationBehaviorUtilityWindow: NSInteger = 4; #[repr(C)] #[derive(Copy, Clone, Debug)] @@ -362,10 +372,6 @@ impl Window { let native_window: id = match options.kind { WindowKind::Normal => msg_send![WINDOW_CLASS, alloc], WindowKind::PopUp => { - #[allow(non_upper_case_globals)] - const NSWindowStyleMaskNonactivatingPanel: NSWindowStyleMask = - unsafe { NSWindowStyleMask::from_bits_unchecked(1 << 7) }; - style_mask |= NSWindowStyleMaskNonactivatingPanel; msg_send![PANEL_CLASS, alloc] } @@ -452,7 +458,16 @@ impl Window { native_window.setTitlebarAppearsTransparent_(YES); } - native_window.setAcceptsMouseMovedEvents_(YES); + let tracking_area: id = msg_send![class!(NSTrackingArea), alloc]; + let rect = NSRect::new(NSPoint::new(0., 0.), NSSize::new(0., 0.)); + let _: () = msg_send![ + tracking_area, + initWithRect: rect + options: NSTrackingMouseMoved | NSTrackingActiveAlways | NSTrackingInVisibleRect + owner: native_view + userInfo: nil + ]; + let _: () = msg_send![native_view, addTrackingArea: tracking_area]; native_view.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); native_view.setWantsBestResolutionOpenGLSurface_(YES); @@ -478,9 +493,6 @@ impl Window { match options.kind { WindowKind::Normal => native_window.setLevel_(NSNormalWindowLevel), WindowKind::PopUp => { - #[allow(non_upper_case_globals)] - const NSWindowAnimationBehaviorUtilityWindow: NSInteger = 4; - native_window.setLevel_(NSPopUpWindowLevel); let _: () = msg_send![ native_window, From ea00a000281387df8962ab45114a0fb01dac707b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 15:56:51 +0200 Subject: [PATCH 73/89] Start showing a filter query in contacts popover --- .../src/contacts_popover.rs | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/crates/contacts_status_item/src/contacts_popover.rs b/crates/contacts_status_item/src/contacts_popover.rs index cd0e361f5b..2998d74ed8 100644 --- a/crates/contacts_status_item/src/contacts_popover.rs +++ b/crates/contacts_status_item/src/contacts_popover.rs @@ -1,11 +1,14 @@ -use gpui::{elements::*, Entity, RenderContext, View, ViewContext}; +use editor::Editor; +use gpui::{elements::*, Entity, RenderContext, View, ViewContext, ViewHandle}; use settings::Settings; pub enum Event { Deactivated, } -pub struct ContactsPopover; +pub struct ContactsPopover { + filter_editor: ViewHandle, +} impl Entity for ContactsPopover { type Event = Event; @@ -18,9 +21,50 @@ impl View for ContactsPopover { fn render(&mut self, cx: &mut RenderContext) -> ElementBox { let theme = &cx.global::().theme.contacts_popover; - Empty::new() + + Flex::row() + .with_child( + ChildView::new(self.filter_editor.clone()) + .contained() + .with_style( + cx.global::() + .theme + .contacts_panel + .user_query_editor + .container, + ) + .flex(1., true) + .boxed(), + ) + // .with_child( + // MouseEventHandler::::new(0, cx, |_, _| { + // Svg::new("icons/user_plus_16.svg") + // .with_color(theme.add_contact_button.color) + // .constrained() + // .with_height(16.) + // .contained() + // .with_style(theme.add_contact_button.container) + // .aligned() + // .boxed() + // }) + // .with_cursor_style(CursorStyle::PointingHand) + // .on_click(MouseButton::Left, |_, cx| { + // cx.dispatch_action(contact_finder::Toggle) + // }) + // .boxed(), + // ) + .constrained() + .with_height( + cx.global::() + .theme + .contacts_panel + .user_query_editor_height, + ) + .aligned() + .top() .contained() .with_background_color(theme.background) + .with_uniform_padding(4.) .boxed() } } @@ -29,7 +73,17 @@ impl ContactsPopover { pub fn new(cx: &mut ViewContext) -> Self { cx.observe_window_activation(Self::window_activation_changed) .detach(); - Self + + let filter_editor = cx.add_view(|cx| { + let mut editor = Editor::single_line( + Some(|theme| theme.contacts_panel.user_query_editor.clone()), + cx, + ); + editor.set_placeholder_text("Filter contacts", cx); + editor + }); + + Self { filter_editor } } fn window_activation_changed(&mut self, is_active: bool, cx: &mut ViewContext) { From 66d13cf42c0178b30e2d0d28de4cf19b6e079c84 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 15:57:02 +0200 Subject: [PATCH 74/89] Query `isKeyWindow` on `windowDidBecomeKey` or `windowDidResignKey` Before we were assuming that receiving a callback meant that the window was in that "key" state accordingly, but with popups that's not always the case. In particular, there was a bug that caused an unrelated window to receive `windowDidBecomeKey` when making an `NSPanel` the key window. --- crates/gpui/src/platform/mac/window.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index a8e9853be6..b0002709e0 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1070,17 +1070,12 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { } } -extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { - let is_active = if selector == sel!(windowDidBecomeKey:) { - true - } else if selector == sel!(windowDidResignKey:) { - false - } else { - unreachable!(); - }; - +extern "C" fn window_did_change_key_status(this: &Object, _: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; - let executor = window_state.as_ref().borrow().executor.clone(); + let window_state_borrow = window_state.borrow(); + let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() }; + let executor = window_state_borrow.executor.clone(); + drop(window_state_borrow); executor .spawn(async move { let mut window_state_borrow = window_state.as_ref().borrow_mut(); From 5898fa61fb711571bb626798afce1923a937702b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 16:03:38 +0200 Subject: [PATCH 75/89] Temporarily disable status item so that we can merge to `main` --- crates/zed/src/main.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index f9e04782ab..3bfd5e6e1a 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -14,7 +14,6 @@ use client::{ http::{self, HttpClient}, UserStore, ZED_SECRET_CLIENT_TOKEN, }; -use contacts_status_item::ContactsStatusItem; use fs::OpenOptions; use futures::{ channel::{mpsc, oneshot}, @@ -89,8 +88,6 @@ fn main() { }); app.run(move |cx| { - cx.add_status_bar_item(|_| ContactsStatusItem::new()); - let client = client::Client::new(http.clone()); let mut languages = LanguageRegistry::new(login_shell_env_loaded); languages.set_language_server_download_dir(zed::paths::LANGUAGES_DIR.clone()); @@ -106,7 +103,6 @@ fn main() { watch_settings_file(default_settings, settings_file, themes.clone(), cx); watch_keymap_file(keymap_file, cx); - contacts_status_item::init(cx); context_menu::init(cx); project::Project::init(&client); client::Channel::init(&client); From cc316423ca3385aaba98578a88af67598076dccb Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 16:07:13 +0200 Subject: [PATCH 76/89] :lipstick: --- styles/src/styleTree/contactsPopover.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/styles/src/styleTree/contactsPopover.ts b/styles/src/styleTree/contactsPopover.ts index d9f7900cce..e9de5dddaf 100644 --- a/styles/src/styleTree/contactsPopover.ts +++ b/styles/src/styleTree/contactsPopover.ts @@ -5,4 +5,4 @@ export default function workspace(theme: Theme) { return { background: backgroundColor(theme, 300), } -} \ No newline at end of file +} From 3ec3f838dbf02e9d99dbda06160bc8c99239c147 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 15 Sep 2022 16:24:02 +0200 Subject: [PATCH 77/89] Autorelease `NSTrackingArea` to avoid leaking it --- crates/gpui/src/platform/mac/window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index b0002709e0..8a140a2361 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -467,7 +467,7 @@ impl Window { owner: native_view userInfo: nil ]; - let _: () = msg_send![native_view, addTrackingArea: tracking_area]; + let _: () = msg_send![native_view, addTrackingArea: tracking_area.autorelease()]; native_view.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable); native_view.setWantsBestResolutionOpenGLSurface_(YES); From 3163366a106120337455dc1ee17466403757dc20 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 09:54:48 +0200 Subject: [PATCH 78/89] Inline empty `NSTrackingArea` rect --- crates/gpui/src/platform/mac/window.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 8a140a2361..defaa612f4 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -459,10 +459,9 @@ impl Window { } let tracking_area: id = msg_send![class!(NSTrackingArea), alloc]; - let rect = NSRect::new(NSPoint::new(0., 0.), NSSize::new(0., 0.)); let _: () = msg_send![ tracking_area, - initWithRect: rect + initWithRect: NSRect::new(NSPoint::new(0., 0.), NSSize::new(0., 0.)) options: NSTrackingMouseMoved | NSTrackingActiveAlways | NSTrackingInVisibleRect owner: native_view userInfo: nil From 7a16e9c04878672bd16151d3c724d075403415d7 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 10:48:20 +0200 Subject: [PATCH 79/89] Allow panels to appear on top of full-screen apps --- crates/gpui/src/platform/mac/window.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index defaa612f4..5ecebc6f39 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -18,7 +18,8 @@ use block::ConcreteBlock; use cocoa::{ appkit::{ CGPoint, NSApplication, NSBackingStoreBuffered, NSScreen, NSView, NSViewHeightSizable, - NSViewWidthSizable, NSWindow, NSWindowButton, NSWindowStyleMask, + NSViewWidthSizable, NSWindow, NSWindowButton, NSWindowCollectionBehavior, + NSWindowStyleMask, }, base::{id, nil}, foundation::{ @@ -497,6 +498,10 @@ impl Window { native_window, setAnimationBehavior: NSWindowAnimationBehaviorUtilityWindow ]; + native_window.setCollectionBehavior_( + NSWindowCollectionBehavior::NSWindowCollectionBehaviorCanJoinAllSpaces | + NSWindowCollectionBehavior::NSWindowCollectionBehaviorFullScreenAuxiliary + ); } } native_window.makeKeyAndOrderFront_(nil); From b6ff8e77499edf36f7ef378fa6ebeb2b9b47ad8b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 10:55:15 +0200 Subject: [PATCH 80/89] Introduce workaround for spurious `windowDidBecomeKey` event --- crates/gpui/src/platform/mac/window.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 5ecebc6f39..375a271eac 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1074,10 +1074,29 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { } } -extern "C" fn window_did_change_key_status(this: &Object, _: Sel, _: id) { +extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.borrow(); let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() }; + + // When opening a pop-up while the application isn't active, Cocoa sends a spurious + // `windowDidBecomeKey` message to the previous key window even though that window + // isn't actually key. This causes a bug if the application is later activated while + // the pop-up is still open, making it impossible to activate the previous key window + // even if the pop-up gets closed. The only way to activate it again is to de-activate + // the app and re-activate it, which is a pretty bad UX. + // The following code detects the spurious event and invokes `resignKeyWindow`: + // in theory, we're not supposed to invoke this method manually but it balances out + // the spurious `becomeKeyWindow` event and helps us work around that bug. + if selector == sel!(windowDidBecomeKey:) { + if !is_active { + unsafe { + let _: () = msg_send![window_state_borrow.native_window, resignKeyWindow]; + return; + } + } + } + let executor = window_state_borrow.executor.clone(); drop(window_state_borrow); executor From 80d7df7664fbafb0220cd0897480477073170c37 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:01:36 +0200 Subject: [PATCH 81/89] Use an older version of the swift toolchain --- crates/live_kit/LiveKitBridge/Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift index fe71558804..76e528bda9 100644 --- a/crates/live_kit/LiveKitBridge/Package.swift +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.6 +// swift-tools-version: 5.5 import PackageDescription From 582ca666d013b5bf80f8dc12cf30c1f926c44a9e Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:02:55 +0200 Subject: [PATCH 82/89] Don't assume `BOOL` is a boolean --- crates/gpui/src/platform/mac/status_item.rs | 2 +- crates/gpui/src/platform/mac/window.rs | 4 ++-- .../LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/gpui/src/platform/mac/status_item.rs b/crates/gpui/src/platform/mac/status_item.rs index 404b5e2ef4..33feb4808f 100644 --- a/crates/gpui/src/platform/mac/status_item.rs +++ b/crates/gpui/src/platform/mac/status_item.rs @@ -144,7 +144,7 @@ impl StatusItem { Weak::into_raw(Rc::downgrade(&state)) as *const c_void, ); native_view.setWantsBestResolutionOpenGLSurface_(YES); - native_view.setWantsLayer(true); + native_view.setWantsLayer(YES); let _: () = msg_send![ native_view, setLayerContentsRedrawPolicy: NSViewLayerContentsRedrawDuringViewResize diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index 375a271eac..be51b67560 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -450,7 +450,7 @@ impl Window { native_window.setTitle_(NSString::alloc(nil).init_str(title)); } - native_window.setMovable_(options.is_movable); + native_window.setMovable_(options.is_movable as BOOL); if options .titlebar @@ -1077,7 +1077,7 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.borrow(); - let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() }; + let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() as bool }; // When opening a pop-up while the application isn't active, Cocoa sends a spurious // `windowDidBecomeKey` message to the previous key window even though that window diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index fb1eb9a79e..f59b829203 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -10,7 +10,7 @@ class LKRoomDelegate: RoomDelegate { self.data = data self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack } - + func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque()) From a0e2b7a6e9ebab3f89769c801e6f64412702eb74 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:08:19 +0200 Subject: [PATCH 83/89] Update `Package.resolved` --- .../live_kit/LiveKitBridge/Package.resolved | 95 ++++++++++--------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved index e95cffe979..b19e2980a4 100644 --- a/crates/live_kit/LiveKitBridge/Package.resolved +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -1,49 +1,52 @@ { - "pins" : [ - { - "identity" : "client-sdk-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/livekit/client-sdk-swift.git", - "state" : { - "revision" : "5cc3c001779ab147199ce3ea0dce465b846368b4" + "object": { + "pins": [ + { + "package": "LiveKit", + "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", + "state": { + "branch": null, + "revision": "5cc3c001779ab147199ce3ea0dce465b846368b4", + "version": null + } + }, + { + "package": "Promises", + "repositoryURL": "https://github.com/google/promises.git", + "state": { + "branch": null, + "revision": "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", + "version": "2.1.1" + } + }, + { + "package": "WebRTC", + "repositoryURL": "https://github.com/webrtc-sdk/Specs.git", + "state": { + "branch": null, + "revision": "5225f2de4b6d0098803b3a0e55b255a41f293dad", + "version": "104.5112.2" + } + }, + { + "package": "swift-log", + "repositoryURL": "https://github.com/apple/swift-log.git", + "state": { + "branch": null, + "revision": "6fe203dc33195667ce1759bf0182975e4653ba1c", + "version": "1.4.4" + } + }, + { + "package": "SwiftProtobuf", + "repositoryURL": "https://github.com/apple/swift-protobuf.git", + "state": { + "branch": null, + "revision": "b8230909dedc640294d7324d37f4c91ad3dcf177", + "version": "1.20.1" + } } - }, - { - "identity" : "promises", - "kind" : "remoteSourceControl", - "location" : "https://github.com/google/promises.git", - "state" : { - "revision" : "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb", - "version" : "2.1.1" - } - }, - { - "identity" : "specs", - "kind" : "remoteSourceControl", - "location" : "https://github.com/webrtc-sdk/Specs.git", - "state" : { - "revision" : "5225f2de4b6d0098803b3a0e55b255a41f293dad", - "version" : "104.5112.2" - } - }, - { - "identity" : "swift-log", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-log.git", - "state" : { - "revision" : "6fe203dc33195667ce1759bf0182975e4653ba1c", - "version" : "1.4.4" - } - }, - { - "identity" : "swift-protobuf", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-protobuf.git", - "state" : { - "revision" : "b8230909dedc640294d7324d37f4c91ad3dcf177", - "version" : "1.20.1" - } - } - ], - "version" : 2 + ] + }, + "version": 1 } From 458a6a73101d06798567c90e0c41cea38c81051e Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:09:30 +0200 Subject: [PATCH 84/89] :lipstick: --- crates/gpui/src/platform/mac/window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpui/src/platform/mac/window.rs b/crates/gpui/src/platform/mac/window.rs index be51b67560..1ce42db9ed 100644 --- a/crates/gpui/src/platform/mac/window.rs +++ b/crates/gpui/src/platform/mac/window.rs @@ -1077,7 +1077,7 @@ fn window_fullscreen_changed(this: &Object, is_fullscreen: bool) { extern "C" fn window_did_change_key_status(this: &Object, selector: Sel, _: id) { let window_state = unsafe { get_window_state(this) }; let window_state_borrow = window_state.borrow(); - let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() as bool }; + let is_active = unsafe { window_state_borrow.native_window.isKeyWindow() == YES }; // When opening a pop-up while the application isn't active, Cocoa sends a spurious // `windowDidBecomeKey` message to the previous key window even though that window From b76f3372a34c3ffd777065b8c3dab777662e4d5e Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:22:17 +0200 Subject: [PATCH 85/89] Target a triple explicitly when building Swift bridge --- crates/live_kit/build.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index 4f23cc4e92..bd72460017 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -1,6 +1,8 @@ use serde::Deserialize; use std::{env, path::PathBuf, process::Command}; +const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SwiftTargetInfo { @@ -42,9 +44,15 @@ fn build_bridge(swift_target: &SwiftTarget) { "cargo:rerun-if-changed={}/Package.swift", SWIFT_PACKAGE_NAME ); + println!( + "cargo:rerun-if-changed={}/Package.resolved", + SWIFT_PACKAGE_NAME + ); let swift_package_root = swift_package_root(); if !Command::new("swift") - .args(&["build", "-c", &env::var("PROFILE").unwrap()]) + .arg("build") + .args(&["--configuration", &env::var("PROFILE").unwrap()]) + .args(&["--triple", &swift_target.target.triple]) .current_dir(&swift_package_root) .status() .unwrap() @@ -116,8 +124,6 @@ fn get_swift_target() -> SwiftTarget { serde_json::from_slice(&swift_target_info_str).unwrap() } -const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; - fn swift_package_root() -> PathBuf { env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) } From 8bd059a293d111a755c563f0475e3cc818fa6d85 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:33:49 +0200 Subject: [PATCH 86/89] Try using the unversioned triple when compiling live_kit --- crates/live_kit/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index bd72460017..f11e4a9bac 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -52,7 +52,7 @@ fn build_bridge(swift_target: &SwiftTarget) { if !Command::new("swift") .arg("build") .args(&["--configuration", &env::var("PROFILE").unwrap()]) - .args(&["--triple", &swift_target.target.triple]) + .args(&["--triple", &swift_target.target.unversioned_triple]) .current_dir(&swift_package_root) .status() .unwrap() From 32c65cfb2ade5343613a86fb7c69faedce076fef Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:35:33 +0200 Subject: [PATCH 87/89] Specify macOS target version manually --- crates/live_kit/build.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index f11e4a9bac..df2127cc3f 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -52,7 +52,13 @@ fn build_bridge(swift_target: &SwiftTarget) { if !Command::new("swift") .arg("build") .args(&["--configuration", &env::var("PROFILE").unwrap()]) - .args(&["--triple", &swift_target.target.unversioned_triple]) + .args(&[ + "--triple", + &format!( + "{}{}", + swift_target.target.unversioned_triple, MACOS_TARGET_VERSION + ), + ]) .current_dir(&swift_package_root) .status() .unwrap() From 20778a0694e786a1e7b0852916fc229264327dbf Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 11:47:17 +0200 Subject: [PATCH 88/89] Allow using `live_kit` with macOS >= 10.15 --- crates/live_kit/build.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index df2127cc3f..e34b4e2a3e 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -28,7 +28,7 @@ pub struct SwiftTarget { pub paths: SwiftPaths, } -const MACOS_TARGET_VERSION: &str = "12"; +const MACOS_TARGET_VERSION: &str = "10.15"; fn main() { let swift_target = get_swift_target(); @@ -52,13 +52,7 @@ fn build_bridge(swift_target: &SwiftTarget) { if !Command::new("swift") .arg("build") .args(&["--configuration", &env::var("PROFILE").unwrap()]) - .args(&[ - "--triple", - &format!( - "{}{}", - swift_target.target.unversioned_triple, MACOS_TARGET_VERSION - ), - ]) + .args(&["--triple", &swift_target.target.triple]) .current_dir(&swift_package_root) .status() .unwrap() @@ -78,10 +72,6 @@ fn build_bridge(swift_target: &SwiftTarget) { } fn link_swift_stdlib(swift_target: &SwiftTarget) { - if swift_target.target.libraries_require_rpath { - panic!("Libraries require RPath! Change minimum MacOS value to fix.") - } - swift_target .paths .runtime_library_paths From e07f4f3f53320a31a53813cb8f487e71cfb227d8 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 16 Sep 2022 18:09:11 +0200 Subject: [PATCH 89/89] Copy WebRTC.framework in the `deps` directory Also, define the `rpath` on `live_kit` to avoid errors when running tests. --- crates/live_kit/build.rs | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/crates/live_kit/build.rs b/crates/live_kit/build.rs index e34b4e2a3e..79d7d84cdd 100644 --- a/crates/live_kit/build.rs +++ b/crates/live_kit/build.rs @@ -1,5 +1,9 @@ use serde::Deserialize; -use std::{env, path::PathBuf, process::Command}; +use std::{ + env, + path::{Path, PathBuf}, + process::Command, +}; const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge"; @@ -88,20 +92,16 @@ fn link_webrtc_framework(swift_target: &SwiftTarget) { "cargo:rustc-link-search=framework={}", swift_out_dir_path.display() ); + // Find WebRTC.framework as a sibling of the executable when running tests. + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); let source_path = swift_out_dir_path.join("WebRTC.framework"); - let target_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); - assert!( - Command::new("cp") - .arg("-r") - .args(&[&source_path, &target_path]) - .status() - .unwrap() - .success(), - "could not copy WebRTC.framework from {:?} to {:?}", - source_path, - target_path - ); + let deps_dir_path = + PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../deps/WebRTC.framework"); + let target_dir_path = + PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); + copy_dir(&source_path, &deps_dir_path); + copy_dir(&source_path, &target_dir_path); } fn get_swift_target() -> SwiftTarget { @@ -124,6 +124,20 @@ fn swift_package_root() -> PathBuf { env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) } +fn copy_dir(source: &Path, destination: &Path) { + assert!( + Command::new("cp") + .arg("-r") + .args(&[source, destination]) + .status() + .unwrap() + .success(), + "could not copy {:?} to {:?}", + source, + destination + ); +} + impl SwiftTarget { fn out_dir_path(&self) -> PathBuf { swift_package_root()