oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Audio fixes and mic denoise" (#38509)

Browse Source 
Reverts zed-industries/zed#38493

Release Notes:

- N/A
This commit is contained in:
David Kleingeld
2025-09-19 10:33:38 -04:00
committed by GitHub
parent aa5b99dc11
commit 2e97ef32c4
13 changed files with 123 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/livekit_client/src/livekit_client.rs
+4 -16
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use anyhow::{Context as _, Result, anyhow};
use anyhow::{Context as _, Result};
use audio::AudioSettings;
use collections::HashMap;
use futures::{SinkExt, channel::mpsc};
@@ -12,10 +12,7 @@ use settings::Settings;
mod playback;
use crate::{
LocalTrack, Participant, RemoteTrack, RoomEvent, TrackPublication,
livekit_client::playback::Speaker,
};
use crate::{LocalTrack, Participant, RemoteTrack, RoomEvent, TrackPublication};
pub use playback::AudioStream;
pub(crate) use playback::{RemoteVideoFrame, play_remote_video_track};
@@ -135,20 +132,11 @@ impl Room {
track: &RemoteAudioTrack,
cx: &mut App,
) -> Result<playback::AudioStream> {
let speaker: Speaker =
serde_urlencoded::from_str(&track.0.name()).unwrap_or_else(|_| Speaker {
name: track.0.name(),
is_staff: false,
legacy_audio_compatible: true,
});
if AudioSettings::get_global(cx).rodio_audio {
info!("Using experimental.rodio_audio audio pipeline for output");
playback::play_remote_audio_track(&track.0, speaker, cx)
} else if speaker.legacy_audio_compatible {
Ok(self.playback.play_remote_audio_track(&track.0))
playback::play_remote_audio_track(&track.0, cx)
} else {
Err(anyhow!("Client version too old to play audio in call"))
Ok(self.playback.play_remote_audio_track(&track.0))
}
}
}
crates/livekit_client/src/livekit_client/playback.rs
+29 -60
View File
@@ -1,6 +1,6 @@
use anyhow::{Context as _, Result};
use audio::{AudioSettings, CHANNEL_COUNT, LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE, SAMPLE_RATE};
use audio::{AudioSettings, CHANNEL_COUNT, SAMPLE_RATE};
use cpal::traits::{DeviceTrait, StreamTrait as _};
use futures::channel::mpsc::UnboundedSender;
use futures::{Stream, StreamExt as _};
@@ -43,17 +43,12 @@ pub(crate) struct AudioStack {
pub(crate) fn play_remote_audio_track(
track: &livekit::track::RemoteAudioTrack,
speaker: Speaker,
cx: &mut gpui::App,
) -> Result<AudioStream> {
let stream = source::LiveKitStream::new(
cx.background_executor(),
track,
speaker.legacy_audio_compatible,
);
let stop_handle = Arc::new(AtomicBool::new(false));
let stop_handle_clone = stop_handle.clone();
let stream = source::LiveKitStream::new(cx.background_executor(), track);
let stream = stream
.stoppable()
.periodic_access(Duration::from_millis(50), move |s| {
@@ -62,6 +57,10 @@ pub(crate) fn play_remote_audio_track(
}
});
let speaker: Speaker = serde_urlencoded::from_str(&track.name()).unwrap_or_else(|_| Speaker {
name: track.name(),
is_staff: false,
});
audio::Audio::play_voip_stream(stream, speaker.name, speaker.is_staff, cx)
.context("Could not play audio")?;
@@ -153,32 +152,17 @@ impl AudioStack {
is_staff: bool,
cx: &AsyncApp,
) -> Result<(crate::LocalAudioTrack, AudioStream)> {
let legacy_audio_compatible =
AudioSettings::try_read_global(cx, |setting| setting.legacy_audio_compatible)
.unwrap_or_default();
let source = if legacy_audio_compatible {
NativeAudioSource::new(
// n.b. this struct's options are always ignored, noise cancellation is provided by apm.
AudioSourceOptions::default(),
LEGACY_SAMPLE_RATE.get(),
LEGACY_CHANNEL_COUNT.get().into(),
10,
)
} else {
NativeAudioSource::new(
// n.b. this struct's options are always ignored, noise cancellation is provided by apm.
AudioSourceOptions::default(),
SAMPLE_RATE.get(),
CHANNEL_COUNT.get().into(),
10,
)
};
let source = NativeAudioSource::new(
// n.b. this struct's options are always ignored, noise cancellation is provided by apm.
AudioSourceOptions::default(),
SAMPLE_RATE.get(),
CHANNEL_COUNT.get().into(),
10,
);
let track_name = serde_urlencoded::to_string(Speaker {
name: user_name,
is_staff,
legacy_audio_compatible,
})
.context("Could not encode user information in track name")?;
@@ -202,32 +186,22 @@ impl AudioStack {
let capture_task = if rodio_pipeline {
info!("Using experimental.rodio_audio audio pipeline");
let voip_parts = audio::VoipParts::new(cx)?;
// Audio needs to run real-time and should never be paused. That is
// why we are using a normal std::thread and not a background task
// Audio needs to run real-time and should never be paused. That is why we are using a
// normal std::thread and not a background task
thread::Builder::new()
.name("MicrophoneToLivekit".to_string())
.name("AudioCapture".to_string())
.spawn(move || {
// microphone is non send on mac
let microphone = match audio::Audio::open_microphone(voip_parts) {
Ok(m) => m,
Err(e) => {
log::error!("Could not open microphone: {e}");
return;
}
};
let microphone = audio::Audio::open_microphone(voip_parts)?;
send_to_livekit(frame_tx, microphone);
Ok::<(), anyhow::Error>(())
})
.expect("should be able to spawn threads");
.unwrap();
Task::ready(Ok(()))
} else {
self.executor.spawn(async move {
Self::capture_input(
apm,
frame_tx,
LEGACY_SAMPLE_RATE.get(),
LEGACY_CHANNEL_COUNT.get().into(),
)
.await
Self::capture_input(apm, frame_tx, SAMPLE_RATE.get(), CHANNEL_COUNT.get().into())
.await
})
};
@@ -415,30 +389,25 @@ impl AudioStack {
}
#[derive(Serialize, Deserialize)]
pub struct Speaker {
pub name: String,
pub is_staff: bool,
pub legacy_audio_compatible: bool,
struct Speaker {
name: String,
is_staff: bool,
}
fn send_to_livekit(frame_tx: UnboundedSender<AudioFrame<'static>>, mut microphone: impl Source) {
use cpal::Sample;
let sample_rate = microphone.sample_rate().get();
let num_channels = microphone.channels().get() as u32;
let buffer_size = sample_rate / 100 * num_channels;
loop {
let sampled: Vec<_> = microphone
.by_ref()
.take(buffer_size as usize)
.take(audio::BUFFER_SIZE)
.map(|s| s.to_sample())
.collect();
if frame_tx
.unbounded_send(AudioFrame {
sample_rate,
num_channels,
samples_per_channel: sampled.len() as u32 / num_channels,
sample_rate: SAMPLE_RATE.get(),
num_channels: CHANNEL_COUNT.get() as u32,
samples_per_channel: sampled.len() as u32 / CHANNEL_COUNT.get() as u32,
data: Cow::Owned(sampled),
})
.is_err()
crates/livekit_client/src/livekit_client/playback/source.rs
+15 -25
View File
@@ -3,19 +3,17 @@ use std::num::NonZero;
use futures::StreamExt;
use libwebrtc::{audio_stream::native::NativeAudioStream, prelude::AudioFrame};
use livekit::track::RemoteAudioTrack;
use rodio::{
ChannelCount, SampleRate, Source, buffer::SamplesBuffer, conversions::SampleTypeConverter,
};
use rodio::{Source, buffer::SamplesBuffer, conversions::SampleTypeConverter, nz};
use audio::{CHANNEL_COUNT, LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE, SAMPLE_RATE};
use audio::{CHANNEL_COUNT, SAMPLE_RATE};
fn frame_to_samplesbuffer(frame: AudioFrame) -> SamplesBuffer {
let samples = frame.data.iter().copied();
let samples = SampleTypeConverter::<_, _>::new(samples);
let samples: Vec<f32> = samples.collect();
SamplesBuffer::new(
NonZero::new(frame.num_channels as u16).expect("zero channels is nonsense"),
NonZero::new(frame.sample_rate).expect("samplerate zero is nonsense"),
nz!(2), // frame always has two channels
NonZero::new(frame.sample_rate).expect("audio frame sample rate is nonzero"),
samples,
)
}
@@ -24,26 +22,14 @@ pub struct LiveKitStream {
// shared_buffer: SharedBuffer,
inner: rodio::queue::SourcesQueueOutput,
_receiver_task: gpui::Task<()>,
channel_count: ChannelCount,
sample_rate: SampleRate,
}
impl LiveKitStream {
pub fn new(
executor: &gpui::BackgroundExecutor,
track: &RemoteAudioTrack,
legacy: bool,
) -> Self {
let (channel_count, sample_rate) = if legacy {
(LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE)
} else {
(CHANNEL_COUNT, SAMPLE_RATE)
};
pub fn new(executor: &gpui::BackgroundExecutor, track: &RemoteAudioTrack) -> Self {
let mut stream = NativeAudioStream::new(
track.rtc_track(),
sample_rate.get() as i32,
channel_count.get().into(),
SAMPLE_RATE.get() as i32,
CHANNEL_COUNT.get().into(),
);
let (queue_input, queue_output) = rodio::queue::queue(true);
// spawn rtc stream
@@ -59,8 +45,6 @@ impl LiveKitStream {
LiveKitStream {
_receiver_task: receiver_task,
inner: queue_output,
sample_rate,
channel_count,
}
}
}
@@ -79,11 +63,17 @@ impl Source for LiveKitStream {
}
fn channels(&self) -> rodio::ChannelCount {
self.channel_count
// This must be hardcoded because the playback source assumes constant
// sample rate and channel count. The queue upon which this is build
// will however report different counts and rates. Even though we put in
// only items with our (constant) CHANNEL_COUNT & SAMPLE_RATE this will
// play silence on one channel and at 44100 which is not what our
// constants are.
CHANNEL_COUNT
}
fn sample_rate(&self) -> rodio::SampleRate {
self.sample_rate
SAMPLE_RATE // see comment on channels
}
fn total_duration(&self) -> Option<std::time::Duration> {