oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix experimental audio, add denoise, auto volume.Prep migration (#38874)

Browse Source 
Uses the previously merged denoising crate (and fixes a bug in it that
snug in during refactoring) to add denoising to the microphone input. 

Adds automatic volume control for microphone and output.

Prepares for migrating to 16kHz SR mono:
The experimental audio path now picks the samplerate and channel count depending on a setting. It can handle incoming streams with both the current (future legacy) and new samplerate & channel count. These are url-encoded into the livekit track name

Release Notes:

- N/A
This commit is contained in:
David Kleingeld
2025-09-25 15:11:12 +02:00
committed by GitHub
parent 783ba389f7
commit 3c5e683fbe
15 changed files with 498 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/audio/src/audio.rs
+54 -22
View File
@@ -9,7 +9,7 @@ mod non_windows_and_freebsd_deps {
pub(super) use log::info;
pub(super) use parking_lot::Mutex;
pub(super) use rodio::cpal::Sample;
pub(super) use rodio::source::{LimitSettings, UniformSourceIterator};
pub(super) use rodio::source::LimitSettings;
pub(super) use std::sync::Arc;
}
@@ -31,18 +31,20 @@ pub use rodio_ext::RodioExt;
use crate::audio_settings::LIVE_SETTINGS;
// NOTE: We used to use WebRTC's mixer which only supported
// 16kHz, 32kHz and 48kHz. As 48 is the most common "next step up"
// for audio output devices like speakers/bluetooth, we just hard-code
// this; and downsample when we need to.
// We are migrating to 16kHz sample rate from 48kHz. In the future
// once we are reasonably sure most users have upgraded we will
// remove the LEGACY parameters.
//
// Since most noise cancelling requires 16kHz we will move to
// that in the future.
pub const SAMPLE_RATE: NonZero<u32> = nz!(48000);
pub const CHANNEL_COUNT: NonZero<u16> = nz!(2);
// We migrate to 16kHz because it is sufficient for speech and required
// by the denoiser and future Speech to Text layers.
pub const SAMPLE_RATE: NonZero<u32> = nz!(16000);
pub const CHANNEL_COUNT: NonZero<u16> = nz!(1);
pub const BUFFER_SIZE: usize = // echo canceller and livekit want 10ms of audio
(SAMPLE_RATE.get() as usize / 100) * CHANNEL_COUNT.get() as usize;
pub const LEGACY_SAMPLE_RATE: NonZero<u32> = nz!(48000);
pub const LEGACY_CHANNEL_COUNT: NonZero<u16> = nz!(2);
pub const REPLAY_DURATION: Duration = Duration::from_secs(30);
pub fn init(cx: &mut App) {
@@ -106,11 +108,16 @@ impl Global for Audio {}
impl Audio {
fn ensure_output_exists(&mut self) -> Result<&Mixer> {
#[cfg(debug_assertions)]
log::warn!(
"Audio does not sound correct without optimizations. Use a release build to debug audio issues"
);
if self.output_handle.is_none() {
self.output_handle = Some(
OutputStreamBuilder::open_default_stream()
.context("Could not open default output stream")?,
);
let output_handle = OutputStreamBuilder::open_default_stream()
.context("Could not open default output stream")?;
info!("Output stream: {:?}", output_handle);
self.output_handle = Some(output_handle);
if let Some(output_handle) = &self.output_handle {
let (mixer, source) = rodio::mixer::mixer(CHANNEL_COUNT, SAMPLE_RATE);
// or the mixer will end immediately as its empty.
@@ -160,13 +167,20 @@ impl Audio {
let stream = rodio::microphone::MicrophoneBuilder::new()
.default_device()?
.default_config()?
.prefer_sample_rates([SAMPLE_RATE, SAMPLE_RATE.saturating_mul(nz!(2))])
// .prefer_channel_counts([nz!(1), nz!(2)])
.prefer_sample_rates([
SAMPLE_RATE, // sample rates trivially resamplable to `SAMPLE_RATE`
SAMPLE_RATE.saturating_mul(nz!(2)),
SAMPLE_RATE.saturating_mul(nz!(3)),
SAMPLE_RATE.saturating_mul(nz!(4)),
])
.prefer_channel_counts([nz!(1), nz!(2), nz!(3), nz!(4)])
.prefer_buffer_sizes(512..)
.open_stream()?;
info!("Opened microphone: {:?}", stream.config());
let (replay, stream) = UniformSourceIterator::new(stream, CHANNEL_COUNT, SAMPLE_RATE)
let stream = stream
.possibly_disconnected_channels_to_mono()
.constant_samplerate(SAMPLE_RATE)
.limit(LimitSettings::live_performance())
.process_buffer::<BUFFER_SIZE, _>(move |buffer| {
let mut int_buffer: [i16; _] = buffer.map(|s| s.to_sample());
@@ -187,15 +201,27 @@ impl Audio {
}
}
})
.automatic_gain_control(1.0, 4.0, 0.0, 5.0)
.denoise()
.context("Could not set up denoiser")?
.automatic_gain_control(0.90, 1.0, 0.0, 5.0)
.periodic_access(Duration::from_millis(100), move |agc_source| {
agc_source.set_enabled(LIVE_SETTINGS.control_input_volume.load(Ordering::Relaxed));
})
.replayable(REPLAY_DURATION)?;
agc_source
.set_enabled(LIVE_SETTINGS.auto_microphone_volume.load(Ordering::Relaxed));
let denoise = agc_source.inner_mut();
denoise.set_enabled(LIVE_SETTINGS.denoise.load(Ordering::Relaxed));
});
let stream = if voip_parts.legacy_audio_compatible {
stream.constant_params(LEGACY_CHANNEL_COUNT, LEGACY_SAMPLE_RATE)
} else {
stream.constant_params(CHANNEL_COUNT, SAMPLE_RATE)
};
let (replay, stream) = stream.replayable(REPLAY_DURATION)?;
voip_parts
.replays
.add_voip_stream("local microphone".to_string(), replay);
Ok(stream)
}
@@ -206,9 +232,10 @@ impl Audio {
cx: &mut App,
) -> anyhow::Result<()> {
let (replay_source, source) = source
.automatic_gain_control(1.0, 4.0, 0.0, 5.0)
.constant_params(CHANNEL_COUNT, SAMPLE_RATE)
.automatic_gain_control(0.90, 1.0, 0.0, 5.0)
.periodic_access(Duration::from_millis(100), move |agc_source| {
agc_source.set_enabled(LIVE_SETTINGS.control_input_volume.load(Ordering::Relaxed));
agc_source.set_enabled(LIVE_SETTINGS.auto_speaker_volume.load(Ordering::Relaxed));
})
.replayable(REPLAY_DURATION)
.expect("REPLAY_DURATION is longer than 100ms");
@@ -269,6 +296,7 @@ impl Audio {
pub struct VoipParts {
echo_canceller: Arc<Mutex<apm::AudioProcessingModule>>,
replays: replays::Replays,
legacy_audio_compatible: bool,
}
#[cfg(not(any(all(target_os = "windows", target_env = "gnu"), target_os = "freebsd")))]
@@ -277,8 +305,12 @@ impl VoipParts {
let (apm, replays) = cx.try_read_default_global::<Audio, _>(|audio, _| {
(Arc::clone(&audio.echo_canceller), audio.replays.clone())
})?;
let legacy_audio_compatible =
AudioSettings::try_read_global(cx, |settings| settings.legacy_audio_compatible)
.unwrap_or(true);
Ok(Self {
legacy_audio_compatible,
echo_canceller: apm,
replays,
})
crates/audio/src/audio_settings.rs
+68 -27
View File
@@ -6,18 +6,38 @@ use settings::{Settings, SettingsStore};
#[derive(Clone, Debug)]
pub struct AudioSettings {
/// Opt into the new audio system.
///
/// You need to rejoin a call for this setting to apply
pub rodio_audio: bool, // default is false
/// Requires 'rodio_audio: true'
///
/// Use the new audio systems automatic gain control for your microphone.
/// This affects how loud you sound to others.
pub control_input_volume: bool,
/// Automatically increase or decrease you microphone's volume. This affects how
/// loud you sound to others.
///
/// Recommended: off (default)
/// Microphones are too quite in zed, until everyone is on experimental
/// audio and has auto speaker volume on this will make you very loud
/// compared to other speakers.
pub auto_microphone_volume: bool,
/// Requires 'rodio_audio: true'
///
/// Use the new audio systems automatic gain control on everyone in the
/// call. This makes call members who are too quite louder and those who are
/// too loud quieter. This only affects how things sound for you.
pub control_output_volume: bool,
/// Automatically increate or decrease the volume of other call members.
/// This only affects how things sound for you.
pub auto_speaker_volume: bool,
/// Requires 'rodio_audio: true'
///
/// Remove background noises. Works great for typing, cars, dogs, AC. Does
/// not work well on music.
pub denoise: bool,
/// Requires 'rodio_audio: true'
///
/// Use audio parameters compatible with the previous versions of
/// experimental audio and non-experimental audio. When this is false you
/// will sound strange to anyone not on the latest experimental audio. In
/// the future we will migrate by setting this to false
///
/// You need to rejoin a call for this setting to apply
pub legacy_audio_compatible: bool,
}
/// Configuration of audio in Zed
@@ -25,46 +45,66 @@ impl Settings for AudioSettings {
fn from_settings(content: &settings::SettingsContent, _cx: &mut App) -> Self {
let audio = &content.audio.as_ref().unwrap();
AudioSettings {
control_input_volume: audio.control_input_volume.unwrap(),
control_output_volume: audio.control_output_volume.unwrap(),
rodio_audio: audio.rodio_audio.unwrap(),
auto_microphone_volume: audio.auto_microphone_volume.unwrap(),
auto_speaker_volume: audio.auto_speaker_volume.unwrap(),
denoise: audio.denoise.unwrap(),
legacy_audio_compatible: audio.legacy_audio_compatible.unwrap(),
}
}
fn import_from_vscode(
_vscode: &settings::VsCodeSettings,
_current: &mut settings::SettingsContent,
) {
}
}
/// See docs on [LIVE_SETTINGS]
pub(crate) struct LiveSettings {
pub(crate) control_input_volume: AtomicBool,
pub(crate) control_output_volume: AtomicBool,
pub(crate) auto_microphone_volume: AtomicBool,
pub(crate) auto_speaker_volume: AtomicBool,
pub(crate) denoise: AtomicBool,
}
impl LiveSettings {
pub(crate) fn initialize(&self, cx: &mut App) {
cx.observe_global::<SettingsStore>(move |cx| {
LIVE_SETTINGS.control_input_volume.store(
AudioSettings::get_global(cx).control_input_volume,
LIVE_SETTINGS.auto_microphone_volume.store(
AudioSettings::get_global(cx).auto_microphone_volume,
Ordering::Relaxed,
);
LIVE_SETTINGS.control_output_volume.store(
AudioSettings::get_global(cx).control_output_volume,
LIVE_SETTINGS.auto_speaker_volume.store(
AudioSettings::get_global(cx).auto_speaker_volume,
Ordering::Relaxed,
);
let denoise_enabled = AudioSettings::get_global(cx).denoise;
#[cfg(debug_assertions)]
{
static DENOISE_WARNING_SEND: AtomicBool = AtomicBool::new(false);
if denoise_enabled && !DENOISE_WARNING_SEND.load(Ordering::Relaxed) {
DENOISE_WARNING_SEND.store(true, Ordering::Relaxed);
log::warn!("Denoise does not work on debug builds, not enabling")
}
}
#[cfg(not(debug_assertions))]
LIVE_SETTINGS
.denoise
.store(denoise_enabled, Ordering::Relaxed);
})
.detach();
let init_settings = AudioSettings::get_global(cx);
LIVE_SETTINGS
.control_input_volume
.store(init_settings.control_input_volume, Ordering::Relaxed);
.auto_microphone_volume
.store(init_settings.auto_microphone_volume, Ordering::Relaxed);
LIVE_SETTINGS
.control_output_volume
.store(init_settings.control_output_volume, Ordering::Relaxed);
.auto_speaker_volume
.store(init_settings.auto_speaker_volume, Ordering::Relaxed);
let denoise_enabled = AudioSettings::get_global(cx).denoise;
#[cfg(debug_assertions)]
if denoise_enabled {
log::warn!("Denoise does not work on debug builds, not enabling")
}
#[cfg(not(debug_assertions))]
LIVE_SETTINGS
.denoise
.store(denoise_enabled, Ordering::Relaxed);
}
}
@@ -73,6 +113,7 @@ impl LiveSettings {
/// real time and must each run in a dedicated OS thread, therefore we can not
/// use the background executor.
pub(crate) static LIVE_SETTINGS: LiveSettings = LiveSettings {
control_input_volume: AtomicBool::new(true),
control_output_volume: AtomicBool::new(true),
auto_microphone_volume: AtomicBool::new(true),
auto_speaker_volume: AtomicBool::new(true),
denoise: AtomicBool::new(true),
};
crates/audio/src/rodio_ext.rs
+166 -1
View File
@@ -1,4 +1,5 @@
use std::{
num::NonZero,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
@@ -7,12 +8,22 @@ use std::{
};
use crossbeam::queue::ArrayQueue;
use rodio::{ChannelCount, Sample, SampleRate, Source};
use denoise::{Denoiser, DenoiserError};
use log::warn;
use rodio::{
ChannelCount, Sample, SampleRate, Source, conversions::SampleRateConverter, nz,
source::UniformSourceIterator,
};
const MAX_CHANNELS: usize = 8;
#[derive(Debug, thiserror::Error)]
#[error("Replay duration is too short must be >= 100ms")]
pub struct ReplayDurationTooShort;
// These all require constant sources (so the span is infinitely long)
// this is not guaranteed by rodio however we know it to be true in all our
// applications. Rodio desperately needs a constant source concept.
pub trait RodioExt: Source + Sized {
fn process_buffer<const N: usize, F>(self, callback: F) -> ProcessBuffer<N, Self, F>
where
@@ -25,6 +36,14 @@ pub trait RodioExt: Source + Sized {
duration: Duration,
) -> Result<(Replay, Replayable<Self>), ReplayDurationTooShort>;
fn take_samples(self, n: usize) -> TakeSamples<Self>;
fn denoise(self) -> Result<Denoiser<Self>, DenoiserError>;
fn constant_params(
self,
channel_count: ChannelCount,
sample_rate: SampleRate,
) -> UniformSourceIterator<Self>;
fn constant_samplerate(self, sample_rate: SampleRate) -> ConstantSampleRate<Self>;
fn possibly_disconnected_channels_to_mono(self) -> ToMono<Self>;
}
impl<S: Source> RodioExt for S {
@@ -101,8 +120,149 @@ impl<S: Source> RodioExt for S {
left_to_take: n,
}
}
fn denoise(self) -> Result<Denoiser<Self>, DenoiserError> {
let res = Denoiser::try_new(self);
res
}
fn constant_params(
self,
channel_count: ChannelCount,
sample_rate: SampleRate,
) -> UniformSourceIterator<Self> {
UniformSourceIterator::new(self, channel_count, sample_rate)
}
fn constant_samplerate(self, sample_rate: SampleRate) -> ConstantSampleRate<Self> {
ConstantSampleRate::new(self, sample_rate)
}
fn possibly_disconnected_channels_to_mono(self) -> ToMono<Self> {
ToMono::new(self)
}
}
pub struct ConstantSampleRate<S: Source> {
inner: SampleRateConverter<S>,
channels: ChannelCount,
sample_rate: SampleRate,
}
impl<S: Source> ConstantSampleRate<S> {
fn new(source: S, target_rate: SampleRate) -> Self {
let input_sample_rate = source.sample_rate();
let channels = source.channels();
let inner = SampleRateConverter::new(source, input_sample_rate, target_rate, channels);
Self {
inner,
channels,
sample_rate: target_rate,
}
}
}
impl<S: Source> Iterator for ConstantSampleRate<S> {
type Item = rodio::Sample;
fn next(&mut self) -> Option<Self::Item> {
self.inner.next()
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
}
impl<S: Source> Source for ConstantSampleRate<S> {
fn current_span_len(&self) -> Option<usize> {
None
}
fn channels(&self) -> ChannelCount {
self.channels
}
fn sample_rate(&self) -> SampleRate {
self.sample_rate
}
fn total_duration(&self) -> Option<Duration> {
None // not supported (not used by us)
}
}
const TYPICAL_NOISE_FLOOR: Sample = 1e-3;
/// constant source, only works on a single span
pub struct ToMono<S> {
inner: S,
input_channel_count: ChannelCount,
connected_channels: ChannelCount,
/// running mean of second channel 'volume'
means: [f32; MAX_CHANNELS],
}
impl<S: Source> ToMono<S> {
fn new(input: S) -> Self {
let channels = input
.channels()
.min(const { NonZero::<u16>::new(MAX_CHANNELS as u16).unwrap() });
if channels < input.channels() {
warn!("Ignoring input channels {}..", channels.get());
}
Self {
connected_channels: channels,
input_channel_count: channels,
inner: input,
means: [TYPICAL_NOISE_FLOOR; MAX_CHANNELS],
}
}
}
impl<S: Source> Source for ToMono<S> {
fn current_span_len(&self) -> Option<usize> {
None
}
fn channels(&self) -> ChannelCount {
rodio::nz!(1)
}
fn sample_rate(&self) -> SampleRate {
self.inner.sample_rate()
}
fn total_duration(&self) -> Option<Duration> {
self.inner.total_duration()
}
}
fn update_mean(mean: &mut f32, sample: Sample) {
const HISTORY: f32 = 500.0;
*mean *= (HISTORY - 1.0) / HISTORY;
*mean += sample.abs() / HISTORY;
}
impl<S: Source> Iterator for ToMono<S> {
type Item = Sample;
fn next(&mut self) -> Option<Self::Item> {
let mut mono_sample = 0f32;
let mut active_channels = 0;
for channel in 0..self.input_channel_count.get() as usize {
let sample = self.inner.next()?;
mono_sample += sample;
update_mean(&mut self.means[channel], sample);
if self.means[channel] > TYPICAL_NOISE_FLOOR / 10.0 {
active_channels += 1;
}
}
mono_sample /= self.connected_channels.get() as f32;
self.connected_channels = NonZero::new(active_channels).unwrap_or(nz!(1));
Some(mono_sample)
}
}
/// constant source, only works on a single span
pub struct TakeSamples<S> {
inner: S,
left_to_take: usize,
@@ -147,6 +307,7 @@ impl<S: Source> Source for TakeSamples<S> {
}
}
/// constant source, only works on a single span
#[derive(Debug)]
struct ReplayQueue {
inner: ArrayQueue<Vec<Sample>>,
@@ -193,6 +354,7 @@ impl ReplayQueue {
}
}
/// constant source, only works on a single span
pub struct ProcessBuffer<const N: usize, S, F>
where
S: Source + Sized,
@@ -260,6 +422,7 @@ where
}
}
/// constant source, only works on a single span
pub struct InspectBuffer<const N: usize, S, F>
where
S: Source + Sized,
@@ -324,6 +487,7 @@ where
}
}
/// constant source, only works on a single span
#[derive(Debug)]
pub struct Replayable<S: Source> {
inner: S,
@@ -375,6 +539,7 @@ impl<S: Source> Source for Replayable<S> {
}
}
/// constant source, only works on a single span
#[derive(Debug)]
pub struct Replay {
rx: Arc<ReplayQueue>,