From 3a83226fd63cf1ad875c40490eca73329370f7ff Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Fri, 28 Aug 2026 01:35:35 +0200 Subject: [PATCH] Improved resampler --- src-tauri/Cargo.lock | 143 ++++++++++++++- src-tauri/Cargo.toml | 2 +- src-tauri/src/commands/audio.rs | 311 ++++++++++++++++++-------------- 3 files changed, 313 insertions(+), 143 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index e5acfe6..06091f3 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -229,6 +229,39 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "audio-codec-algorithms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1254ebf6529f3763c491acfb5ab6e960809e3c75a38584cc664f8c5667fb7107" + +[[package]] +name = "audioadapter" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1292ef9edf681b7426ed089004b021a42896492f58bd0c909ad44e5faec9ac4" + +[[package]] +name = "audioadapter-buffers" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46289a81a3bfa26d0f8b2415f8ca9decde9a66308a150568af5c102381b3c8ce" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5258faecf4edbe35ec483bda9b1f7438e6b391891819269c6b5461aeaa0e8d7d" +dependencies = [ + "audio-codec-algorithms", + "num-traits", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -962,8 +995,8 @@ name = "enclave" version = "0.1.0" dependencies = [ "cpal", - "resampler", "ringbuf", + "rubato", "serde", "serde_json", "tauri", @@ -2239,6 +2272,15 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2256,6 +2298,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2792,6 +2843,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -2890,6 +2950,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2993,12 +3062,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "resampler" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28fdbea87ff02ebbfd904079d1e52138c0c7fbaa3aaddca4a1b9c7d3f85749f2" - [[package]] name = "ringbuf" version = "0.5.1" @@ -3010,6 +3073,22 @@ dependencies = [ "portable-atomic-util", ] +[[package]] +name = "rubato" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cb1ffaf8738df50aab642a7f6465df81c6ba9e2818268053487165298114be" +dependencies = [ + "audioadapter", + "audioadapter-buffers", + "num-complex", + "num-integer", + "num-traits", + "realfft", + "visibility", + "windowfunctions", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3025,6 +3104,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3429,6 +3522,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.9.0" @@ -4183,6 +4282,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.24.2" @@ -4342,6 +4451,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -4612,6 +4732,15 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.61.3" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 8b27ac7..6cdf2a9 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -23,6 +23,6 @@ tauri-plugin-opener = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" cpal = "0.18.2" -resampler = "0.5.1" ringbuf = "0.5.1" +rubato = "5.0.0" diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index b8a2ac3..17f3eaf 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -19,20 +19,16 @@ use tauri::State; use crate::commands::config::ConfigState; -const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz +const TARGET_SAMPLE_RATE: u32 = 48_000; +const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz mono const HEADER_SIZE: usize = 4; const MAX_PACKET_SIZE: usize = 4096; -const INITIAL_PACKET_CUSHION: usize = 3; // ~60ms cushion +const INITIAL_PACKET_CUSHION: usize = 3; -// --- Voice Activity Detection (VAD) Settings --- const VAD_THRESHOLD: f32 = 0.01; const VAD_HANGOVER_FRAMES: usize = 10; -// ============================================================================ -// STATE & TYPES -// ============================================================================ - type AudioProducer = Arc>>>>>; type AudioConsumer = Arc>>>>>; @@ -60,6 +56,39 @@ pub struct VoiceSession { pub consumer_out: AudioConsumer, } +// Simple Linear Resampler for real-time audio conversion +struct LinearResampler { + phase: f64, +} + +impl LinearResampler { + fn new() -> Self { + Self { phase: 0.0 } + } + + /// Resamples dynamic buffers from `src_rate` to `dst_rate` + fn process(&mut self, input: &[f32], src_rate: u32, dst_rate: u32, output: &mut Vec) { + if src_rate == dst_rate { + output.extend_from_slice(input); + return; + } + + let ratio = src_rate as f64 / dst_rate as f64; + while self.phase < input.len() as f64 { + let idx = self.phase as usize; + let frac = (self.phase - idx as f64) as f32; + let next_idx = (idx + 1).min(input.len() - 1); + + let sample = input[idx] * (1.0 - frac) + input[next_idx] * frac; + output.push(sample); + + self.phase += ratio; + } + + self.phase -= input.len() as f64; + } +} + impl VoiceSession { pub fn update_input_device( &mut self, @@ -89,32 +118,7 @@ impl VoiceSession { .config(); let producer = Arc::clone(&self.producer_in); - let inner_clone = Arc::clone(&state_inner); - - let new_stream = device - .build_input_stream( - input_config, - move |data: &[f32], _| { - if let Ok(mut prod) = producer.lock() { - let _ = prod.push_slice(data); - } - }, - move |err| { - eprintln!("[vc] Input error: {err}. Attempting input stream recovery..."); - if let Ok(mut lock) = inner_clone.session.lock() { - if let Some(session) = lock.as_mut() { - let target_device = session.current_input_device.clone(); - if let Err(e) = - session.update_input_device(target_device, Arc::clone(&inner_clone)) - { - eprintln!("[vc] Input recovery failed: {e}"); - } - } - } - }, - None, - ) - .map_err(|e| e.to_string())?; + let new_stream = build_input_stream(&device, input_config, producer, state_inner)?; new_stream .play() @@ -167,9 +171,136 @@ impl VoiceSession { } } -// ============================================================================ -// DEVICES -// ============================================================================ +// Helper to build normalized Input Stream (Resampled & Downmixed to 48kHz Mono) +fn build_input_stream( + device: &cpal::Device, + config: cpal::StreamConfig, + producer: AudioProducer, + state_inner: Arc, +) -> Result { + let native_sample_rate = config.sample_rate; + let channels = config.channels as usize; + let mut resampler = LinearResampler::new(); + let mut mono_buffer = Vec::with_capacity(2048); + let mut resampled_buffer = Vec::with_capacity(2048); + + let inner_input_err = Arc::clone(&state_inner); + + device + .build_input_stream( + config, + move |data: &[f32], _| { + mono_buffer.clear(); + resampled_buffer.clear(); + + // Downmix channels to mono + for chunk in data.chunks_exact(channels) { + let sum: f32 = chunk.iter().sum(); + mono_buffer.push(sum / channels as f32); + } + + // Resample to 48kHz standard target + resampler.process( + &mono_buffer, + native_sample_rate, + TARGET_SAMPLE_RATE, + &mut resampled_buffer, + ); + + if let Ok(mut prod) = producer.lock() { + let _ = prod.push_slice(&resampled_buffer); + } + }, + move |err| { + eprintln!("[vc] Input error: {err}. Attempting recovery..."); + if let Ok(mut lock) = inner_input_err.session.lock() { + if let Some(session) = lock.as_mut() { + let target_device = session.current_input_device.clone(); + let _ = session + .update_input_device(target_device, Arc::clone(&inner_input_err)); + } + } + }, + None, + ) + .map_err(|e| e.to_string()) +} + +// Helper to build normalized Output Stream (48kHz Mono -> Device Native Channels & Rate) +fn build_output_stream( + device: &cpal::Device, + config: cpal::StreamConfig, + consumer: AudioConsumer, + state_inner: Arc, +) -> Result { + let native_sample_rate = config.sample_rate; + let channels = config.channels as usize; + let mut resampler = LinearResampler::new(); + let mut raw_mono_samples = Vec::with_capacity(2048); + let mut resampled_mono = Vec::with_capacity(2048); + let mut last_sample = 0.0f32; + + let inner_output_err = Arc::clone(&state_inner); + + device + .build_output_stream( + config, + move |data: &mut [f32], _| { + let required_mono_samples = (data.len() / channels) * TARGET_SAMPLE_RATE as usize + / native_sample_rate as usize; + + raw_mono_samples.clear(); + resampled_mono.clear(); + + if let Ok(mut cons) = consumer.lock() { + for _ in 0..required_mono_samples { + if let Some(s) = cons.try_pop() { + last_sample = s; + raw_mono_samples.push(s); + } else { + // Exponential decay to prevent clicking when underflowing + last_sample *= 0.92; + raw_mono_samples.push(last_sample); + } + } + } + + // Resample from 48kHz mono to target native output rate + resampler.process( + &raw_mono_samples, + TARGET_SAMPLE_RATE, + native_sample_rate, + &mut resampled_mono, + ); + + // Interleave mono into hardware channels + let mut res_idx = 0; + let mut out_idx = 0; + while out_idx < data.len() && res_idx < resampled_mono.len() { + let mono_val = resampled_mono[res_idx]; + for ch in 0..channels { + if out_idx + ch < data.len() { + data[out_idx + ch] = mono_val; + } + } + out_idx += channels; + res_idx += 1; + } + }, + move |err| { + eprintln!("[vc] Output error: {err}. Attempting recovery..."); + if let Ok(mut lock) = inner_output_err.session.lock() { + if let Some(session) = lock.as_mut() { + let target_device = session.current_output_device.clone(); + let _ = session + .update_output_device(target_device, Arc::clone(&inner_output_err)); + } + } + }, + None, + ) + .map_err(|e| e.to_string()) +} #[tauri::command] pub fn list_input_devices() -> Result, String> { @@ -191,10 +322,6 @@ pub fn list_output_devices() -> Result, String> { .collect()) } -// ============================================================================ -// DISCONNECT -// ============================================================================ - #[tauri::command] pub fn disconnect_from_vc(voice_state: State<'_, VoiceState>) -> Result<(), String> { let mut lock = voice_state @@ -205,7 +332,6 @@ pub fn disconnect_from_vc(voice_state: State<'_, VoiceState>) -> Result<(), Stri if let Some(session) = lock.take() { session.shutdown.store(true, Ordering::SeqCst); - let _ = session.input_stream.pause(); if let Ok(output) = session.output_stream.lock() { let _ = output.pause(); @@ -215,10 +341,6 @@ pub fn disconnect_from_vc(voice_state: State<'_, VoiceState>) -> Result<(), Stri Ok(()) } -// ============================================================================ -// CONNECT -// ============================================================================ - #[tauri::command] pub fn connect_to_vc( hostname: String, @@ -280,7 +402,7 @@ pub fn connect_to_vc( let shutdown = Arc::new(AtomicBool::new(false)); - // Output Ring Buffer setup + // Ring Buffer Setup let rb_out = HeapRb::::new(19200); let (producer_out, consumer_out) = rb_out.split(); let mut producer_out = producer_out; @@ -302,7 +424,6 @@ pub fn connect_to_vc( .map_err(|e| format!("Failed to start output: {e}"))?; let output_stream = Arc::new(Mutex::new(output_stream)); - // Input Ring Buffer setup let rb_in = HeapRb::::new(19200); let (producer_in, mut consumer_in) = rb_in.split(); let shared_producer_in = Arc::new(Mutex::new(producer_in)); @@ -312,35 +433,14 @@ pub fn connect_to_vc( .map_err(|e| format!("Failed to get default input config: {e}"))? .config(); - let cb_producer = Arc::clone(&shared_producer_in); - let inner_input_err = Arc::clone(&state_inner); + let input_stream = build_input_stream( + &input_device, + input_config, + Arc::clone(&shared_producer_in), + Arc::clone(&state_inner), + )?; - let input_stream = input_device - .build_input_stream( - input_config, - move |data: &[f32], _| { - if let Ok(mut prod) = cb_producer.lock() { - let _ = prod.push_slice(data); - } - }, - move |err| { - eprintln!("[vc] Input error: {err}. Attempting input stream recovery..."); - if let Ok(mut lock) = inner_input_err.session.lock() { - if let Some(session) = lock.as_mut() { - let target_device = session.current_input_device.clone(); - if let Err(e) = - session.update_input_device(target_device, Arc::clone(&inner_input_err)) - { - eprintln!("[vc] Input recovery failed: {e}"); - } - } - } - }, - None, - ) - .map_err(|e| e.to_string())?; - - // UDP Sender Thread + // Sender Thread { let input_socket = socket.clone(); let shutdown = shutdown.clone(); @@ -386,7 +486,7 @@ pub fn connect_to_vc( }); } - // UDP Receiver Thread + // Receiver Thread { let socket = socket.clone(); let shutdown = shutdown.clone(); @@ -477,65 +577,6 @@ pub fn connect_to_vc( Ok(()) } -// ============================================================================ -// AUDIO CALLBACK -// ============================================================================ - -fn build_output_stream( - device: &cpal::Device, - config: cpal::StreamConfig, - consumer: AudioConsumer, - state_inner: Arc, -) -> Result { - let channels = config.channels as usize; - let mut last_sample = 0.0f32; - let inner_output_err = Arc::clone(&state_inner); - - device - .build_output_stream( - config, - move |data: &mut [f32], _| { - let mut idx = 0; - let mut cons_guard = consumer.lock().ok(); - - while idx < data.len() { - let sample = cons_guard.as_mut().and_then(|c| c.try_pop()); - if let Some(mono_sample) = sample { - last_sample = mono_sample; - for ch in 0..channels { - data[idx + ch] = mono_sample; - } - idx += channels; - } else { - while idx < data.len() { - last_sample *= 0.92; - for ch in 0..channels { - data[idx + ch] = last_sample; - } - idx += channels; - } - break; - } - } - }, - move |err| { - eprintln!("[vc] Output error: {err}. Attempting output stream recovery..."); - if let Ok(mut lock) = inner_output_err.session.lock() { - if let Some(session) = lock.as_mut() { - let target_device = session.current_output_device.clone(); - if let Err(e) = session - .update_output_device(target_device, Arc::clone(&inner_output_err)) - { - eprintln!("[vc] Output recovery failed: {e}"); - } - } - } - }, - None, - ) - .map_err(|e| e.to_string()) -} - impl Drop for VoiceSession { fn drop(&mut self) { self.shutdown.store(true, Ordering::SeqCst);