enclave-chat/enclave
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Improved resampler

Browse Source
This commit is contained in:
selimaj-dev
2026-08-28 01:35:35 +02:00
parent cbe0019d7c
commit 3a83226fd6
3 changed files with 313 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/Cargo.lock
Generated
+136 -7
View File
@@ -229,6 +229,39 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "audio-codec-algorithms"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1254ebf6529f3763c491acfb5ab6e960809e3c75a38584cc664f8c5667fb7107"
[[package]]
name = "audioadapter"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1292ef9edf681b7426ed089004b021a42896492f58bd0c909ad44e5faec9ac4"
[[package]]
name = "audioadapter-buffers"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46289a81a3bfa26d0f8b2415f8ca9decde9a66308a150568af5c102381b3c8ce"
dependencies = [
"audioadapter",
"audioadapter-sample",
"num-traits",
]
[[package]]
name = "audioadapter-sample"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5258faecf4edbe35ec483bda9b1f7438e6b391891819269c6b5461aeaa0e8d7d"
dependencies = [
"audio-codec-algorithms",
"num-traits",
]
[[package]]
name = "autocfg"
version = "1.5.1"
@@ -962,8 +995,8 @@ name = "enclave"
version = "0.1.0"
dependencies = [
"cpal",
"resampler",
"ringbuf",
"rubato",
"serde",
"serde_json",
"tauri",
@@ -2239,6 +2272,15 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.2"
@@ -2256,6 +2298,15 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "num-integer"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
dependencies = [
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -2792,6 +2843,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "primal-check"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
dependencies = [
"num-integer",
]
[[package]]
name = "proc-macro-crate"
version = "1.3.1"
@@ -2890,6 +2950,15 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"
[[package]]
name = "realfft"
version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677"
dependencies = [
"rustfft",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -2993,12 +3062,6 @@ dependencies = [
"web-sys",
]
[[package]]
name = "resampler"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28fdbea87ff02ebbfd904079d1e52138c0c7fbaa3aaddca4a1b9c7d3f85749f2"
[[package]]
name = "ringbuf"
version = "0.5.1"
@@ -3010,6 +3073,22 @@ dependencies = [
"portable-atomic-util",
]
[[package]]
name = "rubato"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7cb1ffaf8738df50aab642a7f6465df81c6ba9e2818268053487165298114be"
dependencies = [
"audioadapter",
"audioadapter-buffers",
"num-complex",
"num-integer",
"num-traits",
"realfft",
"visibility",
"windowfunctions",
]
[[package]]
name = "rustc-hash"
version = "2.1.3"
@@ -3025,6 +3104,20 @@ dependencies = [
"semver",
]
[[package]]
name = "rustfft"
version = "6.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89"
dependencies = [
"num-complex",
"num-integer",
"num-traits",
"primal-check",
"strength_reduce",
"transpose",
]
[[package]]
name = "rustix"
version = "1.1.4"
@@ -3429,6 +3522,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
[[package]]
name = "strength_reduce"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
[[package]]
name = "string_cache"
version = "0.9.0"
@@ -4183,6 +4282,16 @@ dependencies = [
"once_cell",
]
[[package]]
name = "transpose"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
dependencies = [
"num-integer",
"strength_reduce",
]
[[package]]
name = "tray-icon"
version = "0.24.2"
@@ -4342,6 +4451,17 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "visibility"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "vswhom"
version = "0.1.0"
@@ -4612,6 +4732,15 @@ dependencies = [
"windows-version",
]
[[package]]
name = "windowfunctions"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a"
dependencies = [
"num-traits",
]
[[package]]
name = "windows"
version = "0.61.3"
src-tauri/Cargo.toml
+1 -1
View File
@@ -23,6 +23,6 @@ tauri-plugin-opener = "2"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
cpal = "0.18.2"
resampler = "0.5.1"
ringbuf = "0.5.1"
rubato = "5.0.0"
src-tauri/src/commands/audio.rs
+175 -134
View File
@@ -19,20 +19,16 @@ use tauri::State;
use crate::commands::config::ConfigState;
const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz
const TARGET_SAMPLE_RATE: u32 = 48_000;
const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz mono
const HEADER_SIZE: usize = 4;
const MAX_PACKET_SIZE: usize = 4096;
const INITIAL_PACKET_CUSHION: usize = 3; // ~60ms cushion
const INITIAL_PACKET_CUSHION: usize = 3;
// --- Voice Activity Detection (VAD) Settings ---
const VAD_THRESHOLD: f32 = 0.01;
const VAD_HANGOVER_FRAMES: usize = 10;
// ============================================================================
// STATE & TYPES
// ============================================================================
type AudioProducer = Arc<Mutex<CachingProd<Arc<SharedRb<Heap<f32>>>>>>;
type AudioConsumer = Arc<Mutex<CachingCons<Arc<SharedRb<Heap<f32>>>>>>;
@@ -60,6 +56,39 @@ pub struct VoiceSession {
pub consumer_out: AudioConsumer,
}
// Simple Linear Resampler for real-time audio conversion
struct LinearResampler {
phase: f64,
}
impl LinearResampler {
fn new() -> Self {
Self { phase: 0.0 }
}
/// Resamples dynamic buffers from `src_rate` to `dst_rate`
fn process(&mut self, input: &[f32], src_rate: u32, dst_rate: u32, output: &mut Vec<f32>) {
if src_rate == dst_rate {
output.extend_from_slice(input);
return;
}
let ratio = src_rate as f64 / dst_rate as f64;
while self.phase < input.len() as f64 {
let idx = self.phase as usize;
let frac = (self.phase - idx as f64) as f32;
let next_idx = (idx + 1).min(input.len() - 1);
let sample = input[idx] * (1.0 - frac) + input[next_idx] * frac;
output.push(sample);
self.phase += ratio;
}
self.phase -= input.len() as f64;
}
}
impl VoiceSession {
pub fn update_input_device(
&mut self,
@@ -89,32 +118,7 @@ impl VoiceSession {
.config();
let producer = Arc::clone(&self.producer_in);
let inner_clone = Arc::clone(&state_inner);
let new_stream = device
.build_input_stream(
input_config,
move |data: &[f32], _| {
if let Ok(mut prod) = producer.lock() {
let _ = prod.push_slice(data);
}
},
move |err| {
eprintln!("[vc] Input error: {err}. Attempting input stream recovery...");
if let Ok(mut lock) = inner_clone.session.lock() {
if let Some(session) = lock.as_mut() {
let target_device = session.current_input_device.clone();
if let Err(e) =
session.update_input_device(target_device, Arc::clone(&inner_clone))
{
eprintln!("[vc] Input recovery failed: {e}");
}
}
}
},
None,
)
.map_err(|e| e.to_string())?;
let new_stream = build_input_stream(&device, input_config, producer, state_inner)?;
new_stream
.play()
@@ -167,9 +171,136 @@ impl VoiceSession {
}
}
// ============================================================================
// DEVICES
// ============================================================================
// Helper to build normalized Input Stream (Resampled & Downmixed to 48kHz Mono)
fn build_input_stream(
device: &cpal::Device,
config: cpal::StreamConfig,
producer: AudioProducer,
state_inner: Arc<VoiceStateInner>,
) -> Result<cpal::Stream, String> {
let native_sample_rate = config.sample_rate;
let channels = config.channels as usize;
let mut resampler = LinearResampler::new();
let mut mono_buffer = Vec::with_capacity(2048);
let mut resampled_buffer = Vec::with_capacity(2048);
let inner_input_err = Arc::clone(&state_inner);
device
.build_input_stream(
config,
move |data: &[f32], _| {
mono_buffer.clear();
resampled_buffer.clear();
// Downmix channels to mono
for chunk in data.chunks_exact(channels) {
let sum: f32 = chunk.iter().sum();
mono_buffer.push(sum / channels as f32);
}
// Resample to 48kHz standard target
resampler.process(
&mono_buffer,
native_sample_rate,
TARGET_SAMPLE_RATE,
&mut resampled_buffer,
);
if let Ok(mut prod) = producer.lock() {
let _ = prod.push_slice(&resampled_buffer);
}
},
move |err| {
eprintln!("[vc] Input error: {err}. Attempting recovery...");
if let Ok(mut lock) = inner_input_err.session.lock() {
if let Some(session) = lock.as_mut() {
let target_device = session.current_input_device.clone();
let _ = session
.update_input_device(target_device, Arc::clone(&inner_input_err));
}
}
},
None,
)
.map_err(|e| e.to_string())
}
// Helper to build normalized Output Stream (48kHz Mono -> Device Native Channels & Rate)
fn build_output_stream(
device: &cpal::Device,
config: cpal::StreamConfig,
consumer: AudioConsumer,
state_inner: Arc<VoiceStateInner>,
) -> Result<cpal::Stream, String> {
let native_sample_rate = config.sample_rate;
let channels = config.channels as usize;
let mut resampler = LinearResampler::new();
let mut raw_mono_samples = Vec::with_capacity(2048);
let mut resampled_mono = Vec::with_capacity(2048);
let mut last_sample = 0.0f32;
let inner_output_err = Arc::clone(&state_inner);
device
.build_output_stream(
config,
move |data: &mut [f32], _| {
let required_mono_samples = (data.len() / channels) * TARGET_SAMPLE_RATE as usize
/ native_sample_rate as usize;
raw_mono_samples.clear();
resampled_mono.clear();
if let Ok(mut cons) = consumer.lock() {
for _ in 0..required_mono_samples {
if let Some(s) = cons.try_pop() {
last_sample = s;
raw_mono_samples.push(s);
} else {
// Exponential decay to prevent clicking when underflowing
last_sample *= 0.92;
raw_mono_samples.push(last_sample);
}
}
}
// Resample from 48kHz mono to target native output rate
resampler.process(
&raw_mono_samples,
TARGET_SAMPLE_RATE,
native_sample_rate,
&mut resampled_mono,
);
// Interleave mono into hardware channels
let mut res_idx = 0;
let mut out_idx = 0;
while out_idx < data.len() && res_idx < resampled_mono.len() {
let mono_val = resampled_mono[res_idx];
for ch in 0..channels {
if out_idx + ch < data.len() {
data[out_idx + ch] = mono_val;
}
}
out_idx += channels;
res_idx += 1;
}
},
move |err| {
eprintln!("[vc] Output error: {err}. Attempting recovery...");
if let Ok(mut lock) = inner_output_err.session.lock() {
if let Some(session) = lock.as_mut() {
let target_device = session.current_output_device.clone();
let _ = session
.update_output_device(target_device, Arc::clone(&inner_output_err));
}
}
},
None,
)
.map_err(|e| e.to_string())
}
#[tauri::command]
pub fn list_input_devices() -> Result<Vec<String>, String> {
@@ -191,10 +322,6 @@ pub fn list_output_devices() -> Result<Vec<String>, String> {
.collect())
}
// ============================================================================
// DISCONNECT
// ============================================================================
#[tauri::command]
pub fn disconnect_from_vc(voice_state: State<'_, VoiceState>) -> Result<(), String> {
let mut lock = voice_state
@@ -205,7 +332,6 @@ pub fn disconnect_from_vc(voice_state: State<'_, VoiceState>) -> Result<(), Stri
if let Some(session) = lock.take() {
session.shutdown.store(true, Ordering::SeqCst);
let _ = session.input_stream.pause();
if let Ok(output) = session.output_stream.lock() {
let _ = output.pause();
@@ -215,10 +341,6 @@ pub fn disconnect_from_vc(voice_state: State<'_, VoiceState>) -> Result<(), Stri
Ok(())
}
// ============================================================================
// CONNECT
// ============================================================================
#[tauri::command]
pub fn connect_to_vc(
hostname: String,
@@ -280,7 +402,7 @@ pub fn connect_to_vc(
let shutdown = Arc::new(AtomicBool::new(false));
// Output Ring Buffer setup
// Ring Buffer Setup
let rb_out = HeapRb::<f32>::new(19200);
let (producer_out, consumer_out) = rb_out.split();
let mut producer_out = producer_out;
@@ -302,7 +424,6 @@ pub fn connect_to_vc(
.map_err(|e| format!("Failed to start output: {e}"))?;
let output_stream = Arc::new(Mutex::new(output_stream));
// Input Ring Buffer setup
let rb_in = HeapRb::<f32>::new(19200);
let (producer_in, mut consumer_in) = rb_in.split();
let shared_producer_in = Arc::new(Mutex::new(producer_in));
@@ -312,35 +433,14 @@ pub fn connect_to_vc(
.map_err(|e| format!("Failed to get default input config: {e}"))?
.config();
let cb_producer = Arc::clone(&shared_producer_in);
let inner_input_err = Arc::clone(&state_inner);
let input_stream = input_device
.build_input_stream(
let input_stream = build_input_stream(
&input_device,
input_config,
move |data: &[f32], _| {
if let Ok(mut prod) = cb_producer.lock() {
let _ = prod.push_slice(data);
}
},
move |err| {
eprintln!("[vc] Input error: {err}. Attempting input stream recovery...");
if let Ok(mut lock) = inner_input_err.session.lock() {
if let Some(session) = lock.as_mut() {
let target_device = session.current_input_device.clone();
if let Err(e) =
session.update_input_device(target_device, Arc::clone(&inner_input_err))
{
eprintln!("[vc] Input recovery failed: {e}");
}
}
}
},
None,
)
.map_err(|e| e.to_string())?;
Arc::clone(&shared_producer_in),
Arc::clone(&state_inner),
)?;
// UDP Sender Thread
// Sender Thread
{
let input_socket = socket.clone();
let shutdown = shutdown.clone();
@@ -386,7 +486,7 @@ pub fn connect_to_vc(
});
}
// UDP Receiver Thread
// Receiver Thread
{
let socket = socket.clone();
let shutdown = shutdown.clone();
@@ -477,65 +577,6 @@ pub fn connect_to_vc(
Ok(())
}
// ============================================================================
// AUDIO CALLBACK
// ============================================================================
fn build_output_stream(
device: &cpal::Device,
config: cpal::StreamConfig,
consumer: AudioConsumer,
state_inner: Arc<VoiceStateInner>,
) -> Result<cpal::Stream, String> {
let channels = config.channels as usize;
let mut last_sample = 0.0f32;
let inner_output_err = Arc::clone(&state_inner);
device
.build_output_stream(
config,
move |data: &mut [f32], _| {
let mut idx = 0;
let mut cons_guard = consumer.lock().ok();
while idx < data.len() {
let sample = cons_guard.as_mut().and_then(|c| c.try_pop());
if let Some(mono_sample) = sample {
last_sample = mono_sample;
for ch in 0..channels {
data[idx + ch] = mono_sample;
}
idx += channels;
} else {
while idx < data.len() {
last_sample *= 0.92;
for ch in 0..channels {
data[idx + ch] = last_sample;
}
idx += channels;
}
break;
}
}
},
move |err| {
eprintln!("[vc] Output error: {err}. Attempting output stream recovery...");
if let Ok(mut lock) = inner_output_err.session.lock() {
if let Some(session) = lock.as_mut() {
let target_device = session.current_output_device.clone();
if let Err(e) = session
.update_output_device(target_device, Arc::clone(&inner_output_err))
{
eprintln!("[vc] Output recovery failed: {e}");
}
}
}
},
None,
)
.map_err(|e| e.to_string())
}
impl Drop for VoiceSession {
fn drop(&mut self) {
self.shutdown.store(true, Ordering::SeqCst);