enclave-chat/enclave
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

More performance stuff

Browse Source
This commit is contained in:
selimaj-dev
2026-08-27 00:23:40 +02:00
parent 52e9583cfd
commit 2935d53ea8
1 changed files with 60 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/commands/audio.rs
+60 -280
View File
@@ -105,30 +105,19 @@ pub fn connect_to_vc(
let config = config_state.0.lock().unwrap().clone(); let config = config_state.0.lock().unwrap().clone();
// ------------------------------------------------------------------------
// UDP
// ------------------------------------------------------------------------
let socket = Arc::new( let socket = Arc::new(
UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?, UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?,
); );
socket socket
.connect(&hostname) .connect(&hostname)
.map_err(|e| format!("Failed to connect UDP socket: {e}"))?; .map_err(|e| format!("Failed to connect UDP socket: {e}"))?;
socket socket
.set_read_timeout(Some(Duration::from_millis(100))) .set_read_timeout(Some(Duration::from_millis(50)))
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
socket socket
.send(&pin.to_be_bytes()) .send(&pin.to_be_bytes())
.map_err(|e| format!("Failed to send pin: {e}"))?; .map_err(|e| format!("Failed to send pin: {e}"))?;
// ------------------------------------------------------------------------
// DEVICES
// ------------------------------------------------------------------------
let host = cpal::default_host(); let host = cpal::default_host();
let input_device = match &config.input_device_name { let input_device = match &config.input_device_name {
@@ -142,7 +131,6 @@ pub fn connect_to_vc(
.unwrap_or(false) .unwrap_or(false)
}) })
.ok_or("Input device not found")?, .ok_or("Input device not found")?,
None => host None => host
.default_input_device() .default_input_device()
.ok_or("No default input device")?, .ok_or("No default input device")?,
@@ -159,7 +147,6 @@ pub fn connect_to_vc(
.unwrap_or(false) .unwrap_or(false)
}) })
.ok_or("Output device not found")?, .ok_or("Output device not found")?,
None => host None => host
.default_output_device() .default_output_device()
.ok_or("No default output device")?, .ok_or("No default output device")?,
@@ -167,52 +154,35 @@ pub fn connect_to_vc(
let shutdown = Arc::new(AtomicBool::new(false)); let shutdown = Arc::new(AtomicBool::new(false));
// ------------------------------------------------------------------------ // Lock-Free Ring Buffer setup (Hold 100ms max buffer)
// PLAYBACK BUFFER (RingBuffer) let rb = HeapRb::<f32>::new(8820);
// ------------------------------------------------------------------------ let (mut producer, consumer) = rb.split();
let max_capacity = 48000;
let rb = HeapRb::<f32>::new(max_capacity);
let (producer, consumer) = rb.split();
let producer_lock = Arc::new(Mutex::new(producer));
let consumer_lock = Arc::new(Mutex::new(consumer));
// ------------------------------------------------------------------------
// OUTPUT
// ------------------------------------------------------------------------
let output_config = build_output_config(&output_device)?; let output_config = build_output_config(&output_device)?;
let output_config = Arc::new(Mutex::new(output_config)); let output_config = Arc::new(Mutex::new(output_config));
let needs_output_rebuild = Arc::new(AtomicBool::new(false)); let needs_output_rebuild = Arc::new(AtomicBool::new(false));
// Consumer moved directly without Mutex wrapping
let output_stream = build_output_stream( let output_stream = build_output_stream(
&output_device, &output_device,
&output_config.lock().unwrap(), &output_config.lock().unwrap(),
consumer_lock.clone(), consumer,
needs_output_rebuild.clone(), needs_output_rebuild.clone(),
)?; )?;
output_stream output_stream
.play() .play()
.map_err(|e| format!("Failed to start output: {e}"))?; .map_err(|e| format!("Failed to start output: {e}"))?;
let output_stream = Arc::new(Mutex::new(output_stream)); let output_stream = Arc::new(Mutex::new(output_stream));
// ------------------------------------------------------------------------ // Setup input stream with stack/pre-allocated buffers
// INPUT
// ------------------------------------------------------------------------
let mut input_config: cpal::StreamConfig = input_device let mut input_config: cpal::StreamConfig = input_device
.default_input_config() .default_input_config()
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.into(); .into();
input_config.channels = 1; input_config.channels = 1;
let input_sample_rate = input_config.sample_rate; let input_sample_rate = input_config.sample_rate;
let input_socket = socket.clone(); let input_socket = socket.clone();
let mut input_resampler = resampler::ResamplerFft::new( let mut input_resampler = resampler::ResamplerFft::new(
@@ -223,9 +193,10 @@ pub fn connect_to_vc(
resampler::SampleRate::Hz44100, resampler::SampleRate::Hz44100,
); );
let mut input_buffer = Vec::<f32>::new(); let mut input_buffer = Vec::with_capacity(4096);
let mut packet_buffer = Vec::<f32>::new(); let mut packet_buffer = Vec::with_capacity(4096);
let mut sequence = 0u32; let mut sequence = 0u32;
let mut net_packet = vec![0u8; HEADER_SIZE + PACKET_SAMPLES * 2];
let input_stream = input_device let input_stream = input_device
.build_input_stream( .build_input_stream(
@@ -237,92 +208,34 @@ pub fn connect_to_vc(
let output_size = input_resampler.chunk_size_output(); let output_size = input_resampler.chunk_size_output();
while input_buffer.len() >= input_size { while input_buffer.len() >= input_size {
let input: Vec<f32> = input_buffer.drain(..input_size).collect(); let input_chunk: Vec<f32> = input_buffer.drain(..input_size).collect();
let mut output = vec![0.0; output_size]; let mut output = vec![0.0; output_size];
if input_resampler.resample(&input, &mut output).is_err() { if input_resampler.resample(&input_chunk, &mut output).is_ok() {
continue; packet_buffer.extend(output);
}
packet_buffer.extend(output); while packet_buffer.len() >= PACKET_SAMPLES {
let samples = packet_buffer.drain(..PACKET_SAMPLES);
while packet_buffer.len() >= PACKET_SAMPLES { net_packet[0..4].copy_from_slice(&sequence.to_be_bytes());
let samples: Vec<f32> = packet_buffer.drain(..PACKET_SAMPLES).collect(); for (i, sample) in samples.enumerate() {
let pcm = (sample.clamp(-1.0, 1.0) * 32767.0) as i16;
let offset = HEADER_SIZE + i * 2;
net_packet[offset..offset + 2].copy_from_slice(&pcm.to_be_bytes());
}
let mut packet = Vec::with_capacity(HEADER_SIZE + PACKET_SAMPLES * 2); let _ = input_socket.send(&net_packet);
sequence = sequence.wrapping_add(1);
packet.extend_from_slice(&sequence.to_be_bytes());
for sample in samples {
let pcm = (sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16;
packet.extend_from_slice(&pcm.to_be_bytes());
} }
let _ = input_socket.send(&packet);
sequence = sequence.wrapping_add(1);
} }
} }
}, },
|err| { |err| eprintln!("[vc] input error: {err}"),
eprintln!("[vc] input error: {err}");
},
None, None,
) )
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
// ------------------------------------------------------------------------ // UDP Receiver Thread
// OUTPUT DEVICE REBUILD WATCHER
// ------------------------------------------------------------------------
{
let output_device = output_device.clone();
let output_stream = output_stream.clone();
let output_config = output_config.clone();
let consumer_lock = consumer_lock.clone();
let needs_rebuild = needs_output_rebuild.clone();
let shutdown = shutdown.clone();
thread::spawn(move || {
while !shutdown.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(100));
if !needs_rebuild.swap(false, Ordering::SeqCst) {
continue;
}
let Ok(new_config) = build_output_config(&output_device) else {
needs_rebuild.store(true, Ordering::SeqCst);
continue;
};
let Ok(new_stream) = build_output_stream(
&output_device,
&new_config,
consumer_lock.clone(),
needs_rebuild.clone(),
) else {
needs_rebuild.store(true, Ordering::SeqCst);
continue;
};
if new_stream.play().is_err() {
needs_rebuild.store(true, Ordering::SeqCst);
continue;
}
*output_config.lock().unwrap() = new_config;
*output_stream.lock().unwrap() = new_stream;
}
});
}
// ------------------------------------------------------------------------
// UDP RECEIVE (JITTER BUFFER INCLUDED)
// ------------------------------------------------------------------------
{ {
let socket = socket.clone(); let socket = socket.clone();
let output_config = output_config.clone(); let output_config = output_config.clone();
@@ -330,182 +243,88 @@ pub fn connect_to_vc(
thread::spawn(move || { thread::spawn(move || {
let initial_config = output_config.lock().unwrap().clone(); let initial_config = output_config.lock().unwrap().clone();
let mut output_resampler = match create_output_resampler(initial_config.sample_rate) { let mut output_resampler = match create_output_resampler(initial_config.sample_rate) {
Ok(r) => r, Ok(r) => r,
Err(e) => { Err(e) => return eprintln!("[vc] output resampler: {e}"),
eprintln!("[vc] output resampler: {e}");
return;
}
}; };
let mut last_sample_rate = initial_config.sample_rate; let mut last_sample_rate = initial_config.sample_rate;
let mut packets: BTreeMap<u32, Vec<f32>> = BTreeMap::new(); let mut packets: BTreeMap<u32, Vec<f32>> = BTreeMap::new();
let mut expected: Option<u32> = None; let mut expected: Option<u32> = None;
let mut is_prebuffering = true; let mut is_prebuffering = true;
let mut resample_buffer = Vec::<f32>::new(); let mut resample_buffer = Vec::with_capacity(8192);
let mut udp_buffer = [0u8; MAX_PACKET_SIZE]; let mut udp_buffer = [0u8; MAX_PACKET_SIZE];
while !shutdown.load(Ordering::SeqCst) { while !shutdown.load(Ordering::Relaxed) {
match socket.recv(&mut udp_buffer) { if let Ok(len) = socket.recv(&mut udp_buffer) {
Ok(len) => { if len > HEADER_SIZE {
if len <= HEADER_SIZE { let sequence = u32::from_be_bytes(udp_buffer[0..4].try_into().unwrap());
continue;
}
let sequence = u32::from_be_bytes([
udp_buffer[0],
udp_buffer[1],
udp_buffer[2],
udp_buffer[3],
]);
let pcm = &udp_buffer[HEADER_SIZE..len]; let pcm = &udp_buffer[HEADER_SIZE..len];
let samples: Vec<f32> = pcm
let mut samples = Vec::with_capacity(pcm.len() / 2); .chunks_exact(2)
.map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32768.0)
for chunk in pcm.chunks_exact(2) { .collect();
let value = i16::from_be_bytes([chunk[0], chunk[1]]);
samples.push(value as f32 / i16::MAX as f32);
}
packets.entry(sequence).or_insert(samples); packets.entry(sequence).or_insert(samples);
} }
Err(e) if e.kind() == std::io::ErrorKind::TimedOut => {
// Reset jitter state if connection completely drops
if packets.is_empty() {
expected = None;
is_prebuffering = true;
}
continue;
}
Err(e) => {
if !shutdown.load(Ordering::SeqCst) {
eprintln!("[vc] UDP receive error: {e}");
}
}
} }
// ---------------------------------------------------------------- let current_sr = output_config.lock().unwrap().sample_rate;
// OUTPUT CONFIG CHANGE if current_sr != last_sample_rate {
// ---------------------------------------------------------------- if let Ok(nr) = create_output_resampler(current_sr) {
output_resampler = nr;
let config = output_config.lock().unwrap().clone(); last_sample_rate = current_sr;
packets.clear();
if config.sample_rate != last_sample_rate { expected = None;
match create_output_resampler(config.sample_rate) { is_prebuffering = true;
Ok(new_resampler) => {
output_resampler = new_resampler;
last_sample_rate = config.sample_rate;
packets.clear();
resample_buffer.clear();
expected = None;
is_prebuffering = true;
}
Err(_) => continue,
} }
} }
// ----------------------------------------------------------------
// JITTER BUFFER STATE MACHINE
// ----------------------------------------------------------------
if is_prebuffering { if is_prebuffering {
// Accumulate target packet cushion before popping
if packets.len() >= INITIAL_PACKET_CUSHION { if packets.len() >= INITIAL_PACKET_CUSHION {
expected = packets.keys().next().copied(); expected = packets.keys().next().copied();
is_prebuffering = false; is_prebuffering = false;
} else { } else {
thread::sleep(Duration::from_millis(2));
continue; continue;
} }
} }
// Initialize sequence if unset
if expected.is_none() {
expected = packets.keys().next().copied();
}
// ----------------------------------------------------------------
// DRAIN PACKETS IN SEQUENTIAL ORDER
// ----------------------------------------------------------------
while let Some(seq) = expected { while let Some(seq) = expected {
if let Some(samples) = packets.remove(&seq) { if let Some(samples) = packets.remove(&seq) {
resample_buffer.extend(samples); resample_buffer.extend(samples);
expected = Some(seq.wrapping_add(1)); expected = Some(seq.wrapping_add(1));
} else if packets.keys().any(|&x| x > seq) {
resample_buffer.extend(std::iter::repeat(0.0).take(PACKET_SAMPLES));
expected = Some(seq.wrapping_add(1));
} else { } else {
// Missing frame strategy: break;
// If future sequence numbers exist, insert Concealment (Silence)
if packets.keys().any(|&x| x > seq) {
resample_buffer.extend(std::iter::repeat(0.0).take(PACKET_SAMPLES));
expected = Some(seq.wrapping_add(1));
} else {
// Waiting on late packets
break;
}
} }
} }
// Trim jitter buffer cache to avoid memory leak spikes on extreme latency drops
if packets.len() > 50 {
packets.clear();
expected = None;
is_prebuffering = true;
}
// ----------------------------------------------------------------
// RESAMPLE & POPULATE RINGBUFFER
// ----------------------------------------------------------------
let input_size = output_resampler.chunk_size_input(); let input_size = output_resampler.chunk_size_input();
let output_size = output_resampler.chunk_size_output(); let output_size = output_resampler.chunk_size_output();
while resample_buffer.len() >= input_size { while resample_buffer.len() >= input_size {
let input: Vec<f32> = resample_buffer.drain(..input_size).collect(); let input_chunk: Vec<f32> = resample_buffer.drain(..input_size).collect();
let mut output = vec![0.0; output_size]; let mut output = vec![0.0; output_size];
if output_resampler.resample(&input, &mut output).is_err() { if output_resampler.resample(&input_chunk, &mut output).is_ok() {
continue; // Push directly without Mutex lock!
} let _ = producer.push_slice(&output);
let mut prod = producer_lock.lock().unwrap();
// Latency Drift Protection: prevent total queue build-up past ~60ms
let target_sample_rate = config.sample_rate as usize;
let max_ring_buffer_samples = (target_sample_rate / 1000) * 60;
// Fix occupied_len check and overflow protection
// Note: We avoid calling try_pop() on a Producer. If the buffer is full,
// try_push will naturally fail and drop the overflow sample.
for sample in output {
if (*prod).occupied_len() < max_ring_buffer_samples {
let _ = (*prod).try_push(sample);
}
} }
} }
thread::sleep(Duration::from_millis(2));
} }
}); });
} }
// ------------------------------------------------------------------------
// START INPUT
// ------------------------------------------------------------------------
input_stream input_stream
.play() .play()
.map_err(|e| format!("Failed to start input: {e}"))?; .map_err(|e| format!("Failed to start input: {e}"))?;
// ------------------------------------------------------------------------
// SAVE
// ------------------------------------------------------------------------
*voice_state.session.lock().unwrap() = Some(VoiceSession { *voice_state.session.lock().unwrap() = Some(VoiceSession {
input_stream, input_stream,
output_stream, output_stream,
@@ -517,10 +336,6 @@ pub fn connect_to_vc(
Ok(()) Ok(())
} }
// ============================================================================
// OUTPUT CONFIG
// ============================================================================
fn build_output_config(device: &cpal::Device) -> Result<cpal::StreamConfig, String> { fn build_output_config(device: &cpal::Device) -> Result<cpal::StreamConfig, String> {
device device
.default_output_config() .default_output_config()
@@ -528,10 +343,6 @@ fn build_output_config(device: &cpal::Device) -> Result<cpal::StreamConfig, Stri
.map_err(|e| e.to_string()) .map_err(|e| e.to_string())
} }
// ============================================================================
// RESAMPLER
// ============================================================================
fn create_output_resampler( fn create_output_resampler(
sample_rate: cpal::SampleRate, sample_rate: cpal::SampleRate,
) -> Result<resampler::ResamplerFft, String> { ) -> Result<resampler::ResamplerFft, String> {
@@ -546,69 +357,38 @@ fn create_output_resampler(
)) ))
} }
// ============================================================================
// OUTPUT STREAM
// ============================================================================
fn build_output_stream<C>( fn build_output_stream<C>(
device: &cpal::Device, device: &cpal::Device,
config: &cpal::StreamConfig, config: &cpal::StreamConfig,
consumer_lock: Arc<Mutex<C>>, mut consumer: C,
needs_rebuild: Arc<AtomicBool>, needs_rebuild: Arc<AtomicBool>,
) -> Result<cpal::Stream, String> ) -> Result<cpal::Stream, String>
where where
C: Consumer<Item = f32> + Send + 'static, C: Consumer<Item = f32> + Send + 'static,
{ {
let channels = config.channels as usize; let channels = config.channels as usize;
let sample_rate = config.sample_rate as usize;
let jitter_cushion_samples = (sample_rate / 1000) * 40; // 40ms stream cushion
let mut is_buffering = true;
device device
.build_output_stream( .build_output_stream(
config.clone(), config.clone(),
move |data: &mut [f32], _| { move |data: &mut [f32], _| {
let mut cons = consumer_lock.lock().unwrap();
// 1. Initial/Recovering Cushioning
if is_buffering {
if (*cons).occupied_len() >= jitter_cushion_samples {
is_buffering = false;
} else {
data.fill(0.0);
return;
}
}
// 2. Sample Extraction & Channel Interleaving
let mut idx = 0; let mut idx = 0;
while idx < data.len() { while idx < data.len() {
if let Some(mono_sample) = (*cons).try_pop() { if let Some(mono_sample) = consumer.try_pop() {
for ch in 0..channels { for ch in 0..channels {
data[idx + ch] = mono_sample; data[idx + ch] = mono_sample;
} }
idx += channels; idx += channels;
} else { } else {
// 3. Underrun: pad rest with silence & re-enter buffering mode
data[idx..].fill(0.0); data[idx..].fill(0.0);
is_buffering = true;
break; break;
} }
} }
}, },
{ move |err| {
let needs_rebuild = needs_rebuild.clone(); eprintln!("[vc] output error: {err}");
move |err| {
eprintln!("[vc] output error: {err}"); needs_rebuild.store(true, Ordering::SeqCst);
let message = err.to_string();
if message.contains("sample rate changed")
|| message.contains("DeviceNotAvailable")
|| message.contains("device not available")
{
needs_rebuild.store(true, Ordering::SeqCst);
}
}
}, },
None, None,
) )