From 1ac43684bf7950b86c4cb03a0c3582f6481b89b5 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Tue, 25 Aug 2026 20:55:24 +0200 Subject: [PATCH 01/29] Added cpal --- src-tauri/Cargo.lock | 279 ++++++++++++++++++++++++++++++++++++++++--- src-tauri/Cargo.toml | 1 + 2 files changed, 263 insertions(+), 17 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index cf16250..c3b0c70 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -32,6 +32,28 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.13.1", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -534,6 +556,50 @@ dependencies = [ "libc", ] +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.13.1", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "cpal" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f02e8d0327b42d3e2e4ab2119af397344eb9fc54a34bf0ddeaa1277af8681f1" +dependencies = [ + "alsa", + "block2", + "coreaudio-rs", + "dasp_sample", + "jni 0.22.4", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", + "web-sys", + "windows 0.62.2", + "windows-core 0.62.2", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -650,6 +716,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "dbus" version = "0.9.12" @@ -889,6 +961,7 @@ checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" name = "enclave" version = "0.1.0" dependencies = [ + "cpal", "serde", "serde_json", "tauri", @@ -1861,6 +1934,36 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + [[package]] name = "jni-sys" version = "0.3.1" @@ -2018,6 +2121,12 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "markup5ever" version = "0.38.0" @@ -2107,6 +2216,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -2128,6 +2243,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2182,6 +2308,32 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.1", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -2193,6 +2345,29 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + [[package]] name = "objc2-core-data" version = "0.3.2" @@ -2210,7 +2385,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.1", + "block2", "dispatch2", + "libc", "objc2", ] @@ -2282,6 +2459,7 @@ checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.13.1", "block2", + "libc", "objc2", "objc2-core-foundation", ] @@ -3134,6 +3312,22 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.3" @@ -3340,7 +3534,7 @@ dependencies = [ "gdkwayland-sys", "gdkx11-sys", "gtk", - "jni", + "jni 0.21.1", "libc", "log", "ndk", @@ -3356,7 +3550,7 @@ dependencies = [ "tao-macros", "unicode-segmentation", "url", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", "x11-dl", @@ -3396,7 +3590,7 @@ dependencies = [ "gtk", "heck 0.5.0", "http", - "jni", + "jni 0.21.1", "libc", "log", "mime", @@ -3427,7 +3621,7 @@ dependencies = [ "webkit2gtk", "webview2-com", "window-vibrancy", - "windows", + "windows 0.61.3", ] [[package]] @@ -3526,7 +3720,7 @@ dependencies = [ "tauri-plugin", "thiserror 2.0.20", "url", - "windows", + "windows 0.61.3", "zbus", ] @@ -3540,7 +3734,7 @@ dependencies = [ "dpi", "gtk", "http", - "jni", + "jni 0.21.1", "objc2", "objc2-ui-kit", "objc2-web-kit", @@ -3552,7 +3746,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", ] [[package]] @@ -3563,7 +3757,7 @@ checksum = "4e6fac707727b7a2f48e4ded90976324267371073edbb415ffb73bb0458d203f" dependencies = [ "gtk", "http", - "jni", + "jni 0.21.1", "log", "objc2", "objc2-app-kit", @@ -3577,7 +3771,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", "wry", ] @@ -4325,7 +4519,7 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a" dependencies = [ "webview2-com-macros", "webview2-com-sys", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-implement", "windows-interface", @@ -4349,7 +4543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c" dependencies = [ "thiserror 2.0.20", - "windows", + "windows 0.61.3", "windows-core 0.61.2", ] @@ -4405,11 +4599,23 @@ version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -4421,6 +4627,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.61.2" @@ -4455,7 +4670,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -4502,6 +4728,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -4605,6 +4841,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-version" version = "0.1.7" @@ -4768,7 +5013,7 @@ dependencies = [ "gtk", "http", "javascriptcore-rs", - "jni", + "jni 0.21.1", "libc", "ndk", "objc2", @@ -4788,7 +5033,7 @@ dependencies = [ "webkit2gtk", "webkit2gtk-sys", "webview2-com", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", "x11-dl", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9f54383..fbed8ef 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -22,4 +22,5 @@ tauri = { version = "2", features = [] } tauri-plugin-opener = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" +cpal = "0.18.2" From 29baa1f486921f4dc1903480c710812ee5fe29d8 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Tue, 25 Aug 2026 23:43:52 +0200 Subject: [PATCH 02/29] Testing audio --- src-tauri/src/commands/audio.rs | 25 +++++++++++++++++++++++++ src-tauri/src/commands/mod.rs | 1 + src-tauri/src/lib.rs | 1 + 3 files changed, 27 insertions(+) create mode 100644 src-tauri/src/commands/audio.rs diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs new file mode 100644 index 0000000..9e32d8c --- /dev/null +++ b/src-tauri/src/commands/audio.rs @@ -0,0 +1,25 @@ +use cpal::traits::{DeviceTrait, HostTrait}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AudioDevice { + pub name: String, +} + +#[tauri::command] +pub fn list_input_devices() -> Result, String> { + let host = cpal::default_host(); + + let devices = host + .input_devices() + .map_err(|e| format!("Failed to enumerate input devices: {e}"))?; + + Ok(devices + .filter_map(|d| { + Some(AudioDevice { + name: d.description().ok()?.name().to_string(), + }) + }) + .collect()) +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index c714d1c..f8c30cc 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -1,2 +1,3 @@ pub mod accounts; +pub mod audio; pub mod server_list; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 6a7af70..72949f3 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -10,6 +10,7 @@ pub fn run() { commands::server_list::get_server_list, commands::accounts::save_accounts, commands::accounts::get_accounts, + commands::audio::list_input_devices, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); From 4a28cd25df3e1a989073b3f3a5f5b89cbe7c1533 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 00:09:39 +0200 Subject: [PATCH 03/29] Settings UI --- src/App.tsx | 3 + src/app/app.ts | 2 + .../settings/AppearanceSettings.tsx | 14 ++++ .../settings/AudioVideoSettings.tsx | 14 ++++ src/components/settings/ProfileSettings.tsx | 14 ++++ src/components/settings/SettingsDialog.tsx | 74 +++++++++++++++++++ src/components/ui/dialog.tsx | 51 +++++++------ src/components/view/AccountCard.tsx | 11 ++- 8 files changed, 156 insertions(+), 27 deletions(-) create mode 100644 src/components/settings/AppearanceSettings.tsx create mode 100644 src/components/settings/AudioVideoSettings.tsx create mode 100644 src/components/settings/ProfileSettings.tsx create mode 100644 src/components/settings/SettingsDialog.tsx diff --git a/src/App.tsx b/src/App.tsx index 45fbf04..bf24892 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -10,6 +10,7 @@ import { import Sidebar from "./components/view/Sidebar"; import { ThemeProvider } from "next-themes"; import Page from "./components/page/PageView"; +import { SettingsDialog } from "./components/settings/SettingsDialog"; export default function App() { const appRef = useRef(null); @@ -54,6 +55,8 @@ export default function App() { + + )} diff --git a/src/app/app.ts b/src/app/app.ts index 1150284..c02b33d 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -28,12 +28,14 @@ export default class Enclave

{ public accounts?: AccountsFile; public server?: EnclaveServer; public serverList: ServerList; + public isSettingsOpen: boolean; public forceRender: () => void; public page?: P; public constructor() { this.serverList = {}; this.forceRender = () => {}; + this.isSettingsOpen = false; } public async init() { diff --git a/src/components/settings/AppearanceSettings.tsx b/src/components/settings/AppearanceSettings.tsx new file mode 100644 index 0000000..19e5c8f --- /dev/null +++ b/src/components/settings/AppearanceSettings.tsx @@ -0,0 +1,14 @@ +import { Palette } from "lucide-react"; + +export function AppearanceSettings() { + return ( +

+

+ Appearance +

+

+ Theme and display settings go here. +

+
+ ); +} diff --git a/src/components/settings/AudioVideoSettings.tsx b/src/components/settings/AudioVideoSettings.tsx new file mode 100644 index 0000000..11b75c2 --- /dev/null +++ b/src/components/settings/AudioVideoSettings.tsx @@ -0,0 +1,14 @@ +import { Volume2 } from "lucide-react"; + +export function AudioVideoSettings() { + return ( +
+

+ Audio & Video +

+

+ Microphone/camera device selection goes here. +

+
+ ); +} diff --git a/src/components/settings/ProfileSettings.tsx b/src/components/settings/ProfileSettings.tsx new file mode 100644 index 0000000..d6957c1 --- /dev/null +++ b/src/components/settings/ProfileSettings.tsx @@ -0,0 +1,14 @@ +import { User } from "lucide-react"; + +export function ProfileSettings() { + return ( +
+

+ Profile +

+

+ Display name and avatar settings go here. +

+
+ ); +} diff --git a/src/components/settings/SettingsDialog.tsx b/src/components/settings/SettingsDialog.tsx new file mode 100644 index 0000000..ade9c34 --- /dev/null +++ b/src/components/settings/SettingsDialog.tsx @@ -0,0 +1,74 @@ +import { useState } from "react"; +import { Dialog, DialogContent } from "@/components/ui/dialog"; +import { cn } from "@/lib/utils"; +import { User, Palette, Mic, Volume2 } from "lucide-react"; +import { ProfileSettings } from "./ProfileSettings"; +import { AppearanceSettings } from "./AppearanceSettings"; +import { AudioVideoSettings } from "./AudioVideoSettings"; +import Enclave from "@/app/app"; + +type SettingsCategory = "profile" | "appearance" | "audio-video"; + +const CATEGORIES: { + id: SettingsCategory; + label: string; + icon: React.ElementType; +}[] = [ + { id: "profile", label: "Profile", icon: User }, + { id: "appearance", label: "Appearance", icon: Palette }, + { id: "audio-video", label: "Audio & Video", icon: Volume2 }, +]; + +export function SettingsDialog({ + appRef, +}: { + appRef: React.RefObject; +}) { + const [category, setCategory] = useState("profile"); + + return ( + { + if (!appRef.current) return; + + appRef.current.isSettingsOpen = open; + appRef.current.forceRender(); + }} + > + +
+ + Settings + + +
+ +
+ {category === "profile" && } + {category === "appearance" && } + {category === "audio-video" && } +
+ + + ); +} diff --git a/src/components/ui/dialog.tsx b/src/components/ui/dialog.tsx index 3fc1dda..3770278 100644 --- a/src/components/ui/dialog.tsx +++ b/src/components/ui/dialog.tsx @@ -1,24 +1,24 @@ -import * as React from "react" -import { Dialog as DialogPrimitive } from "@base-ui/react/dialog" +import * as React from "react"; +import { Dialog as DialogPrimitive } from "@base-ui/react/dialog"; -import { cn } from "@/lib/utils" -import { Button } from "@/components/ui/button" -import { XIcon } from "lucide-react" +import { cn } from "@/lib/utils"; +import { Button } from "@/components/ui/button"; +import { XIcon } from "lucide-react"; function Dialog({ ...props }: DialogPrimitive.Root.Props) { - return + return ; } function DialogTrigger({ ...props }: DialogPrimitive.Trigger.Props) { - return + return ; } function DialogPortal({ ...props }: DialogPrimitive.Portal.Props) { - return + return ; } function DialogClose({ ...props }: DialogPrimitive.Close.Props) { - return + return ; } function DialogOverlay({ @@ -29,12 +29,12 @@ function DialogOverlay({ - ) + ); } function DialogContent({ @@ -43,7 +43,7 @@ function DialogContent({ showCloseButton = true, ...props }: DialogPrimitive.Popup.Props & { - showCloseButton?: boolean + showCloseButton?: boolean; }) { return ( @@ -52,7 +52,7 @@ function DialogContent({ data-slot="dialog-content" className={cn( "fixed top-1/2 left-1/2 z-50 grid w-full max-w-[calc(100%-2rem)] -translate-x-1/2 -translate-y-1/2 gap-4 rounded-xl bg-popover p-4 text-sm text-popover-foreground ring-1 ring-foreground/10 duration-100 outline-none sm:max-w-sm data-open:animate-in data-open:fade-in-0 data-open:zoom-in-95 data-closed:animate-out data-closed:fade-out-0 data-closed:zoom-out-95", - className + className, )} {...props} > @@ -68,14 +68,13 @@ function DialogContent({ /> } > - + Close )} - ) + ); } function DialogHeader({ className, ...props }: React.ComponentProps<"div">) { @@ -85,7 +84,7 @@ function DialogHeader({ className, ...props }: React.ComponentProps<"div">) { className={cn("flex flex-col gap-2", className)} {...props} /> - ) + ); } function DialogFooter({ @@ -94,14 +93,14 @@ function DialogFooter({ children, ...props }: React.ComponentProps<"div"> & { - showCloseButton?: boolean + showCloseButton?: boolean; }) { return (
@@ -112,7 +111,7 @@ function DialogFooter({ )}
- ) + ); } function DialogTitle({ className, ...props }: DialogPrimitive.Title.Props) { @@ -121,11 +120,11 @@ function DialogTitle({ className, ...props }: DialogPrimitive.Title.Props) { data-slot="dialog-title" className={cn( "font-heading text-base leading-none font-medium", - className + className, )} {...props} /> - ) + ); } function DialogDescription({ @@ -137,11 +136,11 @@ function DialogDescription({ data-slot="dialog-description" className={cn( "text-sm text-muted-foreground *:[a]:underline *:[a]:underline-offset-3 *:[a]:hover:text-foreground", - className + className, )} {...props} /> - ) + ); } export { @@ -155,4 +154,4 @@ export { DialogPortal, DialogTitle, DialogTrigger, -} +}; diff --git a/src/components/view/AccountCard.tsx b/src/components/view/AccountCard.tsx index 67b8a9a..d5d63e4 100644 --- a/src/components/view/AccountCard.tsx +++ b/src/components/view/AccountCard.tsx @@ -41,7 +41,16 @@ export default function AccountCard({
-
From 3099c915586d0f2f47157254f5772ebcb21bad0d Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 01:04:08 +0200 Subject: [PATCH 04/29] Config --- src-tauri/src/commands/config.rs | 64 ++++++++++++++++++++++++++++++++ src-tauri/src/commands/mod.rs | 1 + src-tauri/src/lib.rs | 8 ++++ src/lib/config.ts | 17 +++++++++ 4 files changed, 90 insertions(+) create mode 100644 src-tauri/src/commands/config.rs create mode 100644 src/lib/config.ts diff --git a/src-tauri/src/commands/config.rs b/src-tauri/src/commands/config.rs new file mode 100644 index 0000000..961b2da --- /dev/null +++ b/src-tauri/src/commands/config.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; +use std::fs; +use std::sync::Mutex; +use tauri::{AppHandle, Manager, State}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Config { + pub audio_device_name: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + audio_device_name: None, + } + } +} + +pub struct ConfigState(pub Mutex); + +fn config_file_path(app: &AppHandle) -> Result { + let dir = app + .path() + .app_data_dir() + .map_err(|e| format!("Failed to resolve app data dir: {e}"))?; + fs::create_dir_all(&dir).map_err(|e| format!("Failed to create app data dir: {e}"))?; + Ok(dir.join("config.json")) +} + +#[tauri::command] +pub fn update_config(state: State, config: Config) -> Result<(), String> { + *state.0.lock().unwrap() = config; + Ok(()) +} + +#[tauri::command] +pub fn save_config(app: AppHandle, state: State) -> Result<(), String> { + let path = config_file_path(&app)?; + let config = state.0.lock().unwrap(); + + let json = serde_json::to_string_pretty(&*config) + .map_err(|e| format!("Failed to serialize config: {e}"))?; + + fs::write(&path, json).map_err(|e| format!("Failed to write config: {e}"))?; + + Ok(()) +} + +#[tauri::command] +pub fn get_config(app: AppHandle, state: State) -> Result { + let path = config_file_path(&app)?; + + let config = if path.exists() { + let json = fs::read_to_string(&path).map_err(|e| format!("Failed to read config: {e}"))?; + serde_json::from_str(&json).map_err(|e| format!("Failed to parse config: {e}"))? + } else { + Config::default() + }; + + *state.0.lock().unwrap() = config.clone(); + + Ok(config) +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index f8c30cc..62c6f59 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -1,3 +1,4 @@ pub mod accounts; pub mod audio; +pub mod config; pub mod server_list; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 72949f3..5309958 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,3 +1,7 @@ +use std::sync::Mutex; + +use crate::commands::config::ConfigState; + pub mod commands; pub mod types; @@ -5,12 +9,16 @@ pub mod types; pub fn run() { tauri::Builder::default() .plugin(tauri_plugin_opener::init()) + .manage(ConfigState(Mutex::new(commands::config::Config::default()))) .invoke_handler(tauri::generate_handler![ commands::server_list::save_server_list, commands::server_list::get_server_list, commands::accounts::save_accounts, commands::accounts::get_accounts, commands::audio::list_input_devices, + commands::config::update_config, + commands::config::save_config, + commands::config::get_config, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src/lib/config.ts b/src/lib/config.ts new file mode 100644 index 0000000..a7194e8 --- /dev/null +++ b/src/lib/config.ts @@ -0,0 +1,17 @@ +import { invoke } from "@tauri-apps/api/core"; + +export type Config = { + audioDeviceName: string | null; +}; + +export async function updateConfig(config: Config): Promise { + await invoke("update_config", { config }); +} + +export async function saveConfig(): Promise { + await invoke("save_config"); +} + +export async function getConfig(): Promise { + return await invoke("get_config"); +} From a702834034fa888734dba60e086adb1d69c93489 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 01:33:42 +0200 Subject: [PATCH 05/29] Audio config --- src-tauri/src/commands/audio.rs | 28 +-- src-tauri/src/commands/config.rs | 17 +- src-tauri/src/lib.rs | 1 + .../settings/AudioVideoSettings.tsx | 125 ++++++++++- src/components/ui/select.tsx | 199 ++++++++++++++++++ src/components/ui/slider.tsx | 52 +++++ src/lib/config.ts | 5 +- 7 files changed, 406 insertions(+), 21 deletions(-) create mode 100644 src/components/ui/select.tsx create mode 100644 src/components/ui/slider.tsx diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 9e32d8c..1b0eabd 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -1,14 +1,7 @@ use cpal::traits::{DeviceTrait, HostTrait}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AudioDevice { - pub name: String, -} #[tauri::command] -pub fn list_input_devices() -> Result, String> { +pub fn list_input_devices() -> Result, String> { let host = cpal::default_host(); let devices = host @@ -16,10 +9,19 @@ pub fn list_input_devices() -> Result, String> { .map_err(|e| format!("Failed to enumerate input devices: {e}"))?; Ok(devices - .filter_map(|d| { - Some(AudioDevice { - name: d.description().ok()?.name().to_string(), - }) - }) + .filter_map(|d| Some(d.description().ok()?.name().to_string())) + .collect()) +} + +#[tauri::command] +pub fn list_output_devices() -> Result, String> { + let host = cpal::default_host(); + + let devices = host + .output_devices() + .map_err(|e| format!("Failed to enumerate output devices: {e}"))?; + + Ok(devices + .filter_map(|d| Some(d.description().ok()?.name().to_string())) .collect()) } diff --git a/src-tauri/src/commands/config.rs b/src-tauri/src/commands/config.rs index 961b2da..cc7cc22 100644 --- a/src-tauri/src/commands/config.rs +++ b/src-tauri/src/commands/config.rs @@ -1,3 +1,4 @@ +use cpal::traits::{DeviceTrait, HostTrait}; use serde::{Deserialize, Serialize}; use std::fs; use std::sync::Mutex; @@ -6,13 +7,25 @@ use tauri::{AppHandle, Manager, State}; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Config { - pub audio_device_name: Option, + pub input_device_name: Option, + pub output_device_name: Option, + pub input_volume: u8, + pub output_volume: u8, } impl Default for Config { fn default() -> Self { + let host = cpal::default_host(); + Self { - audio_device_name: None, + input_device_name: host + .default_input_device() + .and_then(|d| Some(d.description().ok()?.name().to_string())), + output_device_name: host + .default_output_device() + .and_then(|d| Some(d.description().ok()?.name().to_string())), + input_volume: 0, + output_volume: 0, } } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 5309958..c7845d6 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -16,6 +16,7 @@ pub fn run() { commands::accounts::save_accounts, commands::accounts::get_accounts, commands::audio::list_input_devices, + commands::audio::list_output_devices, commands::config::update_config, commands::config::save_config, commands::config::get_config, diff --git a/src/components/settings/AudioVideoSettings.tsx b/src/components/settings/AudioVideoSettings.tsx index 11b75c2..a474f92 100644 --- a/src/components/settings/AudioVideoSettings.tsx +++ b/src/components/settings/AudioVideoSettings.tsx @@ -1,14 +1,129 @@ -import { Volume2 } from "lucide-react"; +import { useEffect, useState } from "react"; +import { Volume2, Mic, Speaker } from "lucide-react"; +import { invoke } from "@tauri-apps/api/core"; +import { Label } from "@/components/ui/label"; +import { Slider } from "@/components/ui/slider"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { getConfig, updateConfig, saveConfig, Config } from "@/lib/config"; export function AudioVideoSettings() { + const [config, setConfig] = useState(null); + const [inputDevices, setInputDevices] = useState([]); + const [outputDevices, setOutputDevices] = useState([]); + + useEffect(() => { + getConfig().then(setConfig); + invoke("list_input_devices").then(setInputDevices); + invoke("list_output_devices").then(setOutputDevices); + }, []); + + async function handleChange(patch: Partial) { + if (!config) return; + const updated = { ...config, ...patch }; + setConfig(updated); + await updateConfig(updated); + await saveConfig(); + } + + if (!config) return null; + return ( -
+

Audio & Video

-

- Microphone/camera device selection goes here. -

+ +
+
+ + Input +
+ +
+ + +
+ +
+
+ + + {config.inputVolume}% + +
+ + handleChange({ inputVolume: value as any }) + } + max={100} + step={1} + /> +
+
+ +
+
+ + Output +
+ +
+ + +
+ +
+
+ + + {config.outputVolume}% + +
+ + handleChange({ outputVolume: value as any }) + } + max={100} + step={1} + /> +
+
); } diff --git a/src/components/ui/select.tsx b/src/components/ui/select.tsx new file mode 100644 index 0000000..56a7734 --- /dev/null +++ b/src/components/ui/select.tsx @@ -0,0 +1,199 @@ +import * as React from "react" +import { Select as SelectPrimitive } from "@base-ui/react/select" + +import { cn } from "@/lib/utils" +import { ChevronDownIcon, CheckIcon, ChevronUpIcon } from "lucide-react" + +const Select = SelectPrimitive.Root + +function SelectGroup({ className, ...props }: SelectPrimitive.Group.Props) { + return ( + + ) +} + +function SelectValue({ className, ...props }: SelectPrimitive.Value.Props) { + return ( + + ) +} + +function SelectTrigger({ + className, + size = "default", + children, + ...props +}: SelectPrimitive.Trigger.Props & { + size?: "sm" | "default" +}) { + return ( + + {children} + + } + /> + + ) +} + +function SelectContent({ + className, + children, + side = "bottom", + sideOffset = 4, + align = "center", + alignOffset = 0, + alignItemWithTrigger = true, + ...props +}: SelectPrimitive.Popup.Props & + Pick< + SelectPrimitive.Positioner.Props, + "align" | "alignOffset" | "side" | "sideOffset" | "alignItemWithTrigger" + >) { + return ( + + + + + {children} + + + + + ) +} + +function SelectLabel({ + className, + ...props +}: SelectPrimitive.GroupLabel.Props) { + return ( + + ) +} + +function SelectItem({ + className, + children, + ...props +}: SelectPrimitive.Item.Props) { + return ( + + + {children} + + + } + > + + + + ) +} + +function SelectSeparator({ + className, + ...props +}: SelectPrimitive.Separator.Props) { + return ( + + ) +} + +function SelectScrollUpButton({ + className, + ...props +}: React.ComponentProps) { + return ( + + + + ) +} + +function SelectScrollDownButton({ + className, + ...props +}: React.ComponentProps) { + return ( + + + + ) +} + +export { + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectLabel, + SelectScrollDownButton, + SelectScrollUpButton, + SelectSeparator, + SelectTrigger, + SelectValue, +} diff --git a/src/components/ui/slider.tsx b/src/components/ui/slider.tsx new file mode 100644 index 0000000..691fff3 --- /dev/null +++ b/src/components/ui/slider.tsx @@ -0,0 +1,52 @@ +import { Slider as SliderPrimitive } from "@base-ui/react/slider" + +import { cn } from "@/lib/utils" + +function Slider({ + className, + defaultValue, + value, + min = 0, + max = 100, + ...props +}: SliderPrimitive.Root.Props) { + const _values = Array.isArray(value) + ? value + : Array.isArray(defaultValue) + ? defaultValue + : [min, max] + + return ( + + + + + + {Array.from({ length: _values.length }, (_, index) => ( + + ))} + + + ) +} + +export { Slider } diff --git a/src/lib/config.ts b/src/lib/config.ts index a7194e8..270e766 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -1,7 +1,10 @@ import { invoke } from "@tauri-apps/api/core"; export type Config = { - audioDeviceName: string | null; + outputDeviceName: string | null; + inputDeviceName: string | null; + inputVolume: number; + outputVolume: number; }; export async function updateConfig(config: Config): Promise { From 0c6f8121433325fab2be094252fd8792e74e4843 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 01:45:22 +0200 Subject: [PATCH 06/29] Voice chat pre UI --- src/components/page/channel/ChannelPage.tsx | 3 +++ src/components/page/channel/VoiceChannel.tsx | 19 +++++++++++++++++++ src/components/view/Sidebar.tsx | 8 +++++--- src/lib/types.ts | 4 +++- 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 src/components/page/channel/VoiceChannel.tsx diff --git a/src/components/page/channel/ChannelPage.tsx b/src/components/page/channel/ChannelPage.tsx index f9b811b..cac049e 100644 --- a/src/components/page/channel/ChannelPage.tsx +++ b/src/components/page/channel/ChannelPage.tsx @@ -1,6 +1,7 @@ import Enclave from "@/app/app"; import { ChannelPageProps } from "@/components/page/PageView"; import TextChannel from "./TextChannel"; +import VoiceChannel from "./VoiceChannel"; export default function ChannelPage({ appRef, @@ -12,5 +13,7 @@ export default function ChannelPage({ switch (appRef.current?.page?.channel.kind) { case "text": return ; + case "voice": + return ; } } diff --git a/src/components/page/channel/VoiceChannel.tsx b/src/components/page/channel/VoiceChannel.tsx new file mode 100644 index 0000000..44a066d --- /dev/null +++ b/src/components/page/channel/VoiceChannel.tsx @@ -0,0 +1,19 @@ +import Enclave from "@/app/app"; +import { ChannelPageProps } from "../PageView"; + +export default function VoiceChannel({ + appRef, +}: { + appRef: React.RefObject | null>; +}) { + const channel = appRef.current?.page?.channel; + if (!channel) return null; + + return ( +
+
+

{channel.name}

+
+
+ ); +} diff --git a/src/components/view/Sidebar.tsx b/src/components/view/Sidebar.tsx index d786dc2..1fe61fa 100644 --- a/src/components/view/Sidebar.tsx +++ b/src/components/view/Sidebar.tsx @@ -1,16 +1,18 @@ import Enclave from "@/app/app"; import { Channel, ChannelKind } from "@/lib/types"; -import { ChevronDown, ChevronUp, HashIcon } from "lucide-react"; +import { ChevronDown, ChevronUp, HashIcon, Volume2Icon } from "lucide-react"; import { useState } from "react"; import AccountCard from "./AccountCard"; import { cn } from "@/lib/utils"; export function ChannelIcon({ kind }: { kind: ChannelKind["kind"] }) { switch (kind) { + case "category": + return null; case "text": return ; - default: - return null; + case "voice": + return ; } } diff --git a/src/lib/types.ts b/src/lib/types.ts index 647e08e..4aecd18 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -24,4 +24,6 @@ export interface StoredMessage extends MessageData { export type Channel = { id: string; name: string } & ChannelKind; export type ChannelKind = - { kind: "text" } | { kind: "category"; channels: Channel[] }; + | { kind: "category"; channels: Channel[] } + | { kind: "voice"; max_users: number } + | { kind: "text" }; From d95ed3215edd31133decb70c2068c92cb9bd6aa8 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 12:21:12 +0200 Subject: [PATCH 07/29] Connect and disconnect to vc --- src-tauri/src/commands/audio.rs | 170 ++++++++++++++++++- src-tauri/src/lib.rs | 2 + src/components/page/channel/VoiceChannel.tsx | 14 ++ 3 files changed, 185 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 1b0eabd..7cfaa84 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -1,4 +1,12 @@ -use cpal::traits::{DeviceTrait, HostTrait}; +use std::{ + net::UdpSocket, + sync::{Arc, Mutex}, +}; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use tauri::State; + +use crate::commands::config::ConfigState; #[tauri::command] pub fn list_input_devices() -> Result, String> { @@ -25,3 +33,163 @@ pub fn list_output_devices() -> Result, String> { .filter_map(|d| Some(d.description().ok()?.name().to_string())) .collect()) } + +pub struct VoiceState { + pub session: Mutex>, +} + +pub struct VoiceSession { + pub input_stream: cpal::Stream, + pub output_stream: cpal::Stream, + pub socket: Arc, + pub pin: u64, +} + +impl Default for VoiceState { + fn default() -> Self { + Self { + session: Mutex::new(None), + } + } +} + +#[tauri::command] +pub fn connect_to_vc( + hostname: String, + pin: u64, + config_state: State, + voice_state: State, +) -> Result<(), String> { + disconnect_from_vc(voice_state.clone())?; + + let config = config_state.0.lock().unwrap().clone(); + + let socket = UdpSocket::bind("0.0.0.0:0").map_err(|e| e.to_string())?; + socket.connect(&hostname).map_err(|e| e.to_string())?; + socket.set_nonblocking(true).map_err(|e| e.to_string())?; + let socket = Arc::new(socket); + + // Send the pin as the very first packet — this is your UDP "initialize" + socket + .send(&pin.to_be_bytes()) + .map_err(|e| format!("Failed to send pincode: {e}"))?; + + let host = cpal::default_host(); + + let input_device = match &config.input_device_name { + Some(name) => host + .input_devices() + .map_err(|e| e.to_string())? + .find(|d| { + d.description() + .ok() + .map(|d| d.name() == name) + .unwrap_or(false) + }) + .ok_or("Input device not found")?, + None => host + .default_input_device() + .ok_or("No default input device")?, + }; + + let output_device = match &config.output_device_name { + Some(name) => host + .output_devices() + .map_err(|e| e.to_string())? + .find(|d| { + d.description() + .ok() + .map(|d| d.name() == name) + .unwrap_or(false) + }) + .ok_or("Output device not found")?, + None => host + .default_output_device() + .ok_or("No default output device")?, + }; + + let input_socket = socket.clone(); + let input_config = input_device + .default_input_config() + .map_err(|e| e.to_string())? + .into(); + + let input_stream = input_device + .build_input_stream( + input_config, + move |data: &[f32], _| { + // convert f32 -> i16 PCM, prefix with pin, send over UDP + let pcm: Vec = data + .iter() + .map(|s| (s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16) + .collect(); + + let mut packet = pin.to_be_bytes().to_vec(); + packet.extend(pcm.iter().flat_map(|s| s.to_be_bytes())); + + let _ = input_socket.send(&packet); // non-blocking, drop on failure + }, + |err| eprintln!("Input stream error: {err}"), + None, + ) + .map_err(|e| e.to_string())?; + + let output_socket = socket.clone(); + let output_config = output_device + .default_output_config() + .map_err(|e| e.to_string())? + .into(); + + // shared ring buffer between the UDP-receiving task and the playback callback + let (audio_tx, audio_rx) = std::sync::mpsc::channel::>(); + + let output_stream = output_device + .build_output_stream( + output_config, + move |data: &mut [f32], _| { + if let Ok(pcm) = audio_rx.try_recv() { + for (out, sample) in data.iter_mut().zip(pcm.iter()) { + *out = *sample as f32 / i16::MAX as f32; + } + } else { + data.fill(0.0); // silence if nothing's arrived yet + } + }, + |err| eprintln!("Output stream error: {err}"), + None, + ) + .map_err(|e| e.to_string())?; + + // background thread reading incoming UDP audio, feeding the playback channel + let recv_socket = output_socket.clone(); + std::thread::spawn(move || { + let mut buf = [0u8; 4096]; + loop { + if let Ok(len) = recv_socket.recv(&mut buf) { + let pcm: Vec = buf[..len] + .chunks_exact(2) + .map(|b| i16::from_be_bytes([b[0], b[1]])) + .collect(); + let _ = audio_tx.send(pcm); + } + } + }); + + input_stream.play().map_err(|e| e.to_string())?; + output_stream.play().map_err(|e| e.to_string())?; + + *voice_state.session.lock().unwrap() = Some(VoiceSession { + input_stream, + output_stream, + socket, + pin, + }); + + Ok(()) +} + +#[tauri::command] +pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { + *voice_state.session.lock().unwrap() = None; // dropping stops both cpal streams + Ok(()) +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index c7845d6..5753e73 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -17,6 +17,8 @@ pub fn run() { commands::accounts::get_accounts, commands::audio::list_input_devices, commands::audio::list_output_devices, + commands::audio::connect_to_vc, + commands::audio::disconnect_from_vc, commands::config::update_config, commands::config::save_config, commands::config::get_config, diff --git a/src/components/page/channel/VoiceChannel.tsx b/src/components/page/channel/VoiceChannel.tsx index 44a066d..f342e63 100644 --- a/src/components/page/channel/VoiceChannel.tsx +++ b/src/components/page/channel/VoiceChannel.tsx @@ -1,5 +1,7 @@ import Enclave from "@/app/app"; import { ChannelPageProps } from "../PageView"; +import { useEffect, useRef } from "react"; +import { invoke } from "@tauri-apps/api/core"; export default function VoiceChannel({ appRef, @@ -9,6 +11,18 @@ export default function VoiceChannel({ const channel = appRef.current?.page?.channel; if (!channel) return null; + const lastChannelId = useRef(null); + + useEffect(() => { + const hostname = appRef.current?.server?.hostname; + if (!hostname) return; + if (lastChannelId.current === channel.id) return; + lastChannelId.current = channel.id; + + invoke("disconnect_from_vc"); + invoke("connect_to_vc", { hostname }); + }, [channel.id]); + return (
From f6cfea162174ed8c9f9fec4258dcaaf3e26076ed Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 14:31:26 +0200 Subject: [PATCH 08/29] Testing voice chat UDP --- src-tauri/src/commands/audio.rs | 12 ++++++------ src-tauri/src/lib.rs | 1 + src/app/app.ts | 4 ++++ src/app/protocol.ts | 9 +++++++++ src/app/server.ts | 1 + src/components/page/channel/VoiceChannel.tsx | 14 +++++++++++++- src/components/settings/SettingsDialog.tsx | 2 +- 7 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 7cfaa84..c32ba92 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -66,10 +66,9 @@ pub fn connect_to_vc( let socket = UdpSocket::bind("0.0.0.0:0").map_err(|e| e.to_string())?; socket.connect(&hostname).map_err(|e| e.to_string())?; - socket.set_nonblocking(true).map_err(|e| e.to_string())?; let socket = Arc::new(socket); - // Send the pin as the very first packet — this is your UDP "initialize" + // Send the pin as the very first packet — authenticates this UDP session socket .send(&pin.to_be_bytes()) .map_err(|e| format!("Failed to send pincode: {e}"))?; @@ -118,16 +117,17 @@ pub fn connect_to_vc( .build_input_stream( input_config, move |data: &[f32], _| { - // convert f32 -> i16 PCM, prefix with pin, send over UDP let pcm: Vec = data .iter() .map(|s| (s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16) .collect(); - let mut packet = pin.to_be_bytes().to_vec(); - packet.extend(pcm.iter().flat_map(|s| s.to_be_bytes())); + let mut packet = Vec::with_capacity(pcm.len() * 2); + for sample in &pcm { + packet.extend_from_slice(&sample.to_be_bytes()); + } - let _ = input_socket.send(&packet); // non-blocking, drop on failure + let _ = input_socket.send(&packet); }, |err| eprintln!("Input stream error: {err}"), None, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 5753e73..cd3b64b 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -10,6 +10,7 @@ pub fn run() { tauri::Builder::default() .plugin(tauri_plugin_opener::init()) .manage(ConfigState(Mutex::new(commands::config::Config::default()))) + .manage(commands::audio::VoiceState::default()) .invoke_handler(tauri::generate_handler![ commands::server_list::save_server_list, commands::server_list::get_server_list, diff --git a/src/app/app.ts b/src/app/app.ts index c02b33d..a34731d 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -206,6 +206,10 @@ export default class Enclave

{ server.messages[msg.channel_id][msg.message.id] = msg.message; this.forceRender(); return; + + case "JoinVoice": + server.voiceJoin && server.voiceJoin(msg.pin, msg.channel_id); + return; } } } diff --git a/src/app/protocol.ts b/src/app/protocol.ts index 0d1c204..db6a5af 100644 --- a/src/app/protocol.ts +++ b/src/app/protocol.ts @@ -29,6 +29,11 @@ export type ClientMethod = method: "MessageDeleted"; channel_id: string; message_id: string; + } + | { + method: "JoinVoice"; + channel_id: string; + pin: number; }; export type ServerMethod = @@ -71,4 +76,8 @@ export type ServerMethod = method: "DeleteMessage"; channel_id: string; message_id: string; + } + | { + method: "JoinVoice"; + channel_id: string; }; diff --git a/src/app/server.ts b/src/app/server.ts index 37ed228..525e121 100644 --- a/src/app/server.ts +++ b/src/app/server.ts @@ -31,6 +31,7 @@ export default class EnclaveServer { public meta?: ServerMeta; public messages: Record>; public users: Record; + public voiceJoin?: (pin: number, channelId: string) => void; public constructor(hostname: string, isSecure: boolean) { this.hostname = hostname; diff --git a/src/components/page/channel/VoiceChannel.tsx b/src/components/page/channel/VoiceChannel.tsx index f342e63..86ec3a8 100644 --- a/src/components/page/channel/VoiceChannel.tsx +++ b/src/components/page/channel/VoiceChannel.tsx @@ -20,7 +20,19 @@ export default function VoiceChannel({ lastChannelId.current = channel.id; invoke("disconnect_from_vc"); - invoke("connect_to_vc", { hostname }); + + if (channel.kind !== "voice" || !appRef.current?.server) return; + + appRef.current.server.voiceJoin = (pin, channelId) => { + invoke("connect_to_vc", { hostname, pin, channelId }).catch( + console.error, + ); + }; + + appRef.current.server.websocket?.send({ + method: "JoinVoice", + channel_id: channel.id, + }); }, [channel.id]); return ( diff --git a/src/components/settings/SettingsDialog.tsx b/src/components/settings/SettingsDialog.tsx index ade9c34..2b6f19c 100644 --- a/src/components/settings/SettingsDialog.tsx +++ b/src/components/settings/SettingsDialog.tsx @@ -1,7 +1,7 @@ import { useState } from "react"; import { Dialog, DialogContent } from "@/components/ui/dialog"; import { cn } from "@/lib/utils"; -import { User, Palette, Mic, Volume2 } from "lucide-react"; +import { User, Palette, Volume2 } from "lucide-react"; import { ProfileSettings } from "./ProfileSettings"; import { AppearanceSettings } from "./AppearanceSettings"; import { AudioVideoSettings } from "./AudioVideoSettings"; From c91a91643e75ea7f110ad203c382ff4e24748efd Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 14:56:11 +0200 Subject: [PATCH 09/29] logging --- src-tauri/src/commands/audio.rs | 12 +++++++++--- src/app/ws.ts | 4 ++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index c32ba92..f8d5054 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -127,9 +127,14 @@ pub fn connect_to_vc( packet.extend_from_slice(&sample.to_be_bytes()); } + eprintln!( + "[vc] mic captured {} samples ({} bytes)", + pcm.len(), + packet.len() + ); let _ = input_socket.send(&packet); }, - |err| eprintln!("Input stream error: {err}"), + |err| eprintln!("[vc] input stream error: {err}"), None, ) .map_err(|e| e.to_string())?; @@ -148,14 +153,15 @@ pub fn connect_to_vc( output_config, move |data: &mut [f32], _| { if let Ok(pcm) = audio_rx.try_recv() { + eprintln!("[vc] playing {} samples", pcm.len()); for (out, sample) in data.iter_mut().zip(pcm.iter()) { *out = *sample as f32 / i16::MAX as f32; } } else { - data.fill(0.0); // silence if nothing's arrived yet + data.fill(0.0); } }, - |err| eprintln!("Output stream error: {err}"), + |err| eprintln!("[vc] output stream error: {err}"), None, ) .map_err(|e| e.to_string())?; diff --git a/src/app/ws.ts b/src/app/ws.ts index 6b08d04..9962d7b 100644 --- a/src/app/ws.ts +++ b/src/app/ws.ts @@ -1,3 +1,4 @@ +import { invoke } from "@tauri-apps/api/core"; import { ClientMethod, ServerMethod } from "./protocol"; /** @@ -15,6 +16,9 @@ export default class EnclaveWebSocket { this.onOpenQueue = new Array(); this.websocket = new WebSocket(hostname); + this.websocket.onclose = () => { + invoke("disconnect_from_vc"); + }; this.websocket.onopen = () => { this.onOpenQueue.forEach((fun) => fun()); }; From a7ae604190c6fedae27984e9a9303db3fa35cf76 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 17:50:42 +0200 Subject: [PATCH 10/29] PCM resampling --- src-tauri/src/commands/audio.rs | 46 +++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index f8d5054..9d5eca3 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -122,16 +122,13 @@ pub fn connect_to_vc( .map(|s| (s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16) .collect(); - let mut packet = Vec::with_capacity(pcm.len() * 2); - for sample in &pcm { + let pcm_44k = resample_linear(&pcm, input_config.sample_rate, 44_100); + + let mut packet = Vec::with_capacity(pcm_44k.len() * 2); + for sample in &pcm_44k { packet.extend_from_slice(&sample.to_be_bytes()); } - eprintln!( - "[vc] mic captured {} samples ({} bytes)", - pcm.len(), - packet.len() - ); let _ = input_socket.send(&packet); }, |err| eprintln!("[vc] input stream error: {err}"), @@ -176,7 +173,16 @@ pub fn connect_to_vc( .chunks_exact(2) .map(|b| i16::from_be_bytes([b[0], b[1]])) .collect(); - let _ = audio_tx.send(pcm); + + let pcm_out = resample_linear(&pcm, 44_100, output_config.sample_rate); + + match audio_tx.send(pcm_out) { + Ok(_) => {} + Err(e) => { + eprintln!("[vc] audio channel disconnected: {e}"); + break; + } + } } } }); @@ -199,3 +205,27 @@ pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> *voice_state.session.lock().unwrap() = None; // dropping stops both cpal streams Ok(()) } + +fn resample_linear(input: &[i16], from_rate: u32, to_rate: u32) -> Vec { + if input.is_empty() || from_rate == to_rate { + return input.to_vec(); + } + + let output_len = (input.len() as u64 * to_rate as u64 / from_rate as u64) as usize; + + let mut output = Vec::with_capacity(output_len); + + for i in 0..output_len { + let position = i as f64 * from_rate as f64 / to_rate as f64; + + let index = position.floor() as usize; + let fraction = position - index as f64; + + let a = input[index.min(input.len() - 1)] as f64; + let b = input[(index + 1).min(input.len() - 1)] as f64; + + output.push((a + (b - a) * fraction) as i16); + } + + output +} From 77add3dea81f75f4ba888e20db2eab482eb40252 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 18:29:50 +0200 Subject: [PATCH 11/29] Testing resampling --- src-tauri/Cargo.lock | 7 ++ src-tauri/Cargo.toml | 1 + src-tauri/src/commands/audio.rs | 111 ++++++++++++++++++-------------- 3 files changed, 71 insertions(+), 48 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index c3b0c70..29c7181 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -962,6 +962,7 @@ name = "enclave" version = "0.1.0" dependencies = [ "cpal", + "resampler", "serde", "serde_json", "tauri", @@ -2991,6 +2992,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resampler" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28fdbea87ff02ebbfd904079d1e52138c0c7fbaa3aaddca4a1b9c7d3f85749f2" + [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index fbed8ef..c24ab14 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -23,4 +23,5 @@ tauri-plugin-opener = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" cpal = "0.18.2" +resampler = "0.5.1" diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 9d5eca3..39ed47d 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -108,28 +108,48 @@ pub fn connect_to_vc( }; let input_socket = socket.clone(); - let input_config = input_device + let mut input_config: cpal::StreamConfig = input_device .default_input_config() .map_err(|e| e.to_string())? .into(); + input_config.channels = 1; + + let mut input_resampler = resampler::ResamplerFft::new( + 1, + input_config + .sample_rate + .try_into() + .map_err(|e| format!("{e:?}"))?, + resampler::SampleRate::Hz44100, + ); + + let mut input_buffer = Vec::::new(); + let input_stream = input_device .build_input_stream( input_config, move |data: &[f32], _| { - let pcm: Vec = data - .iter() - .map(|s| (s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16) - .collect(); + input_buffer.extend_from_slice(data); - let pcm_44k = resample_linear(&pcm, input_config.sample_rate, 44_100); + while input_buffer.len() >= 1024 { + let input: Vec = input_buffer.drain(..1024).collect(); - let mut packet = Vec::with_capacity(pcm_44k.len() * 2); - for sample in &pcm_44k { - packet.extend_from_slice(&sample.to_be_bytes()); + let mut output = vec![0.0f32; 1024]; + + if let Err(e) = input_resampler.resample(&input, &mut output) { + eprintln!("[vc] failed to resample input: {e}"); + continue; + } + + let mut packet = Vec::with_capacity(output.len() * 4); + + for sample in output { + packet.extend_from_slice(&sample.to_be_bytes()); + } + + let _ = input_socket.send(&packet); } - - let _ = input_socket.send(&packet); }, |err| eprintln!("[vc] input stream error: {err}"), None, @@ -137,13 +157,23 @@ pub fn connect_to_vc( .map_err(|e| e.to_string())?; let output_socket = socket.clone(); - let output_config = output_device + let mut output_config: cpal::StreamConfig = output_device .default_output_config() .map_err(|e| e.to_string())? .into(); + let mut output_resampler = resampler::ResamplerFft::new( + 1, + resampler::SampleRate::Hz44100, + output_config + .sample_rate + .try_into() + .map_err(|e| format!("{e:?}"))?, + ); + + output_config.channels = 1; // shared ring buffer between the UDP-receiving task and the playback callback - let (audio_tx, audio_rx) = std::sync::mpsc::channel::>(); + let (audio_tx, audio_rx) = std::sync::mpsc::channel::<[f32; 1024]>(); let output_stream = output_device .build_output_stream( @@ -152,7 +182,7 @@ pub fn connect_to_vc( if let Ok(pcm) = audio_rx.try_recv() { eprintln!("[vc] playing {} samples", pcm.len()); for (out, sample) in data.iter_mut().zip(pcm.iter()) { - *out = *sample as f32 / i16::MAX as f32; + *out = *sample; } } else { data.fill(0.0); @@ -167,20 +197,29 @@ pub fn connect_to_vc( let recv_socket = output_socket.clone(); std::thread::spawn(move || { let mut buf = [0u8; 4096]; + let mut output_buffer = Vec::::new(); + loop { if let Ok(len) = recv_socket.recv(&mut buf) { - let pcm: Vec = buf[..len] - .chunks_exact(2) - .map(|b| i16::from_be_bytes([b[0], b[1]])) - .collect(); + let pcm = buf[..len] + .chunks_exact(4) + .map(|b| f32::from_be_bytes([b[0], b[1], b[2], b[3]])) + .collect::>(); - let pcm_out = resample_linear(&pcm, 44_100, output_config.sample_rate); + output_buffer.extend_from_slice(&pcm); - match audio_tx.send(pcm_out) { - Ok(_) => {} - Err(e) => { - eprintln!("[vc] audio channel disconnected: {e}"); - break; + while output_buffer.len() >= 1024 { + let input: Vec = output_buffer.drain(..1024).collect(); + + let mut pcm_out = [0.0f32; 1024]; + + if let Err(e) = output_resampler.resample(&input, &mut pcm_out) { + eprintln!("[vc] failed to resample output: {e}"); + continue; + } + + if audio_tx.send(pcm_out).is_err() { + return; } } } @@ -205,27 +244,3 @@ pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> *voice_state.session.lock().unwrap() = None; // dropping stops both cpal streams Ok(()) } - -fn resample_linear(input: &[i16], from_rate: u32, to_rate: u32) -> Vec { - if input.is_empty() || from_rate == to_rate { - return input.to_vec(); - } - - let output_len = (input.len() as u64 * to_rate as u64 / from_rate as u64) as usize; - - let mut output = Vec::with_capacity(output_len); - - for i in 0..output_len { - let position = i as f64 * from_rate as f64 / to_rate as f64; - - let index = position.floor() as usize; - let fraction = position - index as f64; - - let a = input[index.min(input.len() - 1)] as f64; - let b = input[(index + 1).min(input.len() - 1)] as f64; - - output.push((a + (b - a) * fraction) as i16); - } - - output -} From 7058517e8cafdb2ead3858031629e699c3dde466 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 18:44:37 +0200 Subject: [PATCH 12/29] More resampling --- src-tauri/src/commands/audio.rs | 47 ++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 39ed47d..b74e1a4 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -135,7 +135,10 @@ pub fn connect_to_vc( while input_buffer.len() >= 1024 { let input: Vec = input_buffer.drain(..1024).collect(); - let mut output = vec![0.0f32; 1024]; + let output_len = + ((1024.0 * 44100.0 / input_config.sample_rate as f32).ceil()) as usize; + + let mut output = vec![0.0f32; output_len]; if let Err(e) = input_resampler.resample(&input, &mut output) { eprintln!("[vc] failed to resample input: {e}"); @@ -157,7 +160,7 @@ pub fn connect_to_vc( .map_err(|e| e.to_string())?; let output_socket = socket.clone(); - let mut output_config: cpal::StreamConfig = output_device + let output_config: cpal::StreamConfig = output_device .default_output_config() .map_err(|e| e.to_string())? .into(); @@ -170,20 +173,23 @@ pub fn connect_to_vc( .map_err(|e| format!("{e:?}"))?, ); - output_config.channels = 1; - // shared ring buffer between the UDP-receiving task and the playback callback - let (audio_tx, audio_rx) = std::sync::mpsc::channel::<[f32; 1024]>(); + let (audio_tx, audio_rx) = std::sync::mpsc::channel::>(); let output_stream = output_device .build_output_stream( - output_config, + output_config.clone(), move |data: &mut [f32], _| { if let Ok(pcm) = audio_rx.try_recv() { - eprintln!("[vc] playing {} samples", pcm.len()); for (out, sample) in data.iter_mut().zip(pcm.iter()) { *out = *sample; } + + // If CPAL asks for more samples than we received, + // fill the remainder with silence. + if pcm.len() < data.len() { + data[pcm.len()..].fill(0.0); + } } else { data.fill(0.0); } @@ -211,14 +217,24 @@ pub fn connect_to_vc( while output_buffer.len() >= 1024 { let input: Vec = output_buffer.drain(..1024).collect(); - let mut pcm_out = [0.0f32; 1024]; + let output_len = + ((1024.0 * output_config.sample_rate as f32 / 44100.0).ceil()) as usize; + + let mut pcm_out = vec![0.0f32; output_len]; if let Err(e) = output_resampler.resample(&input, &mut pcm_out) { eprintln!("[vc] failed to resample output: {e}"); continue; } - if audio_tx.send(pcm_out).is_err() { + if audio_tx + .send(if output_config.channels > 1 { + stereo_to_mono(&pcm_out) + } else { + pcm_out + }) + .is_err() + { return; } } @@ -244,3 +260,16 @@ pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> *voice_state.session.lock().unwrap() = None; // dropping stops both cpal streams Ok(()) } + +fn stereo_to_mono(stereo_data: &[f32]) -> Vec { + let mut mono_data = Vec::with_capacity(stereo_data.len() / 2); + + for chunk in stereo_data.chunks_exact(2) { + let left = chunk[0]; + let right = chunk[1]; + let mono_sample = (left + right) / 2.0; + mono_data.push(mono_sample); + } + + mono_data +} From 5e8d5b2877541e73c52a64420178140b9029c346 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 19:49:08 +0200 Subject: [PATCH 13/29] Working audio (not really) --- src-tauri/src/commands/audio.rs | 289 ++++++++++++++++++++++++-------- 1 file changed, 215 insertions(+), 74 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index b74e1a4..a754f81 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -1,6 +1,9 @@ use std::{ net::UdpSocket, - sync::{Arc, Mutex}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, }; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; @@ -40,9 +43,12 @@ pub struct VoiceState { pub struct VoiceSession { pub input_stream: cpal::Stream, - pub output_stream: cpal::Stream, + // Wrapped so the watcher thread can swap in a freshly rebuilt stream. + pub output_stream: Arc>, pub socket: Arc, pub pin: u64, + // Set to true when the watcher thread should stop (on disconnect). + pub shutdown: Arc, } impl Default for VoiceState { @@ -53,6 +59,16 @@ impl Default for VoiceState { } } +#[tauri::command] +pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { + if let Some(session) = voice_state.session.lock().unwrap().take() { + // Tell the watcher thread to stop before dropping the streams. + session.shutdown.store(true, Ordering::SeqCst); + } + // Dropping the session stops both cpal streams. + Ok(()) +} + #[tauri::command] pub fn connect_to_vc( hostname: String, @@ -107,6 +123,8 @@ pub fn connect_to_vc( .ok_or("No default output device")?, }; + // ---------- INPUT (mic -> UDP) ---------- + let input_socket = socket.clone(); let mut input_config: cpal::StreamConfig = input_device .default_input_config() @@ -114,13 +132,11 @@ pub fn connect_to_vc( .into(); input_config.channels = 1; + let input_sample_rate = input_config.sample_rate; let mut input_resampler = resampler::ResamplerFft::new( 1, - input_config - .sample_rate - .try_into() - .map_err(|e| format!("{e:?}"))?, + input_sample_rate.try_into().map_err(|e| format!("{e:?}"))?, resampler::SampleRate::Hz44100, ); @@ -132,26 +148,38 @@ pub fn connect_to_vc( move |data: &[f32], _| { input_buffer.extend_from_slice(data); - while input_buffer.len() >= 1024 { - let input: Vec = input_buffer.drain(..1024).collect(); + let frame_size = input_resampler.chunk_size_input(); - let output_len = - ((1024.0 * 44100.0 / input_config.sample_rate as f32).ceil()) as usize; + while input_buffer.len() >= frame_size { + let input: Vec = input_buffer.drain(..frame_size).collect(); - let mut output = vec![0.0f32; output_len]; + let mut output = vec![0.0f32; input_resampler.chunk_size_output()]; if let Err(e) = input_resampler.resample(&input, &mut output) { eprintln!("[vc] failed to resample input: {e}"); continue; } - let mut packet = Vec::with_capacity(output.len() * 4); + // f32 -> i16 PCM + let mut packet = Vec::with_capacity(output.len() * 2); for sample in output { - packet.extend_from_slice(&sample.to_be_bytes()); + let sample = sample.clamp(-1.0, 1.0); + let pcm = (sample * i16::MAX as f32) as i16; + + packet.extend_from_slice(&pcm.to_be_bytes()); } - let _ = input_socket.send(&packet); + eprintln!( + "[vc] sending audio: input={} samples, packet={} bytes", + frame_size, + packet.len() + ); + + match input_socket.send(&packet) { + Ok(n) => eprintln!("[vc] UDP sent {n} bytes"), + Err(e) => eprintln!("[vc] UDP send failed: {e}"), + } } }, |err| eprintln!("[vc] input stream error: {err}"), @@ -159,54 +187,104 @@ pub fn connect_to_vc( ) .map_err(|e| e.to_string())?; - let output_socket = socket.clone(); - let output_config: cpal::StreamConfig = output_device - .default_output_config() - .map_err(|e| e.to_string())? - .into(); - let mut output_resampler = resampler::ResamplerFft::new( - 1, - resampler::SampleRate::Hz44100, - output_config - .sample_rate - .try_into() - .map_err(|e| format!("{e:?}"))?, - ); + // ---------- OUTPUT (UDP -> speakers), rebuildable on device change ---------- - // shared ring buffer between the UDP-receiving task and the playback callback let (audio_tx, audio_rx) = std::sync::mpsc::channel::>(); + let audio_rx = Arc::new(Mutex::new(audio_rx)); - let output_stream = output_device - .build_output_stream( - output_config.clone(), - move |data: &mut [f32], _| { - if let Ok(pcm) = audio_rx.try_recv() { - for (out, sample) in data.iter_mut().zip(pcm.iter()) { - *out = *sample; - } + let needs_output_rebuild = Arc::new(AtomicBool::new(false)); + let shutdown = Arc::new(AtomicBool::new(false)); - // If CPAL asks for more samples than we received, - // fill the remainder with silence. - if pcm.len() < data.len() { - data[pcm.len()..].fill(0.0); + // Output config is tracked in a shared cell so the UDP-receiving thread + // (below) always resamples toward whatever the *current* live stream expects, + // even after a rebuild changes the device's rate. + let output_config = build_output_config(&output_device)?; + let output_config_cell = Arc::new(Mutex::new(output_config.clone())); + + let initial_output_stream = build_output_stream( + &output_device, + &output_config, + audio_rx.clone(), + needs_output_rebuild.clone(), + )?; + initial_output_stream.play().map_err(|e| e.to_string())?; + + let output_stream = Arc::new(Mutex::new(initial_output_stream)); + + // Watcher thread: rebuilds the output stream whenever the device signals + // its config has changed (e.g. "Device sample rate changed"), since cpal + // streams can't be reconfigured in place — only rebuilt from scratch. + { + let output_device = output_device.clone(); + let audio_rx = audio_rx.clone(); + let needs_output_rebuild = needs_output_rebuild.clone(); + let output_stream = output_stream.clone(); + let output_config_cell = output_config_cell.clone(); + let shutdown = shutdown.clone(); + + std::thread::spawn(move || loop { + if shutdown.load(Ordering::SeqCst) { + return; + } + + std::thread::sleep(std::time::Duration::from_millis(200)); + + if needs_output_rebuild.swap(false, Ordering::SeqCst) { + match build_output_config(&output_device).and_then(|new_config| { + build_output_stream( + &output_device, + &new_config, + audio_rx.clone(), + needs_output_rebuild.clone(), + ) + .map(|stream| (new_config, stream)) + }) { + Ok((new_config, new_stream)) => { + if let Err(e) = new_stream.play() { + eprintln!("[vc] failed to start rebuilt output stream: {e}"); + continue; + } + *output_config_cell.lock().unwrap() = new_config; + *output_stream.lock().unwrap() = new_stream; + eprintln!("[vc] output stream rebuilt after device change"); } - } else { - data.fill(0.0); + Err(e) => eprintln!("[vc] failed to rebuild output stream: {e}"), } - }, - |err| eprintln!("[vc] output stream error: {err}"), - None, - ) - .map_err(|e| e.to_string())?; + } + }); + } - // background thread reading incoming UDP audio, feeding the playback channel - let recv_socket = output_socket.clone(); - std::thread::spawn(move || { - let mut buf = [0u8; 4096]; - let mut output_buffer = Vec::::new(); + // ---------- UDP receive thread: incoming audio -> resample -> playback channel ---------- + + let recv_socket = socket.clone(); + { + let output_config_cell = output_config_cell.clone(); + let shutdown = shutdown.clone(); + + std::thread::spawn(move || { + let mut buf = [0u8; 4096]; + let mut output_buffer = Vec::::new(); + let mut output_resampler = resampler::ResamplerFft::new( + 1, + resampler::SampleRate::Hz44100, + output_config_cell + .lock() + .unwrap() + .sample_rate + .try_into() + .unwrap(), + ); + let mut last_rate = output_config_cell.lock().unwrap().sample_rate; + + loop { + if shutdown.load(Ordering::SeqCst) { + return; + } + + let Ok(len) = recv_socket.recv(&mut buf) else { + continue; + }; - loop { - if let Ok(len) = recv_socket.recv(&mut buf) { let pcm = buf[..len] .chunks_exact(4) .map(|b| f32::from_be_bytes([b[0], b[1], b[2], b[3]])) @@ -214,51 +292,114 @@ pub fn connect_to_vc( output_buffer.extend_from_slice(&pcm); - while output_buffer.len() >= 1024 { - let input: Vec = output_buffer.drain(..1024).collect(); + let current_config = output_config_cell.lock().unwrap().clone(); - let output_len = - ((1024.0 * output_config.sample_rate as f32 / 44100.0).ceil()) as usize; + // If the output device's rate changed underneath us (rebuild + // happened), rebuild the resampler to target the new rate too. + if current_config.sample_rate != last_rate { + output_resampler = resampler::ResamplerFft::new( + 1, + resampler::SampleRate::Hz44100, + match current_config.sample_rate.try_into() { + Ok(rate) => rate, + Err(_) => { + eprintln!("[vc] unsupported output sample rate, skipping rebuild"); + last_rate = current_config.sample_rate; + continue; + } + }, + ); + last_rate = current_config.sample_rate; + } - let mut pcm_out = vec![0.0f32; output_len]; + let frame_size = output_resampler.chunk_size_input(); + + while output_buffer.len() >= frame_size { + let input: Vec = output_buffer.drain(..frame_size).collect(); + + let mut pcm_out = vec![0.0f32; output_resampler.chunk_size_output()]; if let Err(e) = output_resampler.resample(&input, &mut pcm_out) { eprintln!("[vc] failed to resample output: {e}"); continue; } - if audio_tx - .send(if output_config.channels > 1 { - stereo_to_mono(&pcm_out) - } else { - pcm_out - }) - .is_err() - { + let mono = if current_config.channels > 1 { + stereo_to_mono(&pcm_out) + } else { + pcm_out + }; + + // audio_rx/tx were built together, so this send only fails + // if every receiver was dropped — i.e. shutdown in progress. + let tx_result = { + let _rx_guard = audio_rx.lock().unwrap(); // keep receiver alive + audio_tx.send(mono) + }; + + if tx_result.is_err() { return; } } } - } - }); + }); + } input_stream.play().map_err(|e| e.to_string())?; - output_stream.play().map_err(|e| e.to_string())?; *voice_state.session.lock().unwrap() = Some(VoiceSession { input_stream, output_stream, socket, pin, + shutdown, }); Ok(()) } -#[tauri::command] -pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { - *voice_state.session.lock().unwrap() = None; // dropping stops both cpal streams - Ok(()) +fn build_output_config(output_device: &cpal::Device) -> Result { + Ok(output_device + .default_output_config() + .map_err(|e| e.to_string())? + .into()) +} + +fn build_output_stream( + output_device: &cpal::Device, + output_config: &cpal::StreamConfig, + audio_rx: Arc>>>, + needs_rebuild: Arc, +) -> Result { + output_device + .build_output_stream( + *output_config, + move |data: &mut [f32], _| { + let rx = audio_rx.lock().unwrap(); + if let Ok(pcm) = rx.try_recv() { + for (out, sample) in data.iter_mut().zip(pcm.iter()) { + *out = *sample; + } + if pcm.len() < data.len() { + data[pcm.len()..].fill(0.0); + } + } else { + data.fill(0.0); + } + }, + { + let needs_rebuild = needs_rebuild.clone(); + move |err| { + eprintln!("[vc] output stream error: {err}"); + let msg = err.to_string(); + if msg.contains("sample rate changed") || msg.contains("DeviceNotAvailable") { + needs_rebuild.store(true, Ordering::SeqCst); + } + } + }, + None, + ) + .map_err(|e| e.to_string()) } fn stereo_to_mono(stereo_data: &[f32]) -> Vec { From 198eb10bcf8fea73a96960f77c2119f03ebddffe Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 19:59:45 +0200 Subject: [PATCH 14/29] Working playback --- src-tauri/src/commands/audio.rs | 84 ++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index a754f81..9f5d73f 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -189,9 +189,7 @@ pub fn connect_to_vc( // ---------- OUTPUT (UDP -> speakers), rebuildable on device change ---------- - let (audio_tx, audio_rx) = std::sync::mpsc::channel::>(); - let audio_rx = Arc::new(Mutex::new(audio_rx)); - + let playback_buffer = Arc::new(Mutex::new(Vec::::new())); let needs_output_rebuild = Arc::new(AtomicBool::new(false)); let shutdown = Arc::new(AtomicBool::new(false)); @@ -204,7 +202,7 @@ pub fn connect_to_vc( let initial_output_stream = build_output_stream( &output_device, &output_config, - audio_rx.clone(), + playback_buffer.clone(), needs_output_rebuild.clone(), )?; initial_output_stream.play().map_err(|e| e.to_string())?; @@ -216,11 +214,11 @@ pub fn connect_to_vc( // streams can't be reconfigured in place — only rebuilt from scratch. { let output_device = output_device.clone(); - let audio_rx = audio_rx.clone(); let needs_output_rebuild = needs_output_rebuild.clone(); let output_stream = output_stream.clone(); let output_config_cell = output_config_cell.clone(); let shutdown = shutdown.clone(); + let playback_buffer = playback_buffer.clone(); std::thread::spawn(move || loop { if shutdown.load(Ordering::SeqCst) { @@ -234,7 +232,7 @@ pub fn connect_to_vc( build_output_stream( &output_device, &new_config, - audio_rx.clone(), + playback_buffer.clone(), needs_output_rebuild.clone(), ) .map(|stream| (new_config, stream)) @@ -264,6 +262,7 @@ pub fn connect_to_vc( std::thread::spawn(move || { let mut buf = [0u8; 4096]; let mut output_buffer = Vec::::new(); + let mut output_resampler = resampler::ResamplerFft::new( 1, resampler::SampleRate::Hz44100, @@ -274,6 +273,7 @@ pub fn connect_to_vc( .try_into() .unwrap(), ); + let mut last_rate = output_config_cell.lock().unwrap().sample_rate; loop { @@ -285,17 +285,19 @@ pub fn connect_to_vc( continue; }; + // UDP contains BIG-ENDIAN i16 PCM. let pcm = buf[..len] - .chunks_exact(4) - .map(|b| f32::from_be_bytes([b[0], b[1], b[2], b[3]])) + .chunks_exact(2) + .map(|b| { + let sample = i16::from_be_bytes([b[0], b[1]]); + sample as f32 / i16::MAX as f32 + }) .collect::>(); output_buffer.extend_from_slice(&pcm); let current_config = output_config_cell.lock().unwrap().clone(); - // If the output device's rate changed underneath us (rebuild - // happened), rebuild the resampler to target the new rate too. if current_config.sample_rate != last_rate { output_resampler = resampler::ResamplerFft::new( 1, @@ -303,12 +305,13 @@ pub fn connect_to_vc( match current_config.sample_rate.try_into() { Ok(rate) => rate, Err(_) => { - eprintln!("[vc] unsupported output sample rate, skipping rebuild"); + eprintln!("[vc] unsupported output sample rate"); last_rate = current_config.sample_rate; continue; } }, ); + last_rate = current_config.sample_rate; } @@ -324,22 +327,10 @@ pub fn connect_to_vc( continue; } - let mono = if current_config.channels > 1 { - stereo_to_mono(&pcm_out) - } else { - pcm_out - }; + // Mono network audio -> device channel layout. + let output = mono_to_output_channels(&pcm_out, current_config.channels); - // audio_rx/tx were built together, so this send only fails - // if every receiver was dropped — i.e. shutdown in progress. - let tx_result = { - let _rx_guard = audio_rx.lock().unwrap(); // keep receiver alive - audio_tx.send(mono) - }; - - if tx_result.is_err() { - return; - } + playback_buffer.lock().unwrap().extend(output); } } }); @@ -368,30 +359,31 @@ fn build_output_config(output_device: &cpal::Device) -> Result>>>, + playback_buffer: Arc>>, needs_rebuild: Arc, ) -> Result { output_device .build_output_stream( - *output_config, + output_config.clone(), move |data: &mut [f32], _| { - let rx = audio_rx.lock().unwrap(); - if let Ok(pcm) = rx.try_recv() { - for (out, sample) in data.iter_mut().zip(pcm.iter()) { - *out = *sample; + let mut buffer = playback_buffer.lock().unwrap(); + + for out in data.iter_mut() { + if buffer.is_empty() { + *out = 0.0; + } else { + *out = buffer.remove(0); } - if pcm.len() < data.len() { - data[pcm.len()..].fill(0.0); - } - } else { - data.fill(0.0); } }, { let needs_rebuild = needs_rebuild.clone(); + move |err| { eprintln!("[vc] output stream error: {err}"); + let msg = err.to_string(); + if msg.contains("sample rate changed") || msg.contains("DeviceNotAvailable") { needs_rebuild.store(true, Ordering::SeqCst); } @@ -414,3 +406,21 @@ fn stereo_to_mono(stereo_data: &[f32]) -> Vec { mono_data } + +fn mono_to_output_channels(mono: &[f32], channels: u16) -> Vec { + if channels <= 1 { + return mono.to_vec(); + } + + let channels = channels as usize; + + let mut output = Vec::with_capacity(mono.len() * channels); + + for &sample in mono { + for _ in 0..channels { + output.push(sample); + } + } + + output +} From 16223c31b1a3fdee10bd90e137a4767ceb7def8c Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 20:08:48 +0200 Subject: [PATCH 15/29] Optimisations --- src-tauri/src/commands/audio.rs | 725 ++++++++++++++++++++++++-------- 1 file changed, 554 insertions(+), 171 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 9f5d73f..e663822 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -1,9 +1,12 @@ use std::{ + collections::VecDeque, net::UdpSocket, sync::{ atomic::{AtomicBool, Ordering}, Arc, Mutex, }, + thread, + time::Duration, }; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; @@ -11,6 +14,76 @@ use tauri::State; use crate::commands::config::ConfigState; +// ============================================================ +// AUDIO CONSTANTS +// ============================================================ + +/// Network audio format. +/// +/// Everything sent over UDP is: +/// 44100 Hz +/// mono +/// signed i16 +/// big endian +/// +/// 20 ms @ 44100 Hz = 882 samples. +/// 882 * 2 = 1764 bytes. +const NETWORK_SAMPLE_RATE: u32 = 44_100; +const PACKET_DURATION_MS: usize = 20; +const NETWORK_PACKET_SAMPLES: usize = NETWORK_SAMPLE_RATE as usize * PACKET_DURATION_MS / 1000; + +/// Amount of audio that must be queued before playback begins. +const PREBUFFER_MS: usize = 40; + +/// If the playback queue gets below this amount, we allow it to +/// continue normally but the callback will output silence if it +/// actually runs dry. +const LOW_WATERMARK_MS: usize = 20; + +/// Maximum amount of queued audio. +/// +/// If this is exceeded, OLD audio is discarded. This is intentional: +/// for voice chat, dropping old audio is much better than accumulating +/// hundreds of milliseconds/seconds of latency. +const MAX_BUFFER_MS: usize = 100; + +/// Maximum UDP packet size we expect. +const MAX_UDP_PACKET_SIZE: usize = 4096; + +// ============================================================ +// STATE +// ============================================================ + +pub struct VoiceState { + pub session: Mutex>, +} + +pub struct VoiceSession { + pub input_stream: cpal::Stream, + + // Wrapped so the watcher thread can replace the stream after + // the output device changes configuration. + pub output_stream: Arc>, + + pub socket: Arc, + + pub pin: u64, + + pub shutdown: Arc, +} + +impl Default for VoiceState { + fn default() -> Self { + Self { + session: Mutex::new(None), + } + } +} + +// ============================================================ +// DEVICE ENUMERATION +// ============================================================ + #[tauri::command] pub fn list_input_devices() -> Result, String> { let host = cpal::default_host(); @@ -20,7 +93,7 @@ pub fn list_input_devices() -> Result, String> { .map_err(|e| format!("Failed to enumerate input devices: {e}"))?; Ok(devices - .filter_map(|d| Some(d.description().ok()?.name().to_string())) + .filter_map(|device| Some(device.description().ok()?.name().to_string())) .collect()) } @@ -33,42 +106,32 @@ pub fn list_output_devices() -> Result, String> { .map_err(|e| format!("Failed to enumerate output devices: {e}"))?; Ok(devices - .filter_map(|d| Some(d.description().ok()?.name().to_string())) + .filter_map(|device| Some(device.description().ok()?.name().to_string())) .collect()) } -pub struct VoiceState { - pub session: Mutex>, -} - -pub struct VoiceSession { - pub input_stream: cpal::Stream, - // Wrapped so the watcher thread can swap in a freshly rebuilt stream. - pub output_stream: Arc>, - pub socket: Arc, - pub pin: u64, - // Set to true when the watcher thread should stop (on disconnect). - pub shutdown: Arc, -} - -impl Default for VoiceState { - fn default() -> Self { - Self { - session: Mutex::new(None), - } - } -} +// ============================================================ +// DISCONNECT +// ============================================================ #[tauri::command] pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { if let Some(session) = voice_state.session.lock().unwrap().take() { - // Tell the watcher thread to stop before dropping the streams. session.shutdown.store(true, Ordering::SeqCst); + + // Streams are dropped here. + // + // The UDP receiver has a 100ms read timeout, so it will notice + // shutdown shortly instead of remaining blocked forever. } - // Dropping the session stops both cpal streams. + Ok(()) } +// ============================================================ +// CONNECT +// ============================================================ + #[tauri::command] pub fn connect_to_vc( hostname: String, @@ -80,68 +143,134 @@ pub fn connect_to_vc( let config = config_state.0.lock().unwrap().clone(); - let socket = UdpSocket::bind("0.0.0.0:0").map_err(|e| e.to_string())?; - socket.connect(&hostname).map_err(|e| e.to_string())?; + // ======================================================== + // UDP SOCKET + // ======================================================== + + let socket = + UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?; + + socket + .connect(&hostname) + .map_err(|e| format!("Failed to connect UDP socket: {e}"))?; + + // IMPORTANT: + // + // Without a timeout, recv() can remain blocked forever and the + // receiver thread can survive after disconnect. + socket + .set_read_timeout(Some(Duration::from_millis(100))) + .map_err(|e| format!("Failed to configure UDP timeout: {e}"))?; + let socket = Arc::new(socket); - // Send the pin as the very first packet — authenticates this UDP session + // PIN is always the first packet. socket .send(&pin.to_be_bytes()) .map_err(|e| format!("Failed to send pincode: {e}"))?; + // ======================================================== + // AUDIO DEVICES + // ======================================================== + let host = cpal::default_host(); let input_device = match &config.input_device_name { Some(name) => host .input_devices() .map_err(|e| e.to_string())? - .find(|d| { - d.description() + .find(|device| { + device + .description() .ok() - .map(|d| d.name() == name) + .map(|description| description.name() == name) .unwrap_or(false) }) - .ok_or("Input device not found")?, + .ok_or_else(|| "Input device not found".to_string())?, + None => host .default_input_device() - .ok_or("No default input device")?, + .ok_or_else(|| "No default input device".to_string())?, }; let output_device = match &config.output_device_name { Some(name) => host .output_devices() .map_err(|e| e.to_string())? - .find(|d| { - d.description() + .find(|device| { + device + .description() .ok() - .map(|d| d.name() == name) + .map(|description| description.name() == name) .unwrap_or(false) }) - .ok_or("Output device not found")?, + .ok_or_else(|| "Output device not found".to_string())?, + None => host .default_output_device() - .ok_or("No default output device")?, + .ok_or_else(|| "No default output device".to_string())?, }; - // ---------- INPUT (mic -> UDP) ---------- + // ======================================================== + // SHARED PLAYBACK STATE + // ======================================================== + + // + // IMPORTANT: + // + // This queue contains samples already converted to the + // CURRENT output device's channel layout. + // + // Therefore: + // + // stereo device: + // L R L R L R ... + // + // mono device: + // M M M M ... + // + let playback_buffer = Arc::new(Mutex::new(VecDeque::::new())); + + let needs_output_rebuild = Arc::new(AtomicBool::new(false)); + + let shutdown = Arc::new(AtomicBool::new(false)); + + let playback_started = Arc::new(AtomicBool::new(false)); + + // ======================================================== + // INPUT: MICROPHONE -> UDP + // ======================================================== let input_socket = socket.clone(); + let mut input_config: cpal::StreamConfig = input_device .default_input_config() - .map_err(|e| e.to_string())? + .map_err(|e| format!("Failed to get input config: {e}"))? .into(); + // We explicitly capture mono. input_config.channels = 1; + let input_sample_rate = input_config.sample_rate; + eprintln!( + "[vc] input: {} Hz -> {} Hz", + input_sample_rate, NETWORK_SAMPLE_RATE + ); + let mut input_resampler = resampler::ResamplerFft::new( 1, - input_sample_rate.try_into().map_err(|e| format!("{e:?}"))?, + input_sample_rate + .try_into() + .map_err(|e| format!("Invalid input sample rate: {e:?}"))?, resampler::SampleRate::Hz44100, ); let mut input_buffer = Vec::::new(); + // Resampled samples waiting to form a network packet. + let mut packet_buffer = Vec::::with_capacity(NETWORK_PACKET_SAMPLES * 2); + let input_stream = input_device .build_input_stream( input_config, @@ -153,190 +282,390 @@ pub fn connect_to_vc( while input_buffer.len() >= frame_size { let input: Vec = input_buffer.drain(..frame_size).collect(); - let mut output = vec![0.0f32; input_resampler.chunk_size_output()]; + let output_size = input_resampler.chunk_size_output(); + + let mut output = vec![0.0f32; output_size]; if let Err(e) = input_resampler.resample(&input, &mut output) { eprintln!("[vc] failed to resample input: {e}"); continue; } - // f32 -> i16 PCM - let mut packet = Vec::with_capacity(output.len() * 2); + packet_buffer.extend(output.into_iter().map(|sample| sample.clamp(-1.0, 1.0))); - for sample in output { - let sample = sample.clamp(-1.0, 1.0); - let pcm = (sample * i16::MAX as f32) as i16; + // ------------------------------------------------ + // Form exact 20ms network packets. + // ------------------------------------------------ - packet.extend_from_slice(&pcm.to_be_bytes()); - } + while packet_buffer.len() >= NETWORK_PACKET_SAMPLES { + let packet_samples: Vec = + packet_buffer.drain(..NETWORK_PACKET_SAMPLES).collect(); - eprintln!( - "[vc] sending audio: input={} samples, packet={} bytes", - frame_size, - packet.len() - ); + let mut packet = Vec::with_capacity(NETWORK_PACKET_SAMPLES * 2); - match input_socket.send(&packet) { - Ok(n) => eprintln!("[vc] UDP sent {n} bytes"), - Err(e) => eprintln!("[vc] UDP send failed: {e}"), + for sample in packet_samples { + let pcm = (sample * i16::MAX as f32) as i16; + + packet.extend_from_slice(&pcm.to_be_bytes()); + } + + match input_socket.send(&packet) { + Ok(_) => {} + + Err(e) => { + eprintln!("[vc] UDP send failed: {e}"); + } + } } } }, - |err| eprintln!("[vc] input stream error: {err}"), + |err| { + eprintln!("[vc] input stream error: {err}"); + }, None, ) .map_err(|e| e.to_string())?; - // ---------- OUTPUT (UDP -> speakers), rebuildable on device change ---------- + // ======================================================== + // OUTPUT CONFIG + // ======================================================== - let playback_buffer = Arc::new(Mutex::new(Vec::::new())); - let needs_output_rebuild = Arc::new(AtomicBool::new(false)); - let shutdown = Arc::new(AtomicBool::new(false)); - - // Output config is tracked in a shared cell so the UDP-receiving thread - // (below) always resamples toward whatever the *current* live stream expects, - // even after a rebuild changes the device's rate. let output_config = build_output_config(&output_device)?; + + eprintln!( + "[vc] output: {} Hz / {} channels", + output_config.sample_rate, output_config.channels + ); + let output_config_cell = Arc::new(Mutex::new(output_config.clone())); + // ======================================================== + // OUTPUT STREAM + // ======================================================== + let initial_output_stream = build_output_stream( &output_device, &output_config, playback_buffer.clone(), needs_output_rebuild.clone(), + playback_started.clone(), )?; - initial_output_stream.play().map_err(|e| e.to_string())?; + + initial_output_stream + .play() + .map_err(|e| format!("Failed to start output stream: {e}"))?; let output_stream = Arc::new(Mutex::new(initial_output_stream)); - // Watcher thread: rebuilds the output stream whenever the device signals - // its config has changed (e.g. "Device sample rate changed"), since cpal - // streams can't be reconfigured in place — only rebuilt from scratch. + // ======================================================== + // OUTPUT DEVICE WATCHER + // ======================================================== + { let output_device = output_device.clone(); + let needs_output_rebuild = needs_output_rebuild.clone(); + let output_stream = output_stream.clone(); + let output_config_cell = output_config_cell.clone(); + let shutdown = shutdown.clone(); + let playback_buffer = playback_buffer.clone(); - std::thread::spawn(move || loop { - if shutdown.load(Ordering::SeqCst) { - return; - } + let playback_started = playback_started.clone(); - std::thread::sleep(std::time::Duration::from_millis(200)); + thread::spawn(move || { + while !shutdown.load(Ordering::SeqCst) { + thread::sleep(Duration::from_millis(100)); - if needs_output_rebuild.swap(false, Ordering::SeqCst) { - match build_output_config(&output_device).and_then(|new_config| { + if shutdown.load(Ordering::SeqCst) { + return; + } + + if !needs_output_rebuild.swap(false, Ordering::SeqCst) { + continue; + } + + eprintln!("[vc] rebuilding output stream..."); + + let result = build_output_config(&output_device).and_then(|new_config| { build_output_stream( &output_device, &new_config, playback_buffer.clone(), needs_output_rebuild.clone(), + playback_started.clone(), ) .map(|stream| (new_config, stream)) - }) { + }); + + match result { Ok((new_config, new_stream)) => { if let Err(e) = new_stream.play() { - eprintln!("[vc] failed to start rebuilt output stream: {e}"); + eprintln!( + "[vc] failed to start rebuilt \ + output stream: {e}" + ); + + needs_output_rebuild.store(true, Ordering::SeqCst); + continue; } - *output_config_cell.lock().unwrap() = new_config; + + // Clear stale audio because it was generated + // for the old output timing/channel layout. + playback_buffer.lock().unwrap().clear(); + + playback_started.store(false, Ordering::SeqCst); + + *output_config_cell.lock().unwrap() = new_config.clone(); + *output_stream.lock().unwrap() = new_stream; - eprintln!("[vc] output stream rebuilt after device change"); + + eprintln!( + "[vc] output rebuilt: {} Hz / {} channels", + new_config.sample_rate, new_config.channels + ); + } + + Err(e) => { + eprintln!("[vc] failed to rebuild output: {e}"); + + needs_output_rebuild.store(true, Ordering::SeqCst); } - Err(e) => eprintln!("[vc] failed to rebuild output stream: {e}"), } } }); } - // ---------- UDP receive thread: incoming audio -> resample -> playback channel ---------- + // ======================================================== + // UDP RECEIVE -> RESAMPLE -> PLAYBACK QUEUE + // ======================================================== - let recv_socket = socket.clone(); { + let recv_socket = socket.clone(); + let output_config_cell = output_config_cell.clone(); + + let playback_buffer = playback_buffer.clone(); + + let playback_started = playback_started.clone(); + let shutdown = shutdown.clone(); - std::thread::spawn(move || { - let mut buf = [0u8; 4096]; - let mut output_buffer = Vec::::new(); + thread::spawn(move || { + let mut buf = [0u8; MAX_UDP_PACKET_SIZE]; - let mut output_resampler = resampler::ResamplerFft::new( - 1, - resampler::SampleRate::Hz44100, - output_config_cell - .lock() - .unwrap() - .sample_rate - .try_into() - .unwrap(), - ); + let initial_config = output_config_cell.lock().unwrap().clone(); - let mut last_rate = output_config_cell.lock().unwrap().sample_rate; + let mut last_sample_rate = initial_config.sample_rate; + + let mut output_resampler = match create_output_resampler(initial_config.sample_rate) { + Ok(resampler) => resampler, + + Err(e) => { + eprintln!( + "[vc] failed to create output \ + resampler: {e}" + ); + + return; + } + }; + + // Audio waiting to be fed into the resampler. + let mut resample_input = Vec::::new(); loop { if shutdown.load(Ordering::SeqCst) { return; } - let Ok(len) = recv_socket.recv(&mut buf) else { - continue; - }; + // ---------------------------------------------------- + // Receive UDP packet. + // ---------------------------------------------------- - // UDP contains BIG-ENDIAN i16 PCM. - let pcm = buf[..len] - .chunks_exact(2) - .map(|b| { - let sample = i16::from_be_bytes([b[0], b[1]]); - sample as f32 / i16::MAX as f32 - }) - .collect::>(); + let len = match recv_socket.recv(&mut buf) { + Ok(len) => len, - output_buffer.extend_from_slice(&pcm); - - let current_config = output_config_cell.lock().unwrap().clone(); - - if current_config.sample_rate != last_rate { - output_resampler = resampler::ResamplerFft::new( - 1, - resampler::SampleRate::Hz44100, - match current_config.sample_rate.try_into() { - Ok(rate) => rate, - Err(_) => { - eprintln!("[vc] unsupported output sample rate"); - last_rate = current_config.sample_rate; - continue; - } - }, - ); - - last_rate = current_config.sample_rate; - } - - let frame_size = output_resampler.chunk_size_input(); - - while output_buffer.len() >= frame_size { - let input: Vec = output_buffer.drain(..frame_size).collect(); - - let mut pcm_out = vec![0.0f32; output_resampler.chunk_size_output()]; - - if let Err(e) = output_resampler.resample(&input, &mut pcm_out) { - eprintln!("[vc] failed to resample output: {e}"); + Err(e) + if e.kind() == std::io::ErrorKind::WouldBlock + || e.kind() == std::io::ErrorKind::TimedOut => + { continue; } - // Mono network audio -> device channel layout. - let output = mono_to_output_channels(&pcm_out, current_config.channels); + Err(e) => { + if !shutdown.load(Ordering::SeqCst) { + eprintln!("[vc] UDP receive failed: {e}"); + } - playback_buffer.lock().unwrap().extend(output); + continue; + } + }; + + // ---------------------------------------------------- + // Ignore malformed packets. + // + // Every audio packet must contain complete i16 + // samples. + // ---------------------------------------------------- + + if len < 2 { + continue; + } + + let usable_len = len - (len % 2); + + // ---------------------------------------------------- + // BIG-ENDIAN i16 -> f32 + // ---------------------------------------------------- + + for chunk in buf[..usable_len].chunks_exact(2) { + let pcm = i16::from_be_bytes([chunk[0], chunk[1]]); + + resample_input.push(pcm as f32 / i16::MAX as f32); + } + + // ---------------------------------------------------- + // Check current output device configuration. + // ---------------------------------------------------- + + let current_config = output_config_cell.lock().unwrap().clone(); + + // ---------------------------------------------------- + // Output sample rate changed. + // + // Recreate the resampler and discard samples from the + // old timing domain. + // ---------------------------------------------------- + + if current_config.sample_rate != last_sample_rate { + eprintln!( + "[vc] output rate changed: {} -> {}", + last_sample_rate, current_config.sample_rate + ); + + match create_output_resampler(current_config.sample_rate) { + Ok(new_resampler) => { + output_resampler = new_resampler; + + last_sample_rate = current_config.sample_rate; + + resample_input.clear(); + + playback_buffer.lock().unwrap().clear(); + + playback_started.store(false, Ordering::SeqCst); + } + + Err(e) => { + eprintln!( + "[vc] unsupported output \ + sample rate {}: {e}", + current_config.sample_rate + ); + + resample_input.clear(); + + continue; + } + } + } + + // ---------------------------------------------------- + // Resample incoming 44.1kHz mono audio into the + // output device's sample rate. + // ---------------------------------------------------- + + let frame_size = output_resampler.chunk_size_input(); + + while resample_input.len() >= frame_size { + let input: Vec = resample_input.drain(..frame_size).collect(); + + let output_size = output_resampler.chunk_size_output(); + + let mut resampled = vec![0.0f32; output_size]; + + if let Err(e) = output_resampler.resample(&input, &mut resampled) { + eprintln!( + "[vc] failed to resample \ + output: {e}" + ); + + continue; + } + + // ------------------------------------------------ + // Mono -> device channels. + // ------------------------------------------------ + + let output = mono_to_output_channels(&resampled, current_config.channels); + + // ------------------------------------------------ + // Push into bounded playback queue. + // ------------------------------------------------ + + let mut queue = playback_buffer.lock().unwrap(); + + queue.extend(output); + + let channels = current_config.channels.max(1) as usize; + + let max_frames = current_config.sample_rate as usize * MAX_BUFFER_MS / 1000; + + let max_samples = max_frames * channels; + + // ------------------------------------------------ + // If we have accumulated too much audio, discard + // OLD audio. + // + // This is critical for voice chat latency. + // ------------------------------------------------ + + while queue.len() > max_samples { + queue.pop_front(); + } + + // ------------------------------------------------ + // Start playback only after we have a small + // amount of audio buffered. + // ------------------------------------------------ + + if !playback_started.load(Ordering::SeqCst) { + let prebuffer_frames = + current_config.sample_rate as usize * PREBUFFER_MS / 1000; + + let prebuffer_samples = prebuffer_frames * channels; + + if queue.len() >= prebuffer_samples { + playback_started.store(true, Ordering::SeqCst); + + eprintln!( + "[vc] playback started \ + with ~{}ms buffered", + PREBUFFER_MS + ); + } + } } } }); } - input_stream.play().map_err(|e| e.to_string())?; + // ======================================================== + // START INPUT + // ======================================================== + + input_stream + .play() + .map_err(|e| format!("Failed to start input stream: {e}"))?; + + // ======================================================== + // STORE SESSION + // ======================================================== *voice_state.session.lock().unwrap() = Some(VoiceSession { input_stream, @@ -349,42 +678,91 @@ pub fn connect_to_vc( Ok(()) } +// ============================================================ +// OUTPUT CONFIG +// ============================================================ + fn build_output_config(output_device: &cpal::Device) -> Result { - Ok(output_device + output_device .default_output_config() - .map_err(|e| e.to_string())? - .into()) + .map_err(|e| e.to_string()) + .map(Into::into) } +// ============================================================ +// OUTPUT RESAMPLER +// ============================================================ + +fn create_output_resampler( + output_sample_rate: cpal::SampleRate, +) -> Result { + let output_rate = output_sample_rate + .try_into() + .map_err(|e| format!("Invalid output sample rate: {e:?}"))?; + + Ok(resampler::ResamplerFft::new( + 1, + resampler::SampleRate::Hz44100, + output_rate, + )) +} + +// ============================================================ +// BUILD OUTPUT STREAM +// ============================================================ + fn build_output_stream( output_device: &cpal::Device, output_config: &cpal::StreamConfig, - playback_buffer: Arc>>, + playback_buffer: Arc>>, needs_rebuild: Arc, + playback_started: Arc, ) -> Result { output_device .build_output_stream( output_config.clone(), + // ==================================================== + // CPAL OUTPUT CALLBACK + // ==================================================== move |data: &mut [f32], _| { - let mut buffer = playback_buffer.lock().unwrap(); + let started = playback_started.load(Ordering::Acquire); - for out in data.iter_mut() { - if buffer.is_empty() { - *out = 0.0; - } else { - *out = buffer.remove(0); - } + if !started { + // Do NOT consume audio before the prebuffer is + // ready. Just output silence. + data.fill(0.0); + return; + } + + let mut queue = playback_buffer.lock().unwrap(); + + // VecDeque::pop_front() is O(1). + // + // This is massively better than: + // + // Vec::remove(0) + // + // which shifts the entire vector every sample. + + for sample in data.iter_mut() { + *sample = queue.pop_front().unwrap_or(0.0); } }, + // ==================================================== + // OUTPUT ERROR CALLBACK + // ==================================================== { let needs_rebuild = needs_rebuild.clone(); move |err| { eprintln!("[vc] output stream error: {err}"); - let msg = err.to_string(); + let message = err.to_string(); - if msg.contains("sample rate changed") || msg.contains("DeviceNotAvailable") { + if message.contains("sample rate changed") + || message.contains("DeviceNotAvailable") + || message.contains("device not available") + { needs_rebuild.store(true, Ordering::SeqCst); } } @@ -394,26 +772,17 @@ fn build_output_stream( .map_err(|e| e.to_string()) } -fn stereo_to_mono(stereo_data: &[f32]) -> Vec { - let mut mono_data = Vec::with_capacity(stereo_data.len() / 2); - - for chunk in stereo_data.chunks_exact(2) { - let left = chunk[0]; - let right = chunk[1]; - let mono_sample = (left + right) / 2.0; - mono_data.push(mono_sample); - } - - mono_data -} +// ============================================================ +// MONO -> OUTPUT CHANNELS +// ============================================================ fn mono_to_output_channels(mono: &[f32], channels: u16) -> Vec { - if channels <= 1 { + let channels = channels.max(1) as usize; + + if channels == 1 { return mono.to_vec(); } - let channels = channels as usize; - let mut output = Vec::with_capacity(mono.len() * channels); for &sample in mono { @@ -424,3 +793,17 @@ fn mono_to_output_channels(mono: &[f32], channels: u16) -> Vec { output } + +// ============================================================ +// OPTIONAL UTILITY +// ============================================================ + +fn stereo_to_mono(stereo_data: &[f32]) -> Vec { + let mut mono = Vec::with_capacity(stereo_data.len() / 2); + + for chunk in stereo_data.chunks_exact(2) { + mono.push((chunk[0] + chunk[1]) * 0.5); + } + + mono +} From b88e17109a13fb15a58f21f607342937e8a482ec Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 20:17:02 +0200 Subject: [PATCH 16/29] Better audio stuff --- src-tauri/src/commands/audio.rs | 760 +++++++++++++++++++++----------- 1 file changed, 503 insertions(+), 257 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index e663822..1e3bc7e 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -1,12 +1,12 @@ use std::{ - collections::VecDeque, + collections::{BTreeMap, VecDeque}, net::UdpSocket, sync::{ atomic::{AtomicBool, Ordering}, Arc, Mutex, }, thread, - time::Duration, + time::{Duration, Instant}, }; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; @@ -14,45 +14,56 @@ use tauri::State; use crate::commands::config::ConfigState; -// ============================================================ -// AUDIO CONSTANTS -// ============================================================ +// ============================================================================ +// AUDIO FORMAT +// ============================================================================ -/// Network audio format. -/// -/// Everything sent over UDP is: -/// 44100 Hz -/// mono -/// signed i16 -/// big endian -/// -/// 20 ms @ 44100 Hz = 882 samples. -/// 882 * 2 = 1764 bytes. const NETWORK_SAMPLE_RATE: u32 = 44_100; + +// 20ms packets. +// +// 44100 * 0.020 = 882 samples const PACKET_DURATION_MS: usize = 20; const NETWORK_PACKET_SAMPLES: usize = NETWORK_SAMPLE_RATE as usize * PACKET_DURATION_MS / 1000; -/// Amount of audio that must be queued before playback begins. -const PREBUFFER_MS: usize = 40; +// PCM is i16 => 2 bytes/sample. +const NETWORK_PACKET_BYTES: usize = NETWORK_PACKET_SAMPLES * 2; -/// If the playback queue gets below this amount, we allow it to -/// continue normally but the callback will output silence if it -/// actually runs dry. -const LOW_WATERMARK_MS: usize = 20; +// Sequence number is a u32. +const AUDIO_HEADER_BYTES: usize = 4; -/// Maximum amount of queued audio. -/// -/// If this is exceeded, OLD audio is discarded. This is intentional: -/// for voice chat, dropping old audio is much better than accumulating -/// hundreds of milliseconds/seconds of latency. -const MAX_BUFFER_MS: usize = 100; - -/// Maximum UDP packet size we expect. +// Maximum UDP packet we accept. const MAX_UDP_PACKET_SIZE: usize = 4096; -// ============================================================ -// STATE -// ============================================================ +// ============================================================================ +// JITTER BUFFER +// ============================================================================ +// +// We intentionally keep this relatively small. +// +// Increasing these values reduces underruns but increases latency. +// +// Current target: +// +// startup: 40ms +// normal target: 40ms +// maximum: 100ms +// +// This is appropriate for low-latency voice. +// + +const JITTER_TARGET_MS: usize = 40; +const JITTER_MAX_MS: usize = 100; + +// If a packet is missing, wait this long before considering it lost. +// +// Since packets are 20ms, 30ms gives us enough room for modest +// out-of-order delivery without making latency enormous. +const PACKET_LOSS_WAIT_MS: u64 = 30; + +// ============================================================================ +// VOICE STATE +// ============================================================================ pub struct VoiceState { pub session: Mutex>, @@ -61,8 +72,6 @@ pub struct VoiceState { pub struct VoiceSession { pub input_stream: cpal::Stream, - // Wrapped so the watcher thread can replace the stream after - // the output device changes configuration. pub output_stream: Arc>, pub socket: Arc, @@ -80,9 +89,9 @@ impl Default for VoiceState { } } -// ============================================================ -// DEVICE ENUMERATION -// ============================================================ +// ============================================================================ +// DEVICE LISTING +// ============================================================================ #[tauri::command] pub fn list_input_devices() -> Result, String> { @@ -110,27 +119,22 @@ pub fn list_output_devices() -> Result, String> { .collect()) } -// ============================================================ +// ============================================================================ // DISCONNECT -// ============================================================ +// ============================================================================ #[tauri::command] pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { if let Some(session) = voice_state.session.lock().unwrap().take() { session.shutdown.store(true, Ordering::SeqCst); - - // Streams are dropped here. - // - // The UDP receiver has a 100ms read timeout, so it will notice - // shutdown shortly instead of remaining blocked forever. } Ok(()) } -// ============================================================ +// ============================================================================ // CONNECT -// ============================================================ +// ============================================================================ #[tauri::command] pub fn connect_to_vc( @@ -143,9 +147,9 @@ pub fn connect_to_vc( let config = config_state.0.lock().unwrap().clone(); - // ======================================================== - // UDP SOCKET - // ======================================================== + // ======================================================================== + // UDP + // ======================================================================== let socket = UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?; @@ -154,24 +158,24 @@ pub fn connect_to_vc( .connect(&hostname) .map_err(|e| format!("Failed to connect UDP socket: {e}"))?; - // IMPORTANT: - // - // Without a timeout, recv() can remain blocked forever and the - // receiver thread can survive after disconnect. + // Required so disconnect can terminate the receiver thread. socket .set_read_timeout(Some(Duration::from_millis(100))) - .map_err(|e| format!("Failed to configure UDP timeout: {e}"))?; + .map_err(|e| format!("Failed to set UDP read timeout: {e}"))?; let socket = Arc::new(socket); - // PIN is always the first packet. + // ======================================================================== + // AUTHENTICATION + // ======================================================================== + socket .send(&pin.to_be_bytes()) .map_err(|e| format!("Failed to send pincode: {e}"))?; - // ======================================================== - // AUDIO DEVICES - // ======================================================== + // ======================================================================== + // DEVICES + // ======================================================================== let host = cpal::default_host(); @@ -211,35 +215,42 @@ pub fn connect_to_vc( .ok_or_else(|| "No default output device".to_string())?, }; - // ======================================================== - // SHARED PLAYBACK STATE - // ======================================================== - - // - // IMPORTANT: - // - // This queue contains samples already converted to the - // CURRENT output device's channel layout. - // - // Therefore: - // - // stereo device: - // L R L R L R ... - // - // mono device: - // M M M M ... - // - let playback_buffer = Arc::new(Mutex::new(VecDeque::::new())); - - let needs_output_rebuild = Arc::new(AtomicBool::new(false)); + // ======================================================================== + // SHUTDOWN + // ======================================================================== let shutdown = Arc::new(AtomicBool::new(false)); + // ======================================================================== + // PLAYBACK BUFFER + // ======================================================================== + // + // This contains samples already converted to the output device's + // channel layout. + // + + let playback_buffer = Arc::new(Mutex::new(VecDeque::::new())); + let playback_started = Arc::new(AtomicBool::new(false)); - // ======================================================== - // INPUT: MICROPHONE -> UDP - // ======================================================== + // ======================================================================== + // OUTPUT CONFIG + // ======================================================================== + + let output_config = build_output_config(&output_device)?; + + eprintln!( + "[vc] output: {} Hz / {} channels", + output_config.sample_rate, output_config.channels + ); + + let output_config_cell = Arc::new(Mutex::new(output_config.clone())); + + let needs_output_rebuild = Arc::new(AtomicBool::new(false)); + + // ======================================================================== + // INPUT + // ======================================================================== let input_socket = socket.clone(); @@ -248,7 +259,7 @@ pub fn connect_to_vc( .map_err(|e| format!("Failed to get input config: {e}"))? .into(); - // We explicitly capture mono. + // Network audio is mono. input_config.channels = 1; let input_sample_rate = input_config.sample_rate; @@ -260,16 +271,22 @@ pub fn connect_to_vc( let mut input_resampler = resampler::ResamplerFft::new( 1, - input_sample_rate - .try_into() - .map_err(|e| format!("Invalid input sample rate: {e:?}"))?, + input_sample_rate.try_into().map_err(|e| { + format!( + "Invalid input sample rate: \ + {e:?}" + ) + })?, resampler::SampleRate::Hz44100, ); let mut input_buffer = Vec::::new(); - // Resampled samples waiting to form a network packet. - let mut packet_buffer = Vec::::with_capacity(NETWORK_PACKET_SAMPLES * 2); + // Resampled samples waiting to form 20ms packets. + let mut packet_samples = Vec::::with_capacity(NETWORK_PACKET_SAMPLES * 2); + + // Sequence number for every audio packet. + let mut sequence: u32 = 0; let input_stream = input_device .build_input_stream( @@ -287,35 +304,50 @@ pub fn connect_to_vc( let mut output = vec![0.0f32; output_size]; if let Err(e) = input_resampler.resample(&input, &mut output) { - eprintln!("[vc] failed to resample input: {e}"); + eprintln!( + "[vc] input resample \ + failed: {e}" + ); + continue; } - packet_buffer.extend(output.into_iter().map(|sample| sample.clamp(-1.0, 1.0))); + packet_samples.extend(output.into_iter().map(|sample| sample.clamp(-1.0, 1.0))); - // ------------------------------------------------ - // Form exact 20ms network packets. - // ------------------------------------------------ + // ======================================================== + // CREATE EXACT 20ms PACKETS + // ======================================================== - while packet_buffer.len() >= NETWORK_PACKET_SAMPLES { - let packet_samples: Vec = - packet_buffer.drain(..NETWORK_PACKET_SAMPLES).collect(); + while packet_samples.len() >= NETWORK_PACKET_SAMPLES { + let samples: Vec = + packet_samples.drain(..NETWORK_PACKET_SAMPLES).collect(); - let mut packet = Vec::with_capacity(NETWORK_PACKET_SAMPLES * 2); + // Header: + // + // [u32 sequence] + // + // Then: + // + // [i16 PCM...] + let mut packet = + Vec::with_capacity(AUDIO_HEADER_BYTES + NETWORK_PACKET_BYTES); - for sample in packet_samples { - let pcm = (sample * i16::MAX as f32) as i16; + packet.extend_from_slice(&sequence.to_be_bytes()); + + for sample in samples { + let pcm = (sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; packet.extend_from_slice(&pcm.to_be_bytes()); } - match input_socket.send(&packet) { - Ok(_) => {} - - Err(e) => { - eprintln!("[vc] UDP send failed: {e}"); - } + if let Err(e) = input_socket.send(&packet) { + eprintln!( + "[vc] UDP send failed: \ + {e}" + ); } + + sequence = sequence.wrapping_add(1); } } }, @@ -326,22 +358,9 @@ pub fn connect_to_vc( ) .map_err(|e| e.to_string())?; - // ======================================================== - // OUTPUT CONFIG - // ======================================================== - - let output_config = build_output_config(&output_device)?; - - eprintln!( - "[vc] output: {} Hz / {} channels", - output_config.sample_rate, output_config.channels - ); - - let output_config_cell = Arc::new(Mutex::new(output_config.clone())); - - // ======================================================== + // ======================================================================== // OUTPUT STREAM - // ======================================================== + // ======================================================================== let initial_output_stream = build_output_stream( &output_device, @@ -357,9 +376,9 @@ pub fn connect_to_vc( let output_stream = Arc::new(Mutex::new(initial_output_stream)); - // ======================================================== + // ======================================================================== // OUTPUT DEVICE WATCHER - // ======================================================== + // ======================================================================== { let output_device = output_device.clone(); @@ -370,12 +389,12 @@ pub fn connect_to_vc( let output_config_cell = output_config_cell.clone(); - let shutdown = shutdown.clone(); - let playback_buffer = playback_buffer.clone(); let playback_started = playback_started.clone(); + let shutdown = shutdown.clone(); + thread::spawn(move || { while !shutdown.load(Ordering::SeqCst) { thread::sleep(Duration::from_millis(100)); @@ -405,8 +424,8 @@ pub fn connect_to_vc( Ok((new_config, new_stream)) => { if let Err(e) = new_stream.play() { eprintln!( - "[vc] failed to start rebuilt \ - output stream: {e}" + "[vc] failed to start \ + rebuilt output: {e}" ); needs_output_rebuild.store(true, Ordering::SeqCst); @@ -414,8 +433,10 @@ pub fn connect_to_vc( continue; } - // Clear stale audio because it was generated - // for the old output timing/channel layout. + // The old output format is no longer valid. + // + // Throw away queued samples rather than playing + // them using the new device timing/layout. playback_buffer.lock().unwrap().clear(); playback_started.store(false, Ordering::SeqCst); @@ -425,13 +446,17 @@ pub fn connect_to_vc( *output_stream.lock().unwrap() = new_stream; eprintln!( - "[vc] output rebuilt: {} Hz / {} channels", + "[vc] output rebuilt: \ + {} Hz / {} channels", new_config.sample_rate, new_config.channels ); } Err(e) => { - eprintln!("[vc] failed to rebuild output: {e}"); + eprintln!( + "[vc] failed to rebuild \ + output: {e}" + ); needs_output_rebuild.store(true, Ordering::SeqCst); } @@ -440,9 +465,9 @@ pub fn connect_to_vc( }); } - // ======================================================== - // UDP RECEIVE -> RESAMPLE -> PLAYBACK QUEUE - // ======================================================== + // ======================================================================== + // UDP RECEIVE + JITTER BUFFER + // ======================================================================== { let recv_socket = socket.clone(); @@ -456,7 +481,20 @@ pub fn connect_to_vc( let shutdown = shutdown.clone(); thread::spawn(move || { - let mut buf = [0u8; MAX_UDP_PACKET_SIZE]; + // ================================================================ + // PACKET REORDER BUFFER + // ================================================================ + + let mut packets: BTreeMap> = BTreeMap::new(); + + let mut expected_sequence: Option = None; + + // When we first notice a missing packet, remember when. + let mut missing_since: Option = None; + + // ================================================================ + // RESAMPLER + // ================================================================ let initial_config = output_config_cell.lock().unwrap().clone(); @@ -467,84 +505,154 @@ pub fn connect_to_vc( Err(e) => { eprintln!( - "[vc] failed to create output \ - resampler: {e}" + "[vc] failed to create \ + output resampler: {e}" ); return; } }; - // Audio waiting to be fed into the resampler. let mut resample_input = Vec::::new(); - loop { - if shutdown.load(Ordering::SeqCst) { - return; - } + // ================================================================ + // UDP BUFFER + // ================================================================ - // ---------------------------------------------------- - // Receive UDP packet. - // ---------------------------------------------------- + let mut udp_buffer = [0u8; MAX_UDP_PACKET_SIZE]; - let len = match recv_socket.recv(&mut buf) { - Ok(len) => len, + // ================================================================ + // MAIN LOOP + // ================================================================ + + while !shutdown.load(Ordering::SeqCst) { + // ============================================================ + // RECEIVE PACKET + // ============================================================ + + match recv_socket.recv(&mut udp_buffer) { + Ok(len) => { + if len <= AUDIO_HEADER_BYTES { + continue; + } + + // ---------------------------------------------------- + // Read sequence number. + // ---------------------------------------------------- + + let sequence = u32::from_be_bytes([ + udp_buffer[0], + udp_buffer[1], + udp_buffer[2], + udp_buffer[3], + ]); + + let pcm_bytes = &udp_buffer[AUDIO_HEADER_BYTES..len]; + + // Must contain complete i16 samples. + let pcm_len = pcm_bytes.len() & !1; + + if pcm_len == 0 { + continue; + } + + let mut samples = Vec::with_capacity(pcm_len / 2); + + for chunk in pcm_bytes[..pcm_len].chunks_exact(2) { + let pcm = i16::from_be_bytes([chunk[0], chunk[1]]); + + samples.push(pcm as f32 / i16::MAX as f32); + } + + // ---------------------------------------------------- + // Ignore packets that are already too old. + // ---------------------------------------------------- + + if let Some(expected) = expected_sequence { + if sequence_before(sequence, expected) { + continue; + } + } + + // ---------------------------------------------------- + // Insert into jitter buffer. + // + // BTreeMap automatically keeps sequence numbers + // ordered. + // ---------------------------------------------------- + + packets.entry(sequence).or_insert(samples); + + // ---------------------------------------------------- + // Don't allow the packet jitter buffer itself to + // become a source of latency. + // ---------------------------------------------------- + + while packets.len() > 8 { + if let Some((&oldest, _)) = packets.iter().next() { + if let Some(expected) = expected_sequence { + if sequence_before(oldest, expected) { + packets.remove(&oldest); + } else { + break; + } + } else { + break; + } + } + } + } Err(e) - if e.kind() == std::io::ErrorKind::WouldBlock - || e.kind() == std::io::ErrorKind::TimedOut => + if e.kind() == std::io::ErrorKind::TimedOut + || e.kind() == std::io::ErrorKind::WouldBlock => { - continue; + // Timeout is expected. It gives us a chance to + // process jitter-buffer timeouts and shutdown. } Err(e) => { if !shutdown.load(Ordering::SeqCst) { - eprintln!("[vc] UDP receive failed: {e}"); + eprintln!( + "[vc] UDP receive failed: \ + {e}" + ); } continue; } - }; - - // ---------------------------------------------------- - // Ignore malformed packets. - // - // Every audio packet must contain complete i16 - // samples. - // ---------------------------------------------------- - - if len < 2 { - continue; } - let usable_len = len - (len % 2); + // ============================================================ + // INITIALIZE EXPECTED SEQUENCE + // ============================================================ - // ---------------------------------------------------- - // BIG-ENDIAN i16 -> f32 - // ---------------------------------------------------- + if expected_sequence.is_none() { + if let Some((&first_sequence, _)) = packets.iter().next() { + expected_sequence = Some(first_sequence); - for chunk in buf[..usable_len].chunks_exact(2) { - let pcm = i16::from_be_bytes([chunk[0], chunk[1]]); - - resample_input.push(pcm as f32 / i16::MAX as f32); + eprintln!( + "[vc] jitter buffer \ + synchronized at packet {}", + first_sequence + ); + } } - // ---------------------------------------------------- - // Check current output device configuration. - // ---------------------------------------------------- + // ============================================================ + // CURRENT OUTPUT CONFIG + // ============================================================ let current_config = output_config_cell.lock().unwrap().clone(); - // ---------------------------------------------------- - // Output sample rate changed. - // - // Recreate the resampler and discard samples from the - // old timing domain. - // ---------------------------------------------------- + // ============================================================ + // OUTPUT DEVICE SAMPLE RATE CHANGE + // ============================================================ if current_config.sample_rate != last_sample_rate { eprintln!( - "[vc] output rate changed: {} -> {}", + "[vc] output rate changed: \ + {} -> {}", last_sample_rate, current_config.sample_rate ); @@ -554,20 +662,26 @@ pub fn connect_to_vc( last_sample_rate = current_config.sample_rate; + packets.clear(); + resample_input.clear(); playback_buffer.lock().unwrap().clear(); playback_started.store(false, Ordering::SeqCst); + + expected_sequence = None; + + missing_since = None; } Err(e) => { eprintln!( - "[vc] unsupported output \ - sample rate {}: {e}", - current_config.sample_rate + "[vc] failed to create \ + output resampler: {e}" ); + packets.clear(); resample_input.clear(); continue; @@ -575,97 +689,200 @@ pub fn connect_to_vc( } } - // ---------------------------------------------------- - // Resample incoming 44.1kHz mono audio into the - // output device's sample rate. - // ---------------------------------------------------- + // ============================================================ + // MOVE READY PACKETS INTO RESAMPLER + // ============================================================ - let frame_size = output_resampler.chunk_size_input(); + loop { + let Some(expected) = expected_sequence else { + break; + }; - while resample_input.len() >= frame_size { - let input: Vec = resample_input.drain(..frame_size).collect(); + // -------------------------------------------------------- + // Expected packet exists. + // -------------------------------------------------------- - let output_size = output_resampler.chunk_size_output(); + if let Some(samples) = packets.remove(&expected) { + resample_input.extend_from_slice(&samples); - let mut resampled = vec![0.0f32; output_size]; + expected_sequence = Some(expected.wrapping_add(1)); + + missing_since = None; + + continue; + } + + // -------------------------------------------------------- + // Expected packet doesn't exist. + // + // If we don't have anything newer, there is nothing + // to do yet. + // -------------------------------------------------------- + + let has_newer_packet = packets.keys().any(|&seq| sequence_after(seq, expected)); + + if !has_newer_packet { + break; + } + + // -------------------------------------------------------- + // We have a later packet. + // + // Therefore the expected packet is either delayed or + // lost. + // + // Wait a short period before declaring it lost. + // -------------------------------------------------------- + + let now = Instant::now(); + + let since = missing_since.get_or_insert(now); + + if since.elapsed() < Duration::from_millis(PACKET_LOSS_WAIT_MS) { + break; + } + + // -------------------------------------------------------- + // Packet is considered lost. + // + // We insert a zero packet here. + // + // Because the missing packet is exactly 20ms, this + // results in a controlled 20ms gap rather than the + // playback clock getting permanently stuck. + // -------------------------------------------------------- + + resample_input.extend(std::iter::repeat(0.0f32).take(NETWORK_PACKET_SAMPLES)); + + expected_sequence = Some(expected.wrapping_add(1)); + + missing_since = None; + + eprintln!("[vc] lost UDP packet {}", expected); + } + + // ============================================================ + // RESAMPLE + // ============================================================ + + let input_frame_size = output_resampler.chunk_size_input(); + + while resample_input.len() >= input_frame_size { + let input: Vec = resample_input.drain(..input_frame_size).collect(); + + let output_frame_size = output_resampler.chunk_size_output(); + + let mut resampled = vec![0.0f32; output_frame_size]; if let Err(e) = output_resampler.resample(&input, &mut resampled) { eprintln!( - "[vc] failed to resample \ - output: {e}" + "[vc] output resample \ + failed: {e}" ); continue; } - // ------------------------------------------------ - // Mono -> device channels. - // ------------------------------------------------ + // ======================================================== + // MONO -> DEVICE CHANNELS + // ======================================================== let output = mono_to_output_channels(&resampled, current_config.channels); - // ------------------------------------------------ - // Push into bounded playback queue. - // ------------------------------------------------ + // ======================================================== + // PLAYBACK QUEUE + // ======================================================== + + let channels = current_config.channels.max(1) as usize; let mut queue = playback_buffer.lock().unwrap(); queue.extend(output); - let channels = current_config.channels.max(1) as usize; + // -------------------------------------------------------- + // Maximum playback queue. + // + // If this gets exceeded, THROW AWAY OLD AUDIO. + // + // This prevents latency from continuously growing. + // -------------------------------------------------------- - let max_frames = current_config.sample_rate as usize * MAX_BUFFER_MS / 1000; + let max_frames = current_config.sample_rate as usize * JITTER_MAX_MS / 1000; let max_samples = max_frames * channels; - // ------------------------------------------------ - // If we have accumulated too much audio, discard - // OLD audio. - // - // This is critical for voice chat latency. - // ------------------------------------------------ - while queue.len() > max_samples { queue.pop_front(); } - // ------------------------------------------------ - // Start playback only after we have a small - // amount of audio buffered. - // ------------------------------------------------ + // -------------------------------------------------------- + // Start playback once enough audio is available. + // -------------------------------------------------------- - if !playback_started.load(Ordering::SeqCst) { - let prebuffer_frames = - current_config.sample_rate as usize * PREBUFFER_MS / 1000; + if !playback_started.load(Ordering::Acquire) { + let target_frames = + current_config.sample_rate as usize * JITTER_TARGET_MS / 1000; - let prebuffer_samples = prebuffer_frames * channels; + let target_samples = target_frames * channels; - if queue.len() >= prebuffer_samples { - playback_started.store(true, Ordering::SeqCst); + if queue.len() >= target_samples { + playback_started.store(true, Ordering::Release); eprintln!( "[vc] playback started \ with ~{}ms buffered", - PREBUFFER_MS + JITTER_TARGET_MS ); } } } + + // ============================================================ + // RECOVER FROM PLAYBACK UNDERRUN + // ============================================================ + // + // If CPAL consumes everything, it will output silence. + // + // Once enough audio has accumulated again, playback can + // resume. + // + // We intentionally don't constantly toggle this state. + // That was one of the sources of the previous flicker. + // ============================================================ + + if playback_started.load(Ordering::Acquire) { + let channels = current_config.channels.max(1) as usize; + + let queue_len = playback_buffer.lock().unwrap().len(); + + let low_frames = current_config.sample_rate as usize * 10 / 1000; + + let low_samples = low_frames * channels; + + // We don't stop playback at 0ms. + // + // Only stop if the queue has actually become empty. + if queue_len == 0 { + playback_started.store(false, Ordering::Release); + } + + let _ = low_samples; + } } }); } - // ======================================================== + // ======================================================================== // START INPUT - // ======================================================== + // ======================================================================== input_stream .play() .map_err(|e| format!("Failed to start input stream: {e}"))?; - // ======================================================== - // STORE SESSION - // ======================================================== + // ======================================================================== + // SAVE SESSION + // ======================================================================== *voice_state.session.lock().unwrap() = Some(VoiceSession { input_stream, @@ -678,9 +895,9 @@ pub fn connect_to_vc( Ok(()) } -// ============================================================ +// ============================================================================ // OUTPUT CONFIG -// ============================================================ +// ============================================================================ fn build_output_config(output_device: &cpal::Device) -> Result { output_device @@ -689,27 +906,30 @@ fn build_output_config(output_device: &cpal::Device) -> Result Result { - let output_rate = output_sample_rate - .try_into() - .map_err(|e| format!("Invalid output sample rate: {e:?}"))?; + let rate = output_sample_rate.try_into().map_err(|e| { + format!( + "Invalid output sample rate: \ + {e:?}" + ) + })?; Ok(resampler::ResamplerFft::new( 1, resampler::SampleRate::Hz44100, - output_rate, + rate, )) } -// ============================================================ -// BUILD OUTPUT STREAM -// ============================================================ +// ============================================================================ +// OUTPUT STREAM +// ============================================================================ fn build_output_stream( output_device: &cpal::Device, @@ -721,36 +941,41 @@ fn build_output_stream( output_device .build_output_stream( output_config.clone(), - // ==================================================== - // CPAL OUTPUT CALLBACK - // ==================================================== + // ================================================================= + // AUDIO CALLBACK + // ================================================================= move |data: &mut [f32], _| { - let started = playback_started.load(Ordering::Acquire); + // ------------------------------------------------------------- + // Don't consume the queue until the jitter buffer has enough + // audio. + // ------------------------------------------------------------- - if !started { - // Do NOT consume audio before the prebuffer is - // ready. Just output silence. + if !playback_started.load(Ordering::Acquire) { data.fill(0.0); return; } let mut queue = playback_buffer.lock().unwrap(); + // ------------------------------------------------------------- + // CRITICAL: + // // VecDeque::pop_front() is O(1). // - // This is massively better than: - // - // Vec::remove(0) - // - // which shifts the entire vector every sample. + // NEVER use Vec::remove(0) here. + // ------------------------------------------------------------- - for sample in data.iter_mut() { - *sample = queue.pop_front().unwrap_or(0.0); + for output in data.iter_mut() { + *output = queue.pop_front().unwrap_or(0.0); } + + // If the callback consumed the entire queue, the receiver + // thread will refill it. We don't modify playback_started + // here because the CPAL callback should stay extremely cheap. }, - // ==================================================== - // OUTPUT ERROR CALLBACK - // ==================================================== + // ================================================================= + // ERROR CALLBACK + // ================================================================= { let needs_rebuild = needs_rebuild.clone(); @@ -772,9 +997,9 @@ fn build_output_stream( .map_err(|e| e.to_string()) } -// ============================================================ -// MONO -> OUTPUT CHANNELS -// ============================================================ +// ============================================================================ +// CHANNEL CONVERSION +// ============================================================================ fn mono_to_output_channels(mono: &[f32], channels: u16) -> Vec { let channels = channels.max(1) as usize; @@ -794,9 +1019,30 @@ fn mono_to_output_channels(mono: &[f32], channels: u16) -> Vec { output } -// ============================================================ +// ============================================================================ +// SEQUENCE NUMBER HELPERS +// ============================================================================ +// +// UDP sequence numbers eventually wrap around u32::MAX. +// +// These helpers make comparisons work correctly across the wrap. +// + +fn sequence_after(a: u32, b: u32) -> bool { + let diff = a.wrapping_sub(b); + + diff != 0 && diff < 0x8000_0000 +} + +fn sequence_before(a: u32, b: u32) -> bool { + let diff = a.wrapping_sub(b); + + diff != 0 && diff >= 0x8000_0000 +} + +// ============================================================================ // OPTIONAL UTILITY -// ============================================================ +// ============================================================================ fn stereo_to_mono(stereo_data: &[f32]) -> Vec { let mut mono = Vec::with_capacity(stereo_data.len() / 2); From 521e2d9bdc0be23b67d1b75bc2c4c49c22d37c60 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 23:03:04 +0200 Subject: [PATCH 17/29] Simplified by removing a lot of bs --- src-tauri/src/commands/audio.rs | 897 +++++++------------------------- 1 file changed, 200 insertions(+), 697 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 1e3bc7e..62367da 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -1,12 +1,12 @@ use std::{ - collections::{BTreeMap, VecDeque}, + collections::BTreeMap, net::UdpSocket, sync::{ atomic::{AtomicBool, Ordering}, Arc, Mutex, }, thread, - time::{Duration, Instant}, + time::Duration, }; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; @@ -14,55 +14,12 @@ use tauri::State; use crate::commands::config::ConfigState; -// ============================================================================ -// AUDIO FORMAT -// ============================================================================ - -const NETWORK_SAMPLE_RATE: u32 = 44_100; - -// 20ms packets. -// -// 44100 * 0.020 = 882 samples -const PACKET_DURATION_MS: usize = 20; -const NETWORK_PACKET_SAMPLES: usize = NETWORK_SAMPLE_RATE as usize * PACKET_DURATION_MS / 1000; - -// PCM is i16 => 2 bytes/sample. -const NETWORK_PACKET_BYTES: usize = NETWORK_PACKET_SAMPLES * 2; - -// Sequence number is a u32. -const AUDIO_HEADER_BYTES: usize = 4; - -// Maximum UDP packet we accept. -const MAX_UDP_PACKET_SIZE: usize = 4096; +const PACKET_SAMPLES: usize = 882; // 20ms @ 44.1kHz +const HEADER_SIZE: usize = 4; +const MAX_PACKET_SIZE: usize = 4096; // ============================================================================ -// JITTER BUFFER -// ============================================================================ -// -// We intentionally keep this relatively small. -// -// Increasing these values reduces underruns but increases latency. -// -// Current target: -// -// startup: 40ms -// normal target: 40ms -// maximum: 100ms -// -// This is appropriate for low-latency voice. -// - -const JITTER_TARGET_MS: usize = 40; -const JITTER_MAX_MS: usize = 100; - -// If a packet is missing, wait this long before considering it lost. -// -// Since packets are 20ms, 30ms gives us enough room for modest -// out-of-order delivery without making latency enormous. -const PACKET_LOSS_WAIT_MS: u64 = 30; - -// ============================================================================ -// VOICE STATE +// STATE // ============================================================================ pub struct VoiceState { @@ -71,13 +28,9 @@ pub struct VoiceState { pub struct VoiceSession { pub input_stream: cpal::Stream, - pub output_stream: Arc>, - pub socket: Arc, - pub pin: u64, - pub shutdown: Arc, } @@ -90,19 +43,17 @@ impl Default for VoiceState { } // ============================================================================ -// DEVICE LISTING +// DEVICES // ============================================================================ #[tauri::command] pub fn list_input_devices() -> Result, String> { let host = cpal::default_host(); - let devices = host + Ok(host .input_devices() - .map_err(|e| format!("Failed to enumerate input devices: {e}"))?; - - Ok(devices - .filter_map(|device| Some(device.description().ok()?.name().to_string())) + .map_err(|e| e.to_string())? + .filter_map(|d| d.description().ok().map(|x| x.name().to_string())) .collect()) } @@ -110,12 +61,10 @@ pub fn list_input_devices() -> Result, String> { pub fn list_output_devices() -> Result, String> { let host = cpal::default_host(); - let devices = host + Ok(host .output_devices() - .map_err(|e| format!("Failed to enumerate output devices: {e}"))?; - - Ok(devices - .filter_map(|device| Some(device.description().ok()?.name().to_string())) + .map_err(|e| e.to_string())? + .filter_map(|d| d.description().ok().map(|x| x.name().to_string())) .collect()) } @@ -147,35 +96,29 @@ pub fn connect_to_vc( let config = config_state.0.lock().unwrap().clone(); - // ======================================================================== + // ------------------------------------------------------------------------ // UDP - // ======================================================================== + // ------------------------------------------------------------------------ - let socket = - UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?; + let socket = Arc::new( + UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?, + ); socket .connect(&hostname) .map_err(|e| format!("Failed to connect UDP socket: {e}"))?; - // Required so disconnect can terminate the receiver thread. socket .set_read_timeout(Some(Duration::from_millis(100))) - .map_err(|e| format!("Failed to set UDP read timeout: {e}"))?; - - let socket = Arc::new(socket); - - // ======================================================================== - // AUTHENTICATION - // ======================================================================== + .map_err(|e| e.to_string())?; socket .send(&pin.to_be_bytes()) - .map_err(|e| format!("Failed to send pincode: {e}"))?; + .map_err(|e| format!("Failed to send pin: {e}"))?; - // ======================================================================== + // ------------------------------------------------------------------------ // DEVICES - // ======================================================================== + // ------------------------------------------------------------------------ let host = cpal::default_host(); @@ -183,110 +126,92 @@ pub fn connect_to_vc( Some(name) => host .input_devices() .map_err(|e| e.to_string())? - .find(|device| { - device - .description() + .find(|d| { + d.description() .ok() - .map(|description| description.name() == name) + .map(|x| x.name() == *name) .unwrap_or(false) }) - .ok_or_else(|| "Input device not found".to_string())?, + .ok_or("Input device not found")?, None => host .default_input_device() - .ok_or_else(|| "No default input device".to_string())?, + .ok_or("No default input device")?, }; let output_device = match &config.output_device_name { Some(name) => host .output_devices() .map_err(|e| e.to_string())? - .find(|device| { - device - .description() + .find(|d| { + d.description() .ok() - .map(|description| description.name() == name) + .map(|x| x.name() == *name) .unwrap_or(false) }) - .ok_or_else(|| "Output device not found".to_string())?, + .ok_or("Output device not found")?, None => host .default_output_device() - .ok_or_else(|| "No default output device".to_string())?, + .ok_or("No default output device")?, }; - // ======================================================================== - // SHUTDOWN - // ======================================================================== - let shutdown = Arc::new(AtomicBool::new(false)); - // ======================================================================== + // ------------------------------------------------------------------------ // PLAYBACK BUFFER - // ======================================================================== - // - // This contains samples already converted to the output device's - // channel layout. - // + // ------------------------------------------------------------------------ - let playback_buffer = Arc::new(Mutex::new(VecDeque::::new())); + let playback_buffer = Arc::new(Mutex::new(std::collections::VecDeque::::new())); - let playback_started = Arc::new(AtomicBool::new(false)); - - // ======================================================================== - // OUTPUT CONFIG - // ======================================================================== + // ------------------------------------------------------------------------ + // OUTPUT + // ------------------------------------------------------------------------ let output_config = build_output_config(&output_device)?; - - eprintln!( - "[vc] output: {} Hz / {} channels", - output_config.sample_rate, output_config.channels - ); - - let output_config_cell = Arc::new(Mutex::new(output_config.clone())); + let output_config = Arc::new(Mutex::new(output_config)); let needs_output_rebuild = Arc::new(AtomicBool::new(false)); - // ======================================================================== - // INPUT - // ======================================================================== + let output_stream = build_output_stream( + &output_device, + &output_config.lock().unwrap(), + playback_buffer.clone(), + needs_output_rebuild.clone(), + )?; - let input_socket = socket.clone(); + output_stream + .play() + .map_err(|e| format!("Failed to start output: {e}"))?; + + let output_stream = Arc::new(Mutex::new(output_stream)); + + // ------------------------------------------------------------------------ + // INPUT + // ------------------------------------------------------------------------ let mut input_config: cpal::StreamConfig = input_device .default_input_config() - .map_err(|e| format!("Failed to get input config: {e}"))? + .map_err(|e| e.to_string())? .into(); - // Network audio is mono. input_config.channels = 1; let input_sample_rate = input_config.sample_rate; - eprintln!( - "[vc] input: {} Hz -> {} Hz", - input_sample_rate, NETWORK_SAMPLE_RATE - ); + let input_socket = socket.clone(); let mut input_resampler = resampler::ResamplerFft::new( 1, - input_sample_rate.try_into().map_err(|e| { - format!( - "Invalid input sample rate: \ - {e:?}" - ) - })?, + input_sample_rate + .try_into() + .map_err(|e| format!("Invalid input sample rate: {e:?}"))?, resampler::SampleRate::Hz44100, ); let mut input_buffer = Vec::::new(); - - // Resampled samples waiting to form 20ms packets. - let mut packet_samples = Vec::::with_capacity(NETWORK_PACKET_SAMPLES * 2); - - // Sequence number for every audio packet. - let mut sequence: u32 = 0; + let mut packet_buffer = Vec::::new(); + let mut sequence = 0u32; let input_stream = input_device .build_input_stream( @@ -294,43 +219,24 @@ pub fn connect_to_vc( move |data: &[f32], _| { input_buffer.extend_from_slice(data); - let frame_size = input_resampler.chunk_size_input(); + let input_size = input_resampler.chunk_size_input(); + let output_size = input_resampler.chunk_size_output(); - while input_buffer.len() >= frame_size { - let input: Vec = input_buffer.drain(..frame_size).collect(); + while input_buffer.len() >= input_size { + let input: Vec = input_buffer.drain(..input_size).collect(); - let output_size = input_resampler.chunk_size_output(); - - let mut output = vec![0.0f32; output_size]; - - if let Err(e) = input_resampler.resample(&input, &mut output) { - eprintln!( - "[vc] input resample \ - failed: {e}" - ); + let mut output = vec![0.0; output_size]; + if input_resampler.resample(&input, &mut output).is_err() { continue; } - packet_samples.extend(output.into_iter().map(|sample| sample.clamp(-1.0, 1.0))); + packet_buffer.extend(output); - // ======================================================== - // CREATE EXACT 20ms PACKETS - // ======================================================== + while packet_buffer.len() >= PACKET_SAMPLES { + let samples: Vec = packet_buffer.drain(..PACKET_SAMPLES).collect(); - while packet_samples.len() >= NETWORK_PACKET_SAMPLES { - let samples: Vec = - packet_samples.drain(..NETWORK_PACKET_SAMPLES).collect(); - - // Header: - // - // [u32 sequence] - // - // Then: - // - // [i16 PCM...] - let mut packet = - Vec::with_capacity(AUDIO_HEADER_BYTES + NETWORK_PACKET_BYTES); + let mut packet = Vec::with_capacity(HEADER_SIZE + PACKET_SAMPLES * 2); packet.extend_from_slice(&sequence.to_be_bytes()); @@ -340,206 +246,103 @@ pub fn connect_to_vc( packet.extend_from_slice(&pcm.to_be_bytes()); } - if let Err(e) = input_socket.send(&packet) { - eprintln!( - "[vc] UDP send failed: \ - {e}" - ); - } + let _ = input_socket.send(&packet); sequence = sequence.wrapping_add(1); } } }, |err| { - eprintln!("[vc] input stream error: {err}"); + eprintln!("[vc] input error: {err}"); }, None, ) .map_err(|e| e.to_string())?; - // ======================================================================== - // OUTPUT STREAM - // ======================================================================== - - let initial_output_stream = build_output_stream( - &output_device, - &output_config, - playback_buffer.clone(), - needs_output_rebuild.clone(), - playback_started.clone(), - )?; - - initial_output_stream - .play() - .map_err(|e| format!("Failed to start output stream: {e}"))?; - - let output_stream = Arc::new(Mutex::new(initial_output_stream)); - - // ======================================================================== - // OUTPUT DEVICE WATCHER - // ======================================================================== + // ------------------------------------------------------------------------ + // OUTPUT DEVICE REBUILD WATCHER + // ------------------------------------------------------------------------ { let output_device = output_device.clone(); - - let needs_output_rebuild = needs_output_rebuild.clone(); - let output_stream = output_stream.clone(); - - let output_config_cell = output_config_cell.clone(); - + let output_config = output_config.clone(); let playback_buffer = playback_buffer.clone(); - - let playback_started = playback_started.clone(); - + let needs_rebuild = needs_output_rebuild.clone(); let shutdown = shutdown.clone(); thread::spawn(move || { while !shutdown.load(Ordering::SeqCst) { thread::sleep(Duration::from_millis(100)); - if shutdown.load(Ordering::SeqCst) { - return; - } - - if !needs_output_rebuild.swap(false, Ordering::SeqCst) { + if !needs_rebuild.swap(false, Ordering::SeqCst) { continue; } - eprintln!("[vc] rebuilding output stream..."); + let Ok(new_config) = build_output_config(&output_device) else { + needs_rebuild.store(true, Ordering::SeqCst); + continue; + }; - let result = build_output_config(&output_device).and_then(|new_config| { - build_output_stream( - &output_device, - &new_config, - playback_buffer.clone(), - needs_output_rebuild.clone(), - playback_started.clone(), - ) - .map(|stream| (new_config, stream)) - }); + let Ok(new_stream) = build_output_stream( + &output_device, + &new_config, + playback_buffer.clone(), + needs_rebuild.clone(), + ) else { + needs_rebuild.store(true, Ordering::SeqCst); + continue; + }; - match result { - Ok((new_config, new_stream)) => { - if let Err(e) = new_stream.play() { - eprintln!( - "[vc] failed to start \ - rebuilt output: {e}" - ); - - needs_output_rebuild.store(true, Ordering::SeqCst); - - continue; - } - - // The old output format is no longer valid. - // - // Throw away queued samples rather than playing - // them using the new device timing/layout. - playback_buffer.lock().unwrap().clear(); - - playback_started.store(false, Ordering::SeqCst); - - *output_config_cell.lock().unwrap() = new_config.clone(); - - *output_stream.lock().unwrap() = new_stream; - - eprintln!( - "[vc] output rebuilt: \ - {} Hz / {} channels", - new_config.sample_rate, new_config.channels - ); - } - - Err(e) => { - eprintln!( - "[vc] failed to rebuild \ - output: {e}" - ); - - needs_output_rebuild.store(true, Ordering::SeqCst); - } + if new_stream.play().is_err() { + needs_rebuild.store(true, Ordering::SeqCst); + continue; } + + playback_buffer.lock().unwrap().clear(); + + *output_config.lock().unwrap() = new_config; + *output_stream.lock().unwrap() = new_stream; } }); } - // ======================================================================== - // UDP RECEIVE + JITTER BUFFER - // ======================================================================== + // ------------------------------------------------------------------------ + // UDP RECEIVE + // ------------------------------------------------------------------------ { - let recv_socket = socket.clone(); - - let output_config_cell = output_config_cell.clone(); - + let socket = socket.clone(); let playback_buffer = playback_buffer.clone(); - - let playback_started = playback_started.clone(); - + let output_config = output_config.clone(); let shutdown = shutdown.clone(); thread::spawn(move || { - // ================================================================ - // PACKET REORDER BUFFER - // ================================================================ - - let mut packets: BTreeMap> = BTreeMap::new(); - - let mut expected_sequence: Option = None; - - // When we first notice a missing packet, remember when. - let mut missing_since: Option = None; - - // ================================================================ - // RESAMPLER - // ================================================================ - - let initial_config = output_config_cell.lock().unwrap().clone(); - - let mut last_sample_rate = initial_config.sample_rate; + let initial_config = output_config.lock().unwrap().clone(); let mut output_resampler = match create_output_resampler(initial_config.sample_rate) { - Ok(resampler) => resampler, - + Ok(r) => r, Err(e) => { - eprintln!( - "[vc] failed to create \ - output resampler: {e}" - ); - + eprintln!("[vc] output resampler: {e}"); return; } }; - let mut resample_input = Vec::::new(); + let mut last_sample_rate = initial_config.sample_rate; - // ================================================================ - // UDP BUFFER - // ================================================================ + let mut packets: BTreeMap> = BTreeMap::new(); + let mut expected: Option = None; - let mut udp_buffer = [0u8; MAX_UDP_PACKET_SIZE]; - - // ================================================================ - // MAIN LOOP - // ================================================================ + let mut resample_buffer = Vec::::new(); + let mut udp_buffer = [0u8; MAX_PACKET_SIZE]; while !shutdown.load(Ordering::SeqCst) { - // ============================================================ - // RECEIVE PACKET - // ============================================================ - - match recv_socket.recv(&mut udp_buffer) { + match socket.recv(&mut udp_buffer) { Ok(len) => { - if len <= AUDIO_HEADER_BYTES { + if len <= HEADER_SIZE { continue; } - // ---------------------------------------------------- - // Read sequence number. - // ---------------------------------------------------- - let sequence = u32::from_be_bytes([ udp_buffer[0], udp_buffer[1], @@ -547,342 +350,131 @@ pub fn connect_to_vc( udp_buffer[3], ]); - let pcm_bytes = &udp_buffer[AUDIO_HEADER_BYTES..len]; + let pcm = &udp_buffer[HEADER_SIZE..len]; - // Must contain complete i16 samples. - let pcm_len = pcm_bytes.len() & !1; + let mut samples = Vec::with_capacity(pcm.len() / 2); - if pcm_len == 0 { - continue; + for chunk in pcm.chunks_exact(2) { + let value = i16::from_be_bytes([chunk[0], chunk[1]]); + + samples.push(value as f32 / i16::MAX as f32); } - let mut samples = Vec::with_capacity(pcm_len / 2); - - for chunk in pcm_bytes[..pcm_len].chunks_exact(2) { - let pcm = i16::from_be_bytes([chunk[0], chunk[1]]); - - samples.push(pcm as f32 / i16::MAX as f32); - } - - // ---------------------------------------------------- - // Ignore packets that are already too old. - // ---------------------------------------------------- - - if let Some(expected) = expected_sequence { - if sequence_before(sequence, expected) { - continue; - } - } - - // ---------------------------------------------------- - // Insert into jitter buffer. - // - // BTreeMap automatically keeps sequence numbers - // ordered. - // ---------------------------------------------------- - packets.entry(sequence).or_insert(samples); - - // ---------------------------------------------------- - // Don't allow the packet jitter buffer itself to - // become a source of latency. - // ---------------------------------------------------- - - while packets.len() > 8 { - if let Some((&oldest, _)) = packets.iter().next() { - if let Some(expected) = expected_sequence { - if sequence_before(oldest, expected) { - packets.remove(&oldest); - } else { - break; - } - } else { - break; - } - } - } } - Err(e) - if e.kind() == std::io::ErrorKind::TimedOut - || e.kind() == std::io::ErrorKind::WouldBlock => - { - // Timeout is expected. It gives us a chance to - // process jitter-buffer timeouts and shutdown. + Err(e) if e.kind() == std::io::ErrorKind::TimedOut => { + continue; } Err(e) => { if !shutdown.load(Ordering::SeqCst) { - eprintln!( - "[vc] UDP receive failed: \ - {e}" - ); + eprintln!("[vc] UDP receive error: {e}"); } - - continue; } } - // ============================================================ - // INITIALIZE EXPECTED SEQUENCE - // ============================================================ + // ---------------------------------------------------------------- + // OUTPUT CONFIG + // ---------------------------------------------------------------- - if expected_sequence.is_none() { - if let Some((&first_sequence, _)) = packets.iter().next() { - expected_sequence = Some(first_sequence); + let config = output_config.lock().unwrap().clone(); - eprintln!( - "[vc] jitter buffer \ - synchronized at packet {}", - first_sequence - ); - } - } - - // ============================================================ - // CURRENT OUTPUT CONFIG - // ============================================================ - - let current_config = output_config_cell.lock().unwrap().clone(); - - // ============================================================ - // OUTPUT DEVICE SAMPLE RATE CHANGE - // ============================================================ - - if current_config.sample_rate != last_sample_rate { - eprintln!( - "[vc] output rate changed: \ - {} -> {}", - last_sample_rate, current_config.sample_rate - ); - - match create_output_resampler(current_config.sample_rate) { + if config.sample_rate != last_sample_rate { + match create_output_resampler(config.sample_rate) { Ok(new_resampler) => { output_resampler = new_resampler; - - last_sample_rate = current_config.sample_rate; + last_sample_rate = config.sample_rate; packets.clear(); - - resample_input.clear(); - + resample_buffer.clear(); playback_buffer.lock().unwrap().clear(); - - playback_started.store(false, Ordering::SeqCst); - - expected_sequence = None; - - missing_since = None; + expected = None; } - Err(e) => { - eprintln!( - "[vc] failed to create \ - output resampler: {e}" - ); - - packets.clear(); - resample_input.clear(); - - continue; - } + Err(_) => continue, } } - // ============================================================ - // MOVE READY PACKETS INTO RESAMPLER - // ============================================================ + // ---------------------------------------------------------------- + // INITIAL SEQUENCE + // ---------------------------------------------------------------- - loop { - let Some(expected) = expected_sequence else { - break; - }; - - // -------------------------------------------------------- - // Expected packet exists. - // -------------------------------------------------------- - - if let Some(samples) = packets.remove(&expected) { - resample_input.extend_from_slice(&samples); - - expected_sequence = Some(expected.wrapping_add(1)); - - missing_since = None; - - continue; - } - - // -------------------------------------------------------- - // Expected packet doesn't exist. - // - // If we don't have anything newer, there is nothing - // to do yet. - // -------------------------------------------------------- - - let has_newer_packet = packets.keys().any(|&seq| sequence_after(seq, expected)); - - if !has_newer_packet { - break; - } - - // -------------------------------------------------------- - // We have a later packet. - // - // Therefore the expected packet is either delayed or - // lost. - // - // Wait a short period before declaring it lost. - // -------------------------------------------------------- - - let now = Instant::now(); - - let since = missing_since.get_or_insert(now); - - if since.elapsed() < Duration::from_millis(PACKET_LOSS_WAIT_MS) { - break; - } - - // -------------------------------------------------------- - // Packet is considered lost. - // - // We insert a zero packet here. - // - // Because the missing packet is exactly 20ms, this - // results in a controlled 20ms gap rather than the - // playback clock getting permanently stuck. - // -------------------------------------------------------- - - resample_input.extend(std::iter::repeat(0.0f32).take(NETWORK_PACKET_SAMPLES)); - - expected_sequence = Some(expected.wrapping_add(1)); - - missing_since = None; - - eprintln!("[vc] lost UDP packet {}", expected); + if expected.is_none() { + expected = packets.keys().next().copied(); } - // ============================================================ + // ---------------------------------------------------------------- + // READ PACKETS IN ORDER + // ---------------------------------------------------------------- + + while let Some(seq) = expected { + if let Some(samples) = packets.remove(&seq) { + resample_buffer.extend(samples); + expected = Some(seq.wrapping_add(1)); + } else { + // Missing packet. + // + // Don't wait for it. + // Just insert 20ms of silence. + if packets.keys().any(|&x| x > seq) { + resample_buffer.extend(std::iter::repeat(0.0).take(PACKET_SAMPLES)); + + expected = Some(seq.wrapping_add(1)); + } + + break; + } + } + + // ---------------------------------------------------------------- // RESAMPLE - // ============================================================ + // ---------------------------------------------------------------- - let input_frame_size = output_resampler.chunk_size_input(); + let input_size = output_resampler.chunk_size_input(); + let output_size = output_resampler.chunk_size_output(); - while resample_input.len() >= input_frame_size { - let input: Vec = resample_input.drain(..input_frame_size).collect(); + while resample_buffer.len() >= input_size { + let input: Vec = resample_buffer.drain(..input_size).collect(); - let output_frame_size = output_resampler.chunk_size_output(); - - let mut resampled = vec![0.0f32; output_frame_size]; - - if let Err(e) = output_resampler.resample(&input, &mut resampled) { - eprintln!( - "[vc] output resample \ - failed: {e}" - ); + let mut output = vec![0.0; output_size]; + if output_resampler.resample(&input, &mut output).is_err() { continue; } - // ======================================================== - // MONO -> DEVICE CHANNELS - // ======================================================== - - let output = mono_to_output_channels(&resampled, current_config.channels); - - // ======================================================== - // PLAYBACK QUEUE - // ======================================================== - - let channels = current_config.channels.max(1) as usize; + let channels = config.channels.max(1) as usize; let mut queue = playback_buffer.lock().unwrap(); - queue.extend(output); + for sample in output { + for _ in 0..channels { + queue.push_back(sample); + } + } - // -------------------------------------------------------- - // Maximum playback queue. - // - // If this gets exceeded, THROW AWAY OLD AUDIO. - // - // This prevents latency from continuously growing. - // -------------------------------------------------------- - - let max_frames = current_config.sample_rate as usize * JITTER_MAX_MS / 1000; - - let max_samples = max_frames * channels; + // Don't let latency grow forever. + let max_samples = config.sample_rate as usize * channels / 5; while queue.len() > max_samples { queue.pop_front(); } - - // -------------------------------------------------------- - // Start playback once enough audio is available. - // -------------------------------------------------------- - - if !playback_started.load(Ordering::Acquire) { - let target_frames = - current_config.sample_rate as usize * JITTER_TARGET_MS / 1000; - - let target_samples = target_frames * channels; - - if queue.len() >= target_samples { - playback_started.store(true, Ordering::Release); - - eprintln!( - "[vc] playback started \ - with ~{}ms buffered", - JITTER_TARGET_MS - ); - } - } - } - - // ============================================================ - // RECOVER FROM PLAYBACK UNDERRUN - // ============================================================ - // - // If CPAL consumes everything, it will output silence. - // - // Once enough audio has accumulated again, playback can - // resume. - // - // We intentionally don't constantly toggle this state. - // That was one of the sources of the previous flicker. - // ============================================================ - - if playback_started.load(Ordering::Acquire) { - let channels = current_config.channels.max(1) as usize; - - let queue_len = playback_buffer.lock().unwrap().len(); - - let low_frames = current_config.sample_rate as usize * 10 / 1000; - - let low_samples = low_frames * channels; - - // We don't stop playback at 0ms. - // - // Only stop if the queue has actually become empty. - if queue_len == 0 { - playback_started.store(false, Ordering::Release); - } - - let _ = low_samples; } } }); } - // ======================================================================== + // ------------------------------------------------------------------------ // START INPUT - // ======================================================================== + // ------------------------------------------------------------------------ input_stream .play() - .map_err(|e| format!("Failed to start input stream: {e}"))?; + .map_err(|e| format!("Failed to start input: {e}"))?; - // ======================================================================== - // SAVE SESSION - // ======================================================================== + // ------------------------------------------------------------------------ + // SAVE + // ------------------------------------------------------------------------ *voice_state.session.lock().unwrap() = Some(VoiceSession { input_stream, @@ -899,26 +491,23 @@ pub fn connect_to_vc( // OUTPUT CONFIG // ============================================================================ -fn build_output_config(output_device: &cpal::Device) -> Result { - output_device +fn build_output_config(device: &cpal::Device) -> Result { + device .default_output_config() - .map_err(|e| e.to_string()) .map(Into::into) + .map_err(|e| e.to_string()) } // ============================================================================ -// OUTPUT RESAMPLER +// RESAMPLER // ============================================================================ fn create_output_resampler( - output_sample_rate: cpal::SampleRate, + sample_rate: cpal::SampleRate, ) -> Result { - let rate = output_sample_rate.try_into().map_err(|e| { - format!( - "Invalid output sample rate: \ - {e:?}" - ) - })?; + let rate = sample_rate + .try_into() + .map_err(|e| format!("Invalid sample rate: {e:?}"))?; Ok(resampler::ResamplerFft::new( 1, @@ -932,55 +521,26 @@ fn create_output_resampler( // ============================================================================ fn build_output_stream( - output_device: &cpal::Device, - output_config: &cpal::StreamConfig, - playback_buffer: Arc>>, + device: &cpal::Device, + config: &cpal::StreamConfig, + playback_buffer: Arc>>, needs_rebuild: Arc, - playback_started: Arc, ) -> Result { - output_device + device .build_output_stream( - output_config.clone(), - // ================================================================= - // AUDIO CALLBACK - // ================================================================= + config.clone(), move |data: &mut [f32], _| { - // ------------------------------------------------------------- - // Don't consume the queue until the jitter buffer has enough - // audio. - // ------------------------------------------------------------- - - if !playback_started.load(Ordering::Acquire) { - data.fill(0.0); - return; - } - let mut queue = playback_buffer.lock().unwrap(); - // ------------------------------------------------------------- - // CRITICAL: - // - // VecDeque::pop_front() is O(1). - // - // NEVER use Vec::remove(0) here. - // ------------------------------------------------------------- - - for output in data.iter_mut() { - *output = queue.pop_front().unwrap_or(0.0); + for sample in data { + *sample = queue.pop_front().unwrap_or(0.0); } - - // If the callback consumed the entire queue, the receiver - // thread will refill it. We don't modify playback_started - // here because the CPAL callback should stay extremely cheap. }, - // ================================================================= - // ERROR CALLBACK - // ================================================================= { let needs_rebuild = needs_rebuild.clone(); move |err| { - eprintln!("[vc] output stream error: {err}"); + eprintln!("[vc] output error: {err}"); let message = err.to_string(); @@ -996,60 +556,3 @@ fn build_output_stream( ) .map_err(|e| e.to_string()) } - -// ============================================================================ -// CHANNEL CONVERSION -// ============================================================================ - -fn mono_to_output_channels(mono: &[f32], channels: u16) -> Vec { - let channels = channels.max(1) as usize; - - if channels == 1 { - return mono.to_vec(); - } - - let mut output = Vec::with_capacity(mono.len() * channels); - - for &sample in mono { - for _ in 0..channels { - output.push(sample); - } - } - - output -} - -// ============================================================================ -// SEQUENCE NUMBER HELPERS -// ============================================================================ -// -// UDP sequence numbers eventually wrap around u32::MAX. -// -// These helpers make comparisons work correctly across the wrap. -// - -fn sequence_after(a: u32, b: u32) -> bool { - let diff = a.wrapping_sub(b); - - diff != 0 && diff < 0x8000_0000 -} - -fn sequence_before(a: u32, b: u32) -> bool { - let diff = a.wrapping_sub(b); - - diff != 0 && diff >= 0x8000_0000 -} - -// ============================================================================ -// OPTIONAL UTILITY -// ============================================================================ - -fn stereo_to_mono(stereo_data: &[f32]) -> Vec { - let mut mono = Vec::with_capacity(stereo_data.len() / 2); - - for chunk in stereo_data.chunks_exact(2) { - mono.push((chunk[0] + chunk[1]) * 0.5); - } - - mono -} From 1bbe820853c759d0f724ef7f9cb56e4c1414cdf3 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Wed, 26 Aug 2026 23:51:56 +0200 Subject: [PATCH 18/29] Using ring buffer --- src-tauri/Cargo.lock | 12 +++++ src-tauri/Cargo.toml | 1 + src-tauri/src/commands/audio.rs | 84 ++++++++++++++++++++++----------- 3 files changed, 70 insertions(+), 27 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 29c7181..e5acfe6 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -963,6 +963,7 @@ version = "0.1.0" dependencies = [ "cpal", "resampler", + "ringbuf", "serde", "serde_json", "tauri", @@ -2998,6 +2999,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28fdbea87ff02ebbfd904079d1e52138c0c7fbaa3aaddca4a1b9c7d3f85749f2" +[[package]] +name = "ringbuf" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a158e09ede21a14b172ca6cdd6208386c6ae2cb6acef58d774368ef8c450dfa7" +dependencies = [ + "crossbeam-utils", + "portable-atomic", + "portable-atomic-util", +] + [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index c24ab14..8b27ac7 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -24,4 +24,5 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" cpal = "0.18.2" resampler = "0.5.1" +ringbuf = "0.5.1" diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 62367da..a07d009 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -10,6 +10,10 @@ use std::{ }; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use ringbuf::{ + traits::{Consumer, Producer, Split}, + HeapRb, +}; use tauri::State; use crate::commands::config::ConfigState; @@ -159,10 +163,17 @@ pub fn connect_to_vc( let shutdown = Arc::new(AtomicBool::new(false)); // ------------------------------------------------------------------------ - // PLAYBACK BUFFER + // PLAYBACK BUFFER (RingBuffer) // ------------------------------------------------------------------------ - let playback_buffer = Arc::new(Mutex::new(std::collections::VecDeque::::new())); + // Buffer 2 seconds of audio max (prevents infinite latency growth) + let max_capacity = 48000; + let rb = HeapRb::::new(max_capacity); + let (producer, consumer) = rb.split(); + + // Split locks: UDP thread and Audio thread will never block each other + let producer_lock = Arc::new(Mutex::new(producer)); + let consumer_lock = Arc::new(Mutex::new(consumer)); // ------------------------------------------------------------------------ // OUTPUT @@ -176,7 +187,7 @@ pub fn connect_to_vc( let output_stream = build_output_stream( &output_device, &output_config.lock().unwrap(), - playback_buffer.clone(), + consumer_lock.clone(), needs_output_rebuild.clone(), )?; @@ -267,7 +278,7 @@ pub fn connect_to_vc( let output_device = output_device.clone(); let output_stream = output_stream.clone(); let output_config = output_config.clone(); - let playback_buffer = playback_buffer.clone(); + let consumer_lock = consumer_lock.clone(); let needs_rebuild = needs_output_rebuild.clone(); let shutdown = shutdown.clone(); @@ -287,7 +298,7 @@ pub fn connect_to_vc( let Ok(new_stream) = build_output_stream( &output_device, &new_config, - playback_buffer.clone(), + consumer_lock.clone(), needs_rebuild.clone(), ) else { needs_rebuild.store(true, Ordering::SeqCst); @@ -299,8 +310,6 @@ pub fn connect_to_vc( continue; } - playback_buffer.lock().unwrap().clear(); - *output_config.lock().unwrap() = new_config; *output_stream.lock().unwrap() = new_stream; } @@ -313,7 +322,6 @@ pub fn connect_to_vc( { let socket = socket.clone(); - let playback_buffer = playback_buffer.clone(); let output_config = output_config.clone(); let shutdown = shutdown.clone(); @@ -388,7 +396,6 @@ pub fn connect_to_vc( packets.clear(); resample_buffer.clear(); - playback_buffer.lock().unwrap().clear(); expected = None; } @@ -445,20 +452,16 @@ pub fn connect_to_vc( let channels = config.channels.max(1) as usize; - let mut queue = playback_buffer.lock().unwrap(); + let mut prod = producer_lock.lock().unwrap(); for sample in output { for _ in 0..channels { - queue.push_back(sample); + // try_push will return an error if the buffer is full (2 seconds). + // By ignoring the error, we simply drop the oldest overflow packets, + // which naturally prevents latency from growing forever. + let _ = prod.try_push(sample); } } - - // Don't let latency grow forever. - let max_samples = config.sample_rate as usize * channels / 5; - - while queue.len() > max_samples { - queue.pop_front(); - } } } }); @@ -520,30 +523,57 @@ fn create_output_resampler( // OUTPUT STREAM // ============================================================================ -fn build_output_stream( +fn build_output_stream( device: &cpal::Device, config: &cpal::StreamConfig, - playback_buffer: Arc>>, + consumer_lock: Arc>, needs_rebuild: Arc, -) -> Result { +) -> Result +where + C: Consumer + Send + 'static, +{ + // Jitter Buffer: wait for ~60ms of audio before starting playback + let channels = config.channels as usize; + let sample_rate = config.sample_rate as usize; + let jitter_cushion_samples = (sample_rate / 1000) * 60 * channels; + + let mut is_buffering = true; + device .build_output_stream( config.clone(), move |data: &mut [f32], _| { - let mut queue = playback_buffer.lock().unwrap(); + // The audio thread only locks the consumer side. + // Contention is zero unless the stream is actively crashing. + let mut cons = consumer_lock.lock().unwrap(); - for sample in data { - *sample = queue.pop_front().unwrap_or(0.0); + // 1. Jitter Buffer State Machine + if is_buffering { + if cons.occupied_len() >= jitter_cushion_samples { + is_buffering = false; // We have enough cushion, start! + } else { + data.fill(0.0); // Output silence while we wait + return; + } + } + + // 2. Lock-free pop + let read = cons.pop_slice(data); + + // 3. Underrun Detection + if read < data.len() { + // We ran out of data. Fill remainder with silence to avoid static + data[read..].fill(0.0); + + // Re-enter buffering mode to rebuild our cushion + is_buffering = true; } }, { let needs_rebuild = needs_rebuild.clone(); - move |err| { eprintln!("[vc] output error: {err}"); - let message = err.to_string(); - if message.contains("sample rate changed") || message.contains("DeviceNotAvailable") || message.contains("device not available") From 52e9583cfdbb8ad364bdc8c918bb2d63d896ff69 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 00:10:04 +0200 Subject: [PATCH 19/29] Jitter buffer --- src-tauri/src/commands/audio.rs | 110 ++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 41 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index a07d009..7d720e1 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -11,7 +11,7 @@ use std::{ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use ringbuf::{ - traits::{Consumer, Producer, Split}, + traits::{Consumer, Observer, Producer, Split}, HeapRb, }; use tauri::State; @@ -22,6 +22,11 @@ const PACKET_SAMPLES: usize = 882; // 20ms @ 44.1kHz const HEADER_SIZE: usize = 4; const MAX_PACKET_SIZE: usize = 4096; +// Jitter buffer settings +const JITTER_BUFFER_MS: usize = 60; // Cushion size before playback starts +const PACKET_DURATION_MS: usize = 20; +const INITIAL_PACKET_CUSHION: usize = JITTER_BUFFER_MS / PACKET_DURATION_MS; // 3 packets + // ============================================================================ // STATE // ============================================================================ @@ -166,12 +171,10 @@ pub fn connect_to_vc( // PLAYBACK BUFFER (RingBuffer) // ------------------------------------------------------------------------ - // Buffer 2 seconds of audio max (prevents infinite latency growth) let max_capacity = 48000; let rb = HeapRb::::new(max_capacity); let (producer, consumer) = rb.split(); - // Split locks: UDP thread and Audio thread will never block each other let producer_lock = Arc::new(Mutex::new(producer)); let consumer_lock = Arc::new(Mutex::new(consumer)); @@ -317,7 +320,7 @@ pub fn connect_to_vc( } // ------------------------------------------------------------------------ - // UDP RECEIVE + // UDP RECEIVE (JITTER BUFFER INCLUDED) // ------------------------------------------------------------------------ { @@ -340,6 +343,7 @@ pub fn connect_to_vc( let mut packets: BTreeMap> = BTreeMap::new(); let mut expected: Option = None; + let mut is_prebuffering = true; let mut resample_buffer = Vec::::new(); let mut udp_buffer = [0u8; MAX_PACKET_SIZE]; @@ -372,6 +376,11 @@ pub fn connect_to_vc( } Err(e) if e.kind() == std::io::ErrorKind::TimedOut => { + // Reset jitter state if connection completely drops + if packets.is_empty() { + expected = None; + is_prebuffering = true; + } continue; } @@ -383,7 +392,7 @@ pub fn connect_to_vc( } // ---------------------------------------------------------------- - // OUTPUT CONFIG + // OUTPUT CONFIG CHANGE // ---------------------------------------------------------------- let config = output_config.lock().unwrap().clone(); @@ -397,22 +406,33 @@ pub fn connect_to_vc( packets.clear(); resample_buffer.clear(); expected = None; + is_prebuffering = true; } - Err(_) => continue, } } // ---------------------------------------------------------------- - // INITIAL SEQUENCE + // JITTER BUFFER STATE MACHINE // ---------------------------------------------------------------- + if is_prebuffering { + // Accumulate target packet cushion before popping + if packets.len() >= INITIAL_PACKET_CUSHION { + expected = packets.keys().next().copied(); + is_prebuffering = false; + } else { + continue; + } + } + + // Initialize sequence if unset if expected.is_none() { expected = packets.keys().next().copied(); } // ---------------------------------------------------------------- - // READ PACKETS IN ORDER + // DRAIN PACKETS IN SEQUENTIAL ORDER // ---------------------------------------------------------------- while let Some(seq) = expected { @@ -420,22 +440,27 @@ pub fn connect_to_vc( resample_buffer.extend(samples); expected = Some(seq.wrapping_add(1)); } else { - // Missing packet. - // - // Don't wait for it. - // Just insert 20ms of silence. + // Missing frame strategy: + // If future sequence numbers exist, insert Concealment (Silence) if packets.keys().any(|&x| x > seq) { resample_buffer.extend(std::iter::repeat(0.0).take(PACKET_SAMPLES)); - expected = Some(seq.wrapping_add(1)); + } else { + // Waiting on late packets + break; } - - break; } } + // Trim jitter buffer cache to avoid memory leak spikes on extreme latency drops + if packets.len() > 50 { + packets.clear(); + expected = None; + is_prebuffering = true; + } + // ---------------------------------------------------------------- - // RESAMPLE + // RESAMPLE & POPULATE RINGBUFFER // ---------------------------------------------------------------- let input_size = output_resampler.chunk_size_input(); @@ -450,16 +475,18 @@ pub fn connect_to_vc( continue; } - let channels = config.channels.max(1) as usize; - let mut prod = producer_lock.lock().unwrap(); + // Latency Drift Protection: prevent total queue build-up past ~60ms + let target_sample_rate = config.sample_rate as usize; + let max_ring_buffer_samples = (target_sample_rate / 1000) * 60; + + // Fix occupied_len check and overflow protection + // Note: We avoid calling try_pop() on a Producer. If the buffer is full, + // try_push will naturally fail and drop the overflow sample. for sample in output { - for _ in 0..channels { - // try_push will return an error if the buffer is full (2 seconds). - // By ignoring the error, we simply drop the oldest overflow packets, - // which naturally prevents latency from growing forever. - let _ = prod.try_push(sample); + if (*prod).occupied_len() < max_ring_buffer_samples { + let _ = (*prod).try_push(sample); } } } @@ -532,10 +559,9 @@ fn build_output_stream( where C: Consumer + Send + 'static, { - // Jitter Buffer: wait for ~60ms of audio before starting playback let channels = config.channels as usize; let sample_rate = config.sample_rate as usize; - let jitter_cushion_samples = (sample_rate / 1000) * 60 * channels; + let jitter_cushion_samples = (sample_rate / 1000) * 40; // 40ms stream cushion let mut is_buffering = true; @@ -543,30 +569,32 @@ where .build_output_stream( config.clone(), move |data: &mut [f32], _| { - // The audio thread only locks the consumer side. - // Contention is zero unless the stream is actively crashing. let mut cons = consumer_lock.lock().unwrap(); - // 1. Jitter Buffer State Machine + // 1. Initial/Recovering Cushioning if is_buffering { - if cons.occupied_len() >= jitter_cushion_samples { - is_buffering = false; // We have enough cushion, start! + if (*cons).occupied_len() >= jitter_cushion_samples { + is_buffering = false; } else { - data.fill(0.0); // Output silence while we wait + data.fill(0.0); return; } } - // 2. Lock-free pop - let read = cons.pop_slice(data); - - // 3. Underrun Detection - if read < data.len() { - // We ran out of data. Fill remainder with silence to avoid static - data[read..].fill(0.0); - - // Re-enter buffering mode to rebuild our cushion - is_buffering = true; + // 2. Sample Extraction & Channel Interleaving + let mut idx = 0; + while idx < data.len() { + if let Some(mono_sample) = (*cons).try_pop() { + for ch in 0..channels { + data[idx + ch] = mono_sample; + } + idx += channels; + } else { + // 3. Underrun: pad rest with silence & re-enter buffering mode + data[idx..].fill(0.0); + is_buffering = true; + break; + } } }, { From 2935d53ea80a0100aa7b962d6794a96579d65854 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 00:23:40 +0200 Subject: [PATCH 20/29] More performance stuff --- src-tauri/src/commands/audio.rs | 340 ++++++-------------------------- 1 file changed, 60 insertions(+), 280 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 7d720e1..d59e24c 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -105,30 +105,19 @@ pub fn connect_to_vc( let config = config_state.0.lock().unwrap().clone(); - // ------------------------------------------------------------------------ - // UDP - // ------------------------------------------------------------------------ - let socket = Arc::new( UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?, ); - socket .connect(&hostname) .map_err(|e| format!("Failed to connect UDP socket: {e}"))?; - socket - .set_read_timeout(Some(Duration::from_millis(100))) + .set_read_timeout(Some(Duration::from_millis(50))) .map_err(|e| e.to_string())?; - socket .send(&pin.to_be_bytes()) .map_err(|e| format!("Failed to send pin: {e}"))?; - // ------------------------------------------------------------------------ - // DEVICES - // ------------------------------------------------------------------------ - let host = cpal::default_host(); let input_device = match &config.input_device_name { @@ -142,7 +131,6 @@ pub fn connect_to_vc( .unwrap_or(false) }) .ok_or("Input device not found")?, - None => host .default_input_device() .ok_or("No default input device")?, @@ -159,7 +147,6 @@ pub fn connect_to_vc( .unwrap_or(false) }) .ok_or("Output device not found")?, - None => host .default_output_device() .ok_or("No default output device")?, @@ -167,52 +154,35 @@ pub fn connect_to_vc( let shutdown = Arc::new(AtomicBool::new(false)); - // ------------------------------------------------------------------------ - // PLAYBACK BUFFER (RingBuffer) - // ------------------------------------------------------------------------ - - let max_capacity = 48000; - let rb = HeapRb::::new(max_capacity); - let (producer, consumer) = rb.split(); - - let producer_lock = Arc::new(Mutex::new(producer)); - let consumer_lock = Arc::new(Mutex::new(consumer)); - - // ------------------------------------------------------------------------ - // OUTPUT - // ------------------------------------------------------------------------ + // Lock-Free Ring Buffer setup (Hold 100ms max buffer) + let rb = HeapRb::::new(8820); + let (mut producer, consumer) = rb.split(); let output_config = build_output_config(&output_device)?; let output_config = Arc::new(Mutex::new(output_config)); - let needs_output_rebuild = Arc::new(AtomicBool::new(false)); + // Consumer moved directly without Mutex wrapping let output_stream = build_output_stream( &output_device, &output_config.lock().unwrap(), - consumer_lock.clone(), + consumer, needs_output_rebuild.clone(), )?; output_stream .play() .map_err(|e| format!("Failed to start output: {e}"))?; - let output_stream = Arc::new(Mutex::new(output_stream)); - // ------------------------------------------------------------------------ - // INPUT - // ------------------------------------------------------------------------ - + // Setup input stream with stack/pre-allocated buffers let mut input_config: cpal::StreamConfig = input_device .default_input_config() .map_err(|e| e.to_string())? .into(); - input_config.channels = 1; let input_sample_rate = input_config.sample_rate; - let input_socket = socket.clone(); let mut input_resampler = resampler::ResamplerFft::new( @@ -223,9 +193,10 @@ pub fn connect_to_vc( resampler::SampleRate::Hz44100, ); - let mut input_buffer = Vec::::new(); - let mut packet_buffer = Vec::::new(); + let mut input_buffer = Vec::with_capacity(4096); + let mut packet_buffer = Vec::with_capacity(4096); let mut sequence = 0u32; + let mut net_packet = vec![0u8; HEADER_SIZE + PACKET_SAMPLES * 2]; let input_stream = input_device .build_input_stream( @@ -237,92 +208,34 @@ pub fn connect_to_vc( let output_size = input_resampler.chunk_size_output(); while input_buffer.len() >= input_size { - let input: Vec = input_buffer.drain(..input_size).collect(); - + let input_chunk: Vec = input_buffer.drain(..input_size).collect(); let mut output = vec![0.0; output_size]; - if input_resampler.resample(&input, &mut output).is_err() { - continue; - } + if input_resampler.resample(&input_chunk, &mut output).is_ok() { + packet_buffer.extend(output); - packet_buffer.extend(output); + while packet_buffer.len() >= PACKET_SAMPLES { + let samples = packet_buffer.drain(..PACKET_SAMPLES); - while packet_buffer.len() >= PACKET_SAMPLES { - let samples: Vec = packet_buffer.drain(..PACKET_SAMPLES).collect(); + net_packet[0..4].copy_from_slice(&sequence.to_be_bytes()); + for (i, sample) in samples.enumerate() { + let pcm = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; + let offset = HEADER_SIZE + i * 2; + net_packet[offset..offset + 2].copy_from_slice(&pcm.to_be_bytes()); + } - let mut packet = Vec::with_capacity(HEADER_SIZE + PACKET_SAMPLES * 2); - - packet.extend_from_slice(&sequence.to_be_bytes()); - - for sample in samples { - let pcm = (sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; - - packet.extend_from_slice(&pcm.to_be_bytes()); + let _ = input_socket.send(&net_packet); + sequence = sequence.wrapping_add(1); } - - let _ = input_socket.send(&packet); - - sequence = sequence.wrapping_add(1); } } }, - |err| { - eprintln!("[vc] input error: {err}"); - }, + |err| eprintln!("[vc] input error: {err}"), None, ) .map_err(|e| e.to_string())?; - // ------------------------------------------------------------------------ - // OUTPUT DEVICE REBUILD WATCHER - // ------------------------------------------------------------------------ - - { - let output_device = output_device.clone(); - let output_stream = output_stream.clone(); - let output_config = output_config.clone(); - let consumer_lock = consumer_lock.clone(); - let needs_rebuild = needs_output_rebuild.clone(); - let shutdown = shutdown.clone(); - - thread::spawn(move || { - while !shutdown.load(Ordering::SeqCst) { - thread::sleep(Duration::from_millis(100)); - - if !needs_rebuild.swap(false, Ordering::SeqCst) { - continue; - } - - let Ok(new_config) = build_output_config(&output_device) else { - needs_rebuild.store(true, Ordering::SeqCst); - continue; - }; - - let Ok(new_stream) = build_output_stream( - &output_device, - &new_config, - consumer_lock.clone(), - needs_rebuild.clone(), - ) else { - needs_rebuild.store(true, Ordering::SeqCst); - continue; - }; - - if new_stream.play().is_err() { - needs_rebuild.store(true, Ordering::SeqCst); - continue; - } - - *output_config.lock().unwrap() = new_config; - *output_stream.lock().unwrap() = new_stream; - } - }); - } - - // ------------------------------------------------------------------------ - // UDP RECEIVE (JITTER BUFFER INCLUDED) - // ------------------------------------------------------------------------ - + // UDP Receiver Thread { let socket = socket.clone(); let output_config = output_config.clone(); @@ -330,182 +243,88 @@ pub fn connect_to_vc( thread::spawn(move || { let initial_config = output_config.lock().unwrap().clone(); - let mut output_resampler = match create_output_resampler(initial_config.sample_rate) { Ok(r) => r, - Err(e) => { - eprintln!("[vc] output resampler: {e}"); - return; - } + Err(e) => return eprintln!("[vc] output resampler: {e}"), }; let mut last_sample_rate = initial_config.sample_rate; - let mut packets: BTreeMap> = BTreeMap::new(); let mut expected: Option = None; let mut is_prebuffering = true; - let mut resample_buffer = Vec::::new(); + let mut resample_buffer = Vec::with_capacity(8192); let mut udp_buffer = [0u8; MAX_PACKET_SIZE]; - while !shutdown.load(Ordering::SeqCst) { - match socket.recv(&mut udp_buffer) { - Ok(len) => { - if len <= HEADER_SIZE { - continue; - } - - let sequence = u32::from_be_bytes([ - udp_buffer[0], - udp_buffer[1], - udp_buffer[2], - udp_buffer[3], - ]); - + while !shutdown.load(Ordering::Relaxed) { + if let Ok(len) = socket.recv(&mut udp_buffer) { + if len > HEADER_SIZE { + let sequence = u32::from_be_bytes(udp_buffer[0..4].try_into().unwrap()); let pcm = &udp_buffer[HEADER_SIZE..len]; - - let mut samples = Vec::with_capacity(pcm.len() / 2); - - for chunk in pcm.chunks_exact(2) { - let value = i16::from_be_bytes([chunk[0], chunk[1]]); - - samples.push(value as f32 / i16::MAX as f32); - } + let samples: Vec = pcm + .chunks_exact(2) + .map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32768.0) + .collect(); packets.entry(sequence).or_insert(samples); } - - Err(e) if e.kind() == std::io::ErrorKind::TimedOut => { - // Reset jitter state if connection completely drops - if packets.is_empty() { - expected = None; - is_prebuffering = true; - } - continue; - } - - Err(e) => { - if !shutdown.load(Ordering::SeqCst) { - eprintln!("[vc] UDP receive error: {e}"); - } - } } - // ---------------------------------------------------------------- - // OUTPUT CONFIG CHANGE - // ---------------------------------------------------------------- - - let config = output_config.lock().unwrap().clone(); - - if config.sample_rate != last_sample_rate { - match create_output_resampler(config.sample_rate) { - Ok(new_resampler) => { - output_resampler = new_resampler; - last_sample_rate = config.sample_rate; - - packets.clear(); - resample_buffer.clear(); - expected = None; - is_prebuffering = true; - } - Err(_) => continue, + let current_sr = output_config.lock().unwrap().sample_rate; + if current_sr != last_sample_rate { + if let Ok(nr) = create_output_resampler(current_sr) { + output_resampler = nr; + last_sample_rate = current_sr; + packets.clear(); + expected = None; + is_prebuffering = true; } } - // ---------------------------------------------------------------- - // JITTER BUFFER STATE MACHINE - // ---------------------------------------------------------------- - if is_prebuffering { - // Accumulate target packet cushion before popping if packets.len() >= INITIAL_PACKET_CUSHION { expected = packets.keys().next().copied(); is_prebuffering = false; } else { + thread::sleep(Duration::from_millis(2)); continue; } } - // Initialize sequence if unset - if expected.is_none() { - expected = packets.keys().next().copied(); - } - - // ---------------------------------------------------------------- - // DRAIN PACKETS IN SEQUENTIAL ORDER - // ---------------------------------------------------------------- - while let Some(seq) = expected { if let Some(samples) = packets.remove(&seq) { resample_buffer.extend(samples); expected = Some(seq.wrapping_add(1)); + } else if packets.keys().any(|&x| x > seq) { + resample_buffer.extend(std::iter::repeat(0.0).take(PACKET_SAMPLES)); + expected = Some(seq.wrapping_add(1)); } else { - // Missing frame strategy: - // If future sequence numbers exist, insert Concealment (Silence) - if packets.keys().any(|&x| x > seq) { - resample_buffer.extend(std::iter::repeat(0.0).take(PACKET_SAMPLES)); - expected = Some(seq.wrapping_add(1)); - } else { - // Waiting on late packets - break; - } + break; } } - // Trim jitter buffer cache to avoid memory leak spikes on extreme latency drops - if packets.len() > 50 { - packets.clear(); - expected = None; - is_prebuffering = true; - } - - // ---------------------------------------------------------------- - // RESAMPLE & POPULATE RINGBUFFER - // ---------------------------------------------------------------- - let input_size = output_resampler.chunk_size_input(); let output_size = output_resampler.chunk_size_output(); while resample_buffer.len() >= input_size { - let input: Vec = resample_buffer.drain(..input_size).collect(); - + let input_chunk: Vec = resample_buffer.drain(..input_size).collect(); let mut output = vec![0.0; output_size]; - if output_resampler.resample(&input, &mut output).is_err() { - continue; - } - - let mut prod = producer_lock.lock().unwrap(); - - // Latency Drift Protection: prevent total queue build-up past ~60ms - let target_sample_rate = config.sample_rate as usize; - let max_ring_buffer_samples = (target_sample_rate / 1000) * 60; - - // Fix occupied_len check and overflow protection - // Note: We avoid calling try_pop() on a Producer. If the buffer is full, - // try_push will naturally fail and drop the overflow sample. - for sample in output { - if (*prod).occupied_len() < max_ring_buffer_samples { - let _ = (*prod).try_push(sample); - } + if output_resampler.resample(&input_chunk, &mut output).is_ok() { + // Push directly without Mutex lock! + let _ = producer.push_slice(&output); } } + + thread::sleep(Duration::from_millis(2)); } }); } - // ------------------------------------------------------------------------ - // START INPUT - // ------------------------------------------------------------------------ - input_stream .play() .map_err(|e| format!("Failed to start input: {e}"))?; - // ------------------------------------------------------------------------ - // SAVE - // ------------------------------------------------------------------------ - *voice_state.session.lock().unwrap() = Some(VoiceSession { input_stream, output_stream, @@ -517,10 +336,6 @@ pub fn connect_to_vc( Ok(()) } -// ============================================================================ -// OUTPUT CONFIG -// ============================================================================ - fn build_output_config(device: &cpal::Device) -> Result { device .default_output_config() @@ -528,10 +343,6 @@ fn build_output_config(device: &cpal::Device) -> Result Result { @@ -546,69 +357,38 @@ fn create_output_resampler( )) } -// ============================================================================ -// OUTPUT STREAM -// ============================================================================ - fn build_output_stream( device: &cpal::Device, config: &cpal::StreamConfig, - consumer_lock: Arc>, + mut consumer: C, needs_rebuild: Arc, ) -> Result where C: Consumer + Send + 'static, { let channels = config.channels as usize; - let sample_rate = config.sample_rate as usize; - let jitter_cushion_samples = (sample_rate / 1000) * 40; // 40ms stream cushion - - let mut is_buffering = true; device .build_output_stream( config.clone(), move |data: &mut [f32], _| { - let mut cons = consumer_lock.lock().unwrap(); - - // 1. Initial/Recovering Cushioning - if is_buffering { - if (*cons).occupied_len() >= jitter_cushion_samples { - is_buffering = false; - } else { - data.fill(0.0); - return; - } - } - - // 2. Sample Extraction & Channel Interleaving let mut idx = 0; while idx < data.len() { - if let Some(mono_sample) = (*cons).try_pop() { + if let Some(mono_sample) = consumer.try_pop() { for ch in 0..channels { data[idx + ch] = mono_sample; } idx += channels; } else { - // 3. Underrun: pad rest with silence & re-enter buffering mode data[idx..].fill(0.0); - is_buffering = true; break; } } }, - { - let needs_rebuild = needs_rebuild.clone(); - move |err| { - eprintln!("[vc] output error: {err}"); - let message = err.to_string(); - if message.contains("sample rate changed") - || message.contains("DeviceNotAvailable") - || message.contains("device not available") - { - needs_rebuild.store(true, Ordering::SeqCst); - } - } + move |err| { + eprintln!("[vc] output error: {err}"); + + needs_rebuild.store(true, Ordering::SeqCst); }, None, ) From a674b4c53a9c3aaba23300a7c6473391cd3476b8 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 00:47:28 +0200 Subject: [PATCH 21/29] idk --- src-tauri/src/commands/audio.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index d59e24c..193f1d1 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -11,7 +11,7 @@ use std::{ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use ringbuf::{ - traits::{Consumer, Observer, Producer, Split}, + traits::{Consumer, Producer, Split}, HeapRb, }; use tauri::State; From 570fae9cfd13d60dfe81aaf8f6623b2c67520a7d Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 00:55:09 +0200 Subject: [PATCH 22/29] Fixing choppy audio (again) --- src-tauri/src/commands/audio.rs | 211 ++++++++++++-------------------- 1 file changed, 78 insertions(+), 133 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 193f1d1..4774a33 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -18,14 +18,14 @@ use tauri::State; use crate::commands::config::ConfigState; -const PACKET_SAMPLES: usize = 882; // 20ms @ 44.1kHz +// Standardize to 48kHz (Native for WebAudio and low-latency VoIP) +const SAMPLE_RATE: u32 = 48000; +const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz const HEADER_SIZE: usize = 4; const MAX_PACKET_SIZE: usize = 4096; -// Jitter buffer settings -const JITTER_BUFFER_MS: usize = 60; // Cushion size before playback starts -const PACKET_DURATION_MS: usize = 20; -const INITIAL_PACKET_CUSHION: usize = JITTER_BUFFER_MS / PACKET_DURATION_MS; // 3 packets +const JITTER_BUFFER_MS: usize = 60; +const INITIAL_PACKET_CUSHION: usize = JITTER_BUFFER_MS / 20; // 3 packets // ============================================================================ // STATE @@ -58,7 +58,6 @@ impl Default for VoiceState { #[tauri::command] pub fn list_input_devices() -> Result, String> { let host = cpal::default_host(); - Ok(host .input_devices() .map_err(|e| e.to_string())? @@ -69,7 +68,6 @@ pub fn list_input_devices() -> Result, String> { #[tauri::command] pub fn list_output_devices() -> Result, String> { let host = cpal::default_host(); - Ok(host .output_devices() .map_err(|e| e.to_string())? @@ -86,7 +84,6 @@ pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> if let Some(session) = voice_state.session.lock().unwrap().take() { session.shutdown.store(true, Ordering::SeqCst); } - Ok(()) } @@ -112,7 +109,7 @@ pub fn connect_to_vc( .connect(&hostname) .map_err(|e| format!("Failed to connect UDP socket: {e}"))?; socket - .set_read_timeout(Some(Duration::from_millis(50))) + .set_read_timeout(Some(Duration::from_millis(10))) .map_err(|e| e.to_string())?; socket .send(&pin.to_be_bytes()) @@ -154,47 +151,31 @@ pub fn connect_to_vc( let shutdown = Arc::new(AtomicBool::new(false)); - // Lock-Free Ring Buffer setup (Hold 100ms max buffer) - let rb = HeapRb::::new(8820); + // Lock-Free Ring Buffer setup (19.2k samples ~ 400ms buffer capacity at 48kHz) + let rb = HeapRb::::new(19200); let (mut producer, consumer) = rb.split(); - let output_config = build_output_config(&output_device)?; - let output_config = Arc::new(Mutex::new(output_config)); - let needs_output_rebuild = Arc::new(AtomicBool::new(false)); - - // Consumer moved directly without Mutex wrapping - let output_stream = build_output_stream( - &output_device, - &output_config.lock().unwrap(), - consumer, - needs_output_rebuild.clone(), - )?; + let output_config = cpal::StreamConfig { + channels: 2, + sample_rate: SAMPLE_RATE, + buffer_size: cpal::BufferSize::Default, + }; + let output_stream = build_output_stream(&output_device, &output_config, consumer)?; output_stream .play() .map_err(|e| format!("Failed to start output: {e}"))?; let output_stream = Arc::new(Mutex::new(output_stream)); - // Setup input stream with stack/pre-allocated buffers - let mut input_config: cpal::StreamConfig = input_device - .default_input_config() - .map_err(|e| e.to_string())? - .into(); - input_config.channels = 1; + // Configure Input Stream + let input_config = cpal::StreamConfig { + channels: 1, + sample_rate: SAMPLE_RATE, + buffer_size: cpal::BufferSize::Default, + }; - let input_sample_rate = input_config.sample_rate; let input_socket = socket.clone(); - - let mut input_resampler = resampler::ResamplerFft::new( - 1, - input_sample_rate - .try_into() - .map_err(|e| format!("Invalid input sample rate: {e:?}"))?, - resampler::SampleRate::Hz44100, - ); - - let mut input_buffer = Vec::with_capacity(4096); - let mut packet_buffer = Vec::with_capacity(4096); + let mut packet_buffer = Vec::with_capacity(PACKET_SAMPLES * 2); let mut sequence = 0u32; let mut net_packet = vec![0u8; HEADER_SIZE + PACKET_SAMPLES * 2]; @@ -202,32 +183,20 @@ pub fn connect_to_vc( .build_input_stream( input_config, move |data: &[f32], _| { - input_buffer.extend_from_slice(data); + packet_buffer.extend_from_slice(data); - let input_size = input_resampler.chunk_size_input(); - let output_size = input_resampler.chunk_size_output(); + while packet_buffer.len() >= PACKET_SAMPLES { + let samples: Vec = packet_buffer.drain(..PACKET_SAMPLES).collect(); - while input_buffer.len() >= input_size { - let input_chunk: Vec = input_buffer.drain(..input_size).collect(); - let mut output = vec![0.0; output_size]; - - if input_resampler.resample(&input_chunk, &mut output).is_ok() { - packet_buffer.extend(output); - - while packet_buffer.len() >= PACKET_SAMPLES { - let samples = packet_buffer.drain(..PACKET_SAMPLES); - - net_packet[0..4].copy_from_slice(&sequence.to_be_bytes()); - for (i, sample) in samples.enumerate() { - let pcm = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; - let offset = HEADER_SIZE + i * 2; - net_packet[offset..offset + 2].copy_from_slice(&pcm.to_be_bytes()); - } - - let _ = input_socket.send(&net_packet); - sequence = sequence.wrapping_add(1); - } + net_packet[0..4].copy_from_slice(&sequence.to_be_bytes()); + for (i, sample) in samples.iter().enumerate() { + let pcm = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; + let offset = HEADER_SIZE + i * 2; + net_packet[offset..offset + 2].copy_from_slice(&pcm.to_be_bytes()); } + + let _ = input_socket.send(&net_packet); + sequence = sequence.wrapping_add(1); } }, |err| eprintln!("[vc] input error: {err}"), @@ -235,49 +204,30 @@ pub fn connect_to_vc( ) .map_err(|e| e.to_string())?; - // UDP Receiver Thread + // UDP Receiver & Jitter Buffer Thread { let socket = socket.clone(); - let output_config = output_config.clone(); let shutdown = shutdown.clone(); thread::spawn(move || { - let initial_config = output_config.lock().unwrap().clone(); - let mut output_resampler = match create_output_resampler(initial_config.sample_rate) { - Ok(r) => r, - Err(e) => return eprintln!("[vc] output resampler: {e}"), - }; - - let mut last_sample_rate = initial_config.sample_rate; let mut packets: BTreeMap> = BTreeMap::new(); let mut expected: Option = None; let mut is_prebuffering = true; - - let mut resample_buffer = Vec::with_capacity(8192); + let mut last_good_frame = vec![0.0f32; PACKET_SAMPLES]; let mut udp_buffer = [0u8; MAX_PACKET_SIZE]; while !shutdown.load(Ordering::Relaxed) { + // Read incoming packets if let Ok(len) = socket.recv(&mut udp_buffer) { if len > HEADER_SIZE { - let sequence = u32::from_be_bytes(udp_buffer[0..4].try_into().unwrap()); + let seq = u32::from_be_bytes(udp_buffer[0..4].try_into().unwrap()); let pcm = &udp_buffer[HEADER_SIZE..len]; let samples: Vec = pcm .chunks_exact(2) .map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32768.0) .collect(); - packets.entry(sequence).or_insert(samples); - } - } - - let current_sr = output_config.lock().unwrap().sample_rate; - if current_sr != last_sample_rate { - if let Ok(nr) = create_output_resampler(current_sr) { - output_resampler = nr; - last_sample_rate = current_sr; - packets.clear(); - expected = None; - is_prebuffering = true; + packets.insert(seq, samples); } } @@ -286,37 +236,45 @@ pub fn connect_to_vc( expected = packets.keys().next().copied(); is_prebuffering = false; } else { - thread::sleep(Duration::from_millis(2)); continue; } } + // Drain ordered frames into ring buffer while let Some(seq) = expected { if let Some(samples) = packets.remove(&seq) { - resample_buffer.extend(samples); + // Dynamically match any incoming frame length without crashing + if samples.len() == PACKET_SAMPLES { + last_good_frame.copy_from_slice(&samples); + } else { + // Resize/truncate safely if source length differs + last_good_frame.clear(); + last_good_frame.extend(samples.iter().take(PACKET_SAMPLES).copied()); + if last_good_frame.len() < PACKET_SAMPLES { + last_good_frame.resize(PACKET_SAMPLES, 0.0); + } + } + + let _ = producer.push_slice(&samples); expected = Some(seq.wrapping_add(1)); } else if packets.keys().any(|&x| x > seq) { - resample_buffer.extend(std::iter::repeat(0.0).take(PACKET_SAMPLES)); + // Packet Loss Concealment (PLC): Decay previous packet amplitude instead of absolute zeros + let mut plc_frame = last_good_frame.clone(); + for sample in plc_frame.iter_mut() { + *sample *= 0.65; // Quick fade out for missing frame + } + last_good_frame.copy_from_slice(&plc_frame); + + let _ = producer.push_slice(&plc_frame); expected = Some(seq.wrapping_add(1)); } else { + // Out of continuous frames, wait for network + if packets.is_empty() { + is_prebuffering = true; + } break; } } - - let input_size = output_resampler.chunk_size_input(); - let output_size = output_resampler.chunk_size_output(); - - while resample_buffer.len() >= input_size { - let input_chunk: Vec = resample_buffer.drain(..input_size).collect(); - let mut output = vec![0.0; output_size]; - - if output_resampler.resample(&input_chunk, &mut output).is_ok() { - // Push directly without Mutex lock! - let _ = producer.push_slice(&output); - } - } - - thread::sleep(Duration::from_millis(2)); } }); } @@ -336,60 +294,47 @@ pub fn connect_to_vc( Ok(()) } -fn build_output_config(device: &cpal::Device) -> Result { - device - .default_output_config() - .map(Into::into) - .map_err(|e| e.to_string()) -} - -fn create_output_resampler( - sample_rate: cpal::SampleRate, -) -> Result { - let rate = sample_rate - .try_into() - .map_err(|e| format!("Invalid sample rate: {e:?}"))?; - - Ok(resampler::ResamplerFft::new( - 1, - resampler::SampleRate::Hz44100, - rate, - )) -} +// ============================================================================ +// AUDIO CALLBACK (Pops Avoidance & Degraded Fill) +// ============================================================================ fn build_output_stream( device: &cpal::Device, config: &cpal::StreamConfig, mut consumer: C, - needs_rebuild: Arc, ) -> Result where C: Consumer + Send + 'static, { let channels = config.channels as usize; + let mut last_sample = 0.0f32; device .build_output_stream( - config.clone(), + *config, move |data: &mut [f32], _| { let mut idx = 0; while idx < data.len() { if let Some(mono_sample) = consumer.try_pop() { + last_sample = mono_sample; for ch in 0..channels { data[idx + ch] = mono_sample; } idx += channels; } else { - data[idx..].fill(0.0); + // Smooth de-zippering / anti-pop decay on buffer underruns + while idx < data.len() { + last_sample *= 0.92; // Rapid smooth fade to silence + for ch in 0..channels { + data[idx + ch] = last_sample; + } + idx += channels; + } break; } } }, - move |err| { - eprintln!("[vc] output error: {err}"); - - needs_rebuild.store(true, Ordering::SeqCst); - }, + |err| eprintln!("[vc] output error: {err}"), None, ) .map_err(|e| e.to_string()) From d450e5e66e2e78192db0bb2e3eab7121702876c3 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 01:00:12 +0200 Subject: [PATCH 23/29] Updated packet cusion --- src-tauri/src/commands/audio.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 4774a33..5f0785d 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -24,8 +24,7 @@ const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz const HEADER_SIZE: usize = 4; const MAX_PACKET_SIZE: usize = 4096; -const JITTER_BUFFER_MS: usize = 60; -const INITIAL_PACKET_CUSHION: usize = JITTER_BUFFER_MS / 20; // 3 packets +const INITIAL_PACKET_CUSHION: usize = 5; // ============================================================================ // STATE From f28c9c0bdf83c2430b4162b67c2d014a7f8116e1 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 01:03:14 +0200 Subject: [PATCH 24/29] Working good quality audio --- src-tauri/src/commands/audio.rs | 134 ++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 59 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 5f0785d..4e0aea7 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -6,25 +6,24 @@ use std::{ Arc, Mutex, }, thread, - time::Duration, + time::{Duration, Instant}, }; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use ringbuf::{ - traits::{Consumer, Producer, Split}, + traits::{Consumer, Observer, Producer, Split}, HeapRb, }; use tauri::State; use crate::commands::config::ConfigState; -// Standardize to 48kHz (Native for WebAudio and low-latency VoIP) const SAMPLE_RATE: u32 = 48000; const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz const HEADER_SIZE: usize = 4; const MAX_PACKET_SIZE: usize = 4096; -const INITIAL_PACKET_CUSHION: usize = 5; +const INITIAL_PACKET_CUSHION: usize = 3; // ~60ms cushion // ============================================================================ // STATE @@ -108,7 +107,7 @@ pub fn connect_to_vc( .connect(&hostname) .map_err(|e| format!("Failed to connect UDP socket: {e}"))?; socket - .set_read_timeout(Some(Duration::from_millis(10))) + .set_read_timeout(Some(Duration::from_millis(5))) .map_err(|e| e.to_string())?; socket .send(&pin.to_be_bytes()) @@ -151,8 +150,8 @@ pub fn connect_to_vc( let shutdown = Arc::new(AtomicBool::new(false)); // Lock-Free Ring Buffer setup (19.2k samples ~ 400ms buffer capacity at 48kHz) - let rb = HeapRb::::new(19200); - let (mut producer, consumer) = rb.split(); + let rb_out = HeapRb::::new(19200); + let (mut producer_out, consumer_out) = rb_out.split(); let output_config = cpal::StreamConfig { channels: 2, @@ -160,35 +159,49 @@ pub fn connect_to_vc( buffer_size: cpal::BufferSize::Default, }; - let output_stream = build_output_stream(&output_device, &output_config, consumer)?; + let output_stream = build_output_stream(&output_device, output_config, consumer_out)?; output_stream .play() .map_err(|e| format!("Failed to start output: {e}"))?; let output_stream = Arc::new(Mutex::new(output_stream)); - // Configure Input Stream + // Configure Input Stream with dedicated thread ring-buffer let input_config = cpal::StreamConfig { channels: 1, sample_rate: SAMPLE_RATE, buffer_size: cpal::BufferSize::Default, }; - let input_socket = socket.clone(); - let mut packet_buffer = Vec::with_capacity(PACKET_SAMPLES * 2); - let mut sequence = 0u32; - let mut net_packet = vec![0u8; HEADER_SIZE + PACKET_SAMPLES * 2]; + let rb_in = HeapRb::::new(19200); + let (mut producer_in, mut consumer_in) = rb_in.split(); let input_stream = input_device .build_input_stream( input_config, move |data: &[f32], _| { - packet_buffer.extend_from_slice(data); + let _ = producer_in.push_slice(data); + }, + |err| eprintln!("[vc] input error: {err}"), + None, + ) + .map_err(|e| e.to_string())?; - while packet_buffer.len() >= PACKET_SAMPLES { - let samples: Vec = packet_buffer.drain(..PACKET_SAMPLES).collect(); + // UDP Sender Thread - Pumps exact 20ms frames smoothly + { + let input_socket = socket.clone(); + let shutdown = shutdown.clone(); + + thread::spawn(move || { + let mut sequence = 0u32; + let mut net_packet = vec![0u8; HEADER_SIZE + PACKET_SAMPLES * 2]; + let mut frame_buf = vec![0.0f32; PACKET_SAMPLES]; + + while !shutdown.load(Ordering::Relaxed) { + if consumer_in.occupied_len() >= PACKET_SAMPLES { + let _ = consumer_in.pop_slice(&mut frame_buf); net_packet[0..4].copy_from_slice(&sequence.to_be_bytes()); - for (i, sample) in samples.iter().enumerate() { + for (i, sample) in frame_buf.iter().enumerate() { let pcm = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; let offset = HEADER_SIZE + i * 2; net_packet[offset..offset + 2].copy_from_slice(&pcm.to_be_bytes()); @@ -196,14 +209,14 @@ pub fn connect_to_vc( let _ = input_socket.send(&net_packet); sequence = sequence.wrapping_add(1); + } else { + thread::sleep(Duration::from_millis(2)); } - }, - |err| eprintln!("[vc] input error: {err}"), - None, - ) - .map_err(|e| e.to_string())?; + } + }); + } - // UDP Receiver & Jitter Buffer Thread + // UDP Receiver Thread - Paced Jitter Buffer { let socket = socket.clone(); let shutdown = shutdown.clone(); @@ -214,9 +227,10 @@ pub fn connect_to_vc( let mut is_prebuffering = true; let mut last_good_frame = vec![0.0f32; PACKET_SAMPLES]; let mut udp_buffer = [0u8; MAX_PACKET_SIZE]; + let mut next_frame_time = Instant::now(); while !shutdown.load(Ordering::Relaxed) { - // Read incoming packets + // Non-blocking UDP receive if let Ok(len) = socket.recv(&mut udp_buffer) { if len > HEADER_SIZE { let seq = u32::from_be_bytes(udp_buffer[0..4].try_into().unwrap()); @@ -234,46 +248,48 @@ pub fn connect_to_vc( if packets.len() >= INITIAL_PACKET_CUSHION { expected = packets.keys().next().copied(); is_prebuffering = false; + next_frame_time = Instant::now(); } else { + thread::sleep(Duration::from_millis(1)); continue; } } - // Drain ordered frames into ring buffer - while let Some(seq) = expected { - if let Some(samples) = packets.remove(&seq) { - // Dynamically match any incoming frame length without crashing - if samples.len() == PACKET_SAMPLES { - last_good_frame.copy_from_slice(&samples); + // Paced playback frame-by-frame (20ms target interval) + if Instant::now() >= next_frame_time { + if let Some(seq) = expected { + if let Some(samples) = packets.remove(&seq) { + if samples.len() == PACKET_SAMPLES { + last_good_frame.copy_from_slice(&samples); + } else { + last_good_frame.clear(); + last_good_frame + .extend(samples.iter().take(PACKET_SAMPLES).copied()); + if last_good_frame.len() < PACKET_SAMPLES { + last_good_frame.resize(PACKET_SAMPLES, 0.0); + } + } + + let _ = producer_out.push_slice(&last_good_frame); + expected = Some(seq.wrapping_add(1)); + } else if packets.keys().any(|&x| x > seq) { + // PLC (Packet Loss Concealment) + for sample in last_good_frame.iter_mut() { + *sample *= 0.65; + } + let _ = producer_out.push_slice(&last_good_frame); + expected = Some(seq.wrapping_add(1)); } else { - // Resize/truncate safely if source length differs - last_good_frame.clear(); - last_good_frame.extend(samples.iter().take(PACKET_SAMPLES).copied()); - if last_good_frame.len() < PACKET_SAMPLES { - last_good_frame.resize(PACKET_SAMPLES, 0.0); + // Re-buffer if packet stream dropped + if packets.is_empty() { + is_prebuffering = true; } } - - let _ = producer.push_slice(&samples); - expected = Some(seq.wrapping_add(1)); - } else if packets.keys().any(|&x| x > seq) { - // Packet Loss Concealment (PLC): Decay previous packet amplitude instead of absolute zeros - let mut plc_frame = last_good_frame.clone(); - for sample in plc_frame.iter_mut() { - *sample *= 0.65; // Quick fade out for missing frame - } - last_good_frame.copy_from_slice(&plc_frame); - - let _ = producer.push_slice(&plc_frame); - expected = Some(seq.wrapping_add(1)); - } else { - // Out of continuous frames, wait for network - if packets.is_empty() { - is_prebuffering = true; - } - break; } + next_frame_time += Duration::from_millis(20); } + + thread::sleep(Duration::from_millis(1)); } }); } @@ -294,12 +310,12 @@ pub fn connect_to_vc( } // ============================================================================ -// AUDIO CALLBACK (Pops Avoidance & Degraded Fill) +// AUDIO CALLBACK // ============================================================================ fn build_output_stream( device: &cpal::Device, - config: &cpal::StreamConfig, + config: cpal::StreamConfig, mut consumer: C, ) -> Result where @@ -310,7 +326,7 @@ where device .build_output_stream( - *config, + config, move |data: &mut [f32], _| { let mut idx = 0; while idx < data.len() { @@ -321,9 +337,9 @@ where } idx += channels; } else { - // Smooth de-zippering / anti-pop decay on buffer underruns + // Smooth fade out to avoid clicks on underruns while idx < data.len() { - last_sample *= 0.92; // Rapid smooth fade to silence + last_sample *= 0.92; for ch in 0..channels { data[idx + ch] = last_sample; } From 23e6a2199fdb834ca5038ccf6948ddf6a8262fa2 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 14:42:35 +0200 Subject: [PATCH 25/29] Idk --- src-tauri/src/commands/audio.rs | 190 +++++++++++++++++++++++++------- src/app/app.ts | 11 ++ src/app/protocol.ts | 11 +- src/app/server.ts | 14 +++ 4 files changed, 186 insertions(+), 40 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index 4e0aea7..b9ae8a8 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -11,14 +11,14 @@ use std::{ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use ringbuf::{ + storage::Heap, traits::{Consumer, Observer, Producer, Split}, - HeapRb, + CachingCons, CachingProd, HeapRb, SharedRb, }; use tauri::State; use crate::commands::config::ConfigState; -const SAMPLE_RATE: u32 = 48000; const PACKET_SAMPLES: usize = 960; // 20ms @ 48kHz const HEADER_SIZE: usize = 4; const MAX_PACKET_SIZE: usize = 4096; @@ -26,9 +26,12 @@ const MAX_PACKET_SIZE: usize = 4096; const INITIAL_PACKET_CUSHION: usize = 3; // ~60ms cushion // ============================================================================ -// STATE +// STATE & TYPES // ============================================================================ +type AudioProducer = Arc>>>>>; +type AudioConsumer = Arc>>>>>; + pub struct VoiceState { pub session: Mutex>, } @@ -39,6 +42,102 @@ pub struct VoiceSession { pub socket: Arc, pub pin: u64, pub shutdown: Arc, + + // Tracked settings for diffing config changes + pub current_input_device: Option, + pub current_output_device: Option, + + // Shared ringbuffer handles for runtime hot-swapping + pub producer_in: AudioProducer, + pub consumer_out: AudioConsumer, +} + +impl VoiceSession { + /// Re-binds the input stream (microphone) without disconnecting the UDP thread + pub fn update_input_device(&mut self, device_name: Option) -> Result<(), String> { + let host = cpal::default_host(); + let device = match &device_name { + Some(name) => host + .input_devices() + .map_err(|e| e.to_string())? + .find(|d| { + d.description() + .ok() + .map(|x| x.name() == *name) + .unwrap_or(false) + }) + .ok_or_else(|| format!("Input device '{name}' not found"))?, + None => host + .default_input_device() + .ok_or("No default input device")?, + }; + + let input_config = device + .default_input_config() + .map_err(|e| format!("Failed to get default input config: {e}"))? + .config(); + + let producer = Arc::clone(&self.producer_in); + let new_stream = device + .build_input_stream( + input_config, + move |data: &[f32], _| { + if let Ok(mut prod) = producer.lock() { + let _ = prod.push_slice(data); + } + }, + |err| eprintln!("[vc] input error: {err}"), + None, + ) + .map_err(|e| e.to_string())?; + + new_stream + .play() + .map_err(|e| format!("Failed to play input stream: {e}"))?; + + // Dropping old stream stops capturing audio hardware + self.input_stream = new_stream; + self.current_input_device = device_name; + Ok(()) + } + + /// Re-binds the output stream (speakers/headphones) without interrupting UDP reception + pub fn update_output_device(&mut self, device_name: Option) -> Result<(), String> { + let host = cpal::default_host(); + let device = match &device_name { + Some(name) => host + .output_devices() + .map_err(|e| e.to_string())? + .find(|d| { + d.description() + .ok() + .map(|x| x.name() == *name) + .unwrap_or(false) + }) + .ok_or_else(|| format!("Output device '{name}' not found"))?, + None => host + .default_output_device() + .ok_or("No default output device")?, + }; + + let output_config = device + .default_output_config() + .map_err(|e| format!("Failed to get default output config: {e}"))? + .config(); + + let consumer = Arc::clone(&self.consumer_out); + let new_stream = build_output_stream(&device, output_config, consumer)?; + new_stream + .play() + .map_err(|e| format!("Failed to play output stream: {e}"))?; + + if let Ok(mut active_stream) = self.output_stream.lock() { + *active_stream = new_stream; + } + + self.current_output_device = device_name; + Ok(()) + } } impl Default for VoiceState { @@ -74,13 +173,20 @@ pub fn list_output_devices() -> Result, String> { } // ============================================================================ -// DISCONNECT +// CONFIG UPDATES & DISCONNECT // ============================================================================ #[tauri::command] pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { if let Some(session) = voice_state.session.lock().unwrap().take() { + // Signal shutdown atomic to terminate sender and receiver loops session.shutdown.store(true, Ordering::SeqCst); + + // Pause CPAL hardware streams explicitly to flush playback drivers immediately + let _ = session.input_stream.pause(); + if let Ok(output) = session.output_stream.lock() { + let _ = output.pause(); + } } Ok(()) } @@ -149,44 +255,53 @@ pub fn connect_to_vc( let shutdown = Arc::new(AtomicBool::new(false)); - // Lock-Free Ring Buffer setup (19.2k samples ~ 400ms buffer capacity at 48kHz) + // Output Ring Buffer setup let rb_out = HeapRb::::new(19200); - let (mut producer_out, consumer_out) = rb_out.split(); + let (producer_out, consumer_out) = rb_out.split(); + let mut producer_out = producer_out; // Handed to receiver thread + let shared_consumer_out = Arc::new(Mutex::new(consumer_out)); - let output_config = cpal::StreamConfig { - channels: 2, - sample_rate: SAMPLE_RATE, - buffer_size: cpal::BufferSize::Default, - }; + // Replace hardcoded output_config: + let output_config = output_device + .default_output_config() + .map_err(|e| format!("Failed to get default output config: {e}"))? + .config(); - let output_stream = build_output_stream(&output_device, output_config, consumer_out)?; + let output_stream = build_output_stream( + &output_device, + output_config, + Arc::clone(&shared_consumer_out), + )?; output_stream .play() .map_err(|e| format!("Failed to start output: {e}"))?; let output_stream = Arc::new(Mutex::new(output_stream)); - // Configure Input Stream with dedicated thread ring-buffer - let input_config = cpal::StreamConfig { - channels: 1, - sample_rate: SAMPLE_RATE, - buffer_size: cpal::BufferSize::Default, - }; - + // Input Ring Buffer setup let rb_in = HeapRb::::new(19200); - let (mut producer_in, mut consumer_in) = rb_in.split(); + let (producer_in, mut consumer_in) = rb_in.split(); + let shared_producer_in = Arc::new(Mutex::new(producer_in)); + let input_config = input_device + .default_input_config() + .map_err(|e| format!("Failed to get default input config: {e}"))? + .config(); + + let cb_producer = Arc::clone(&shared_producer_in); let input_stream = input_device .build_input_stream( input_config, move |data: &[f32], _| { - let _ = producer_in.push_slice(data); + if let Ok(mut prod) = cb_producer.lock() { + let _ = prod.push_slice(data); + } }, |err| eprintln!("[vc] input error: {err}"), None, ) .map_err(|e| e.to_string())?; - // UDP Sender Thread - Pumps exact 20ms frames smoothly + // UDP Sender Thread { let input_socket = socket.clone(); let shutdown = shutdown.clone(); @@ -216,7 +331,7 @@ pub fn connect_to_vc( }); } - // UDP Receiver Thread - Paced Jitter Buffer + // UDP Receiver Thread { let socket = socket.clone(); let shutdown = shutdown.clone(); @@ -230,7 +345,6 @@ pub fn connect_to_vc( let mut next_frame_time = Instant::now(); while !shutdown.load(Ordering::Relaxed) { - // Non-blocking UDP receive if let Ok(len) = socket.recv(&mut udp_buffer) { if len > HEADER_SIZE { let seq = u32::from_be_bytes(udp_buffer[0..4].try_into().unwrap()); @@ -255,7 +369,6 @@ pub fn connect_to_vc( } } - // Paced playback frame-by-frame (20ms target interval) if Instant::now() >= next_frame_time { if let Some(seq) = expected { if let Some(samples) = packets.remove(&seq) { @@ -273,17 +386,13 @@ pub fn connect_to_vc( let _ = producer_out.push_slice(&last_good_frame); expected = Some(seq.wrapping_add(1)); } else if packets.keys().any(|&x| x > seq) { - // PLC (Packet Loss Concealment) for sample in last_good_frame.iter_mut() { *sample *= 0.65; } let _ = producer_out.push_slice(&last_good_frame); expected = Some(seq.wrapping_add(1)); - } else { - // Re-buffer if packet stream dropped - if packets.is_empty() { - is_prebuffering = true; - } + } else if packets.is_empty() { + is_prebuffering = true; } } next_frame_time += Duration::from_millis(20); @@ -304,6 +413,10 @@ pub fn connect_to_vc( socket, pin, shutdown, + current_input_device: config.input_device_name, + current_output_device: config.output_device_name, + producer_in: shared_producer_in, + consumer_out: shared_consumer_out, }); Ok(()) @@ -313,14 +426,11 @@ pub fn connect_to_vc( // AUDIO CALLBACK // ============================================================================ -fn build_output_stream( +fn build_output_stream( device: &cpal::Device, config: cpal::StreamConfig, - mut consumer: C, -) -> Result -where - C: Consumer + Send + 'static, -{ + consumer: AudioConsumer, +) -> Result { let channels = config.channels as usize; let mut last_sample = 0.0f32; @@ -329,15 +439,17 @@ where config, move |data: &mut [f32], _| { let mut idx = 0; + let mut cons_guard = consumer.lock().ok(); + while idx < data.len() { - if let Some(mono_sample) = consumer.try_pop() { + let sample = cons_guard.as_mut().and_then(|c| c.try_pop()); + if let Some(mono_sample) = sample { last_sample = mono_sample; for ch in 0..channels { data[idx + ch] = mono_sample; } idx += channels; } else { - // Smooth fade out to avoid clicks on underruns while idx < data.len() { last_sample *= 0.92; for ch in 0..channels { diff --git a/src/app/app.ts b/src/app/app.ts index a34731d..de7e654 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -210,6 +210,17 @@ export default class Enclave

{ case "JoinVoice": server.voiceJoin && server.voiceJoin(msg.pin, msg.channel_id); return; + + case "UserJoinedVoice": + if (!server.voiceChatUsers[msg.channel_id]) + server.voiceChatUsers[msg.channel_id] = []; + + server.voiceChatUsers[msg.channel_id].push(msg.pubkey); + return; + + case "Speaking": + server.voiceChatSpeaker(msg.pubkey); + return; } } } diff --git a/src/app/protocol.ts b/src/app/protocol.ts index db6a5af..93f5f49 100644 --- a/src/app/protocol.ts +++ b/src/app/protocol.ts @@ -33,7 +33,16 @@ export type ClientMethod = | { method: "JoinVoice"; channel_id: string; - pin: number; + pin: bigint; + } + | { + method: "UserJoinedVoice"; + channel_id: string; + pubkey: string; + } + | { + method: "Speaking"; + pubkey: string; }; export type ServerMethod = diff --git a/src/app/server.ts b/src/app/server.ts index 525e121..35efabf 100644 --- a/src/app/server.ts +++ b/src/app/server.ts @@ -29,15 +29,21 @@ export default class EnclaveServer { public isSecure: boolean; public websocket?: EnclaveWebSocket; public meta?: ServerMeta; + public messages: Record>; public users: Record; + public voiceJoin?: (pin: number, channelId: string) => void; + public voiceChatUsers: Record; + public voiceChatSpeakers: Record; public constructor(hostname: string, isSecure: boolean) { this.hostname = hostname; this.isSecure = isSecure; this.messages = {}; this.users = {}; + this.voiceChatUsers = {}; + this.voiceChatSpeakers = {}; } public disconnect() { @@ -107,4 +113,12 @@ export default class EnclaveServer { this.websocket?.send({ method: "GetUsers", pubkeys: filteredPubkeys }); } + + public voiceChatSpeaker(pubkey: string) { + clearTimeout(this.voiceChatSpeakers[pubkey]); + + this.voiceChatSpeakers[pubkey] = setTimeout(() => { + delete this.voiceChatSpeakers[pubkey]; + }, 1000); + } } From d7f9b32a99c7d7016c4a2d6988cba380c132b480 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 17:18:00 +0200 Subject: [PATCH 26/29] VAD --- src-tauri/src/commands/audio.rs | 45 +++++++++++++++----- src/components/page/channel/VoiceChannel.tsx | 3 ++ 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index b9ae8a8..ff2ffb3 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -25,6 +25,13 @@ const MAX_PACKET_SIZE: usize = 4096; const INITIAL_PACKET_CUSHION: usize = 3; // ~60ms cushion +// --- Voice Activity Detection (VAD) Settings --- +/// RMS energy threshold to classify audio as speech (Range: 0.0 to 1.0). +/// 0.01 ≈ -40 dBFS. Adjust upward if room noise is triggering transmission. +const VAD_THRESHOLD: f32 = 0.01; +/// Number of 20ms frames to keep transmitting after falling below threshold (~200ms hangover) +const VAD_HANGOVER_FRAMES: usize = 10; + // ============================================================================ // STATE & TYPES // ============================================================================ @@ -261,7 +268,6 @@ pub fn connect_to_vc( let mut producer_out = producer_out; // Handed to receiver thread let shared_consumer_out = Arc::new(Mutex::new(consumer_out)); - // Replace hardcoded output_config: let output_config = output_device .default_output_config() .map_err(|e| format!("Failed to get default output config: {e}"))? @@ -301,7 +307,7 @@ pub fn connect_to_vc( ) .map_err(|e| e.to_string())?; - // UDP Sender Thread + // UDP Sender Thread (With Voice Activity Detection) { let input_socket = socket.clone(); let shutdown = shutdown.clone(); @@ -310,20 +316,39 @@ pub fn connect_to_vc( let mut sequence = 0u32; let mut net_packet = vec![0u8; HEADER_SIZE + PACKET_SAMPLES * 2]; let mut frame_buf = vec![0.0f32; PACKET_SAMPLES]; + let mut hangover_counter = 0; while !shutdown.load(Ordering::Relaxed) { if consumer_in.occupied_len() >= PACKET_SAMPLES { let _ = consumer_in.pop_slice(&mut frame_buf); - net_packet[0..4].copy_from_slice(&sequence.to_be_bytes()); - for (i, sample) in frame_buf.iter().enumerate() { - let pcm = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; - let offset = HEADER_SIZE + i * 2; - net_packet[offset..offset + 2].copy_from_slice(&pcm.to_be_bytes()); - } + // 1. Calculate RMS energy of current audio frame + let sum_squares: f32 = frame_buf.iter().map(|&s| s * s).sum(); + let rms = (sum_squares / PACKET_SAMPLES as f32).sqrt(); - let _ = input_socket.send(&net_packet); - sequence = sequence.wrapping_add(1); + // 2. Check threshold and manage hangover counter + let is_speaking = if rms >= VAD_THRESHOLD { + hangover_counter = VAD_HANGOVER_FRAMES; + true + } else if hangover_counter > 0 { + hangover_counter -= 1; + true + } else { + false + }; + + // 3. Only encode and transmit if VAD is active + if is_speaking { + net_packet[0..4].copy_from_slice(&sequence.to_be_bytes()); + for (i, sample) in frame_buf.iter().enumerate() { + let pcm = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; + let offset = HEADER_SIZE + i * 2; + net_packet[offset..offset + 2].copy_from_slice(&pcm.to_be_bytes()); + } + + let _ = input_socket.send(&net_packet); + sequence = sequence.wrapping_add(1); + } } else { thread::sleep(Duration::from_millis(2)); } diff --git a/src/components/page/channel/VoiceChannel.tsx b/src/components/page/channel/VoiceChannel.tsx index 86ec3a8..2d7a576 100644 --- a/src/components/page/channel/VoiceChannel.tsx +++ b/src/components/page/channel/VoiceChannel.tsx @@ -40,6 +40,9 @@ export default function VoiceChannel({

{channel.name}

+
+ {JSON.stringify(appRef.current?.server?.voiceChatSpeakers)} +
); } From ee71a99f5d8e6639ab5609fd93d6baf48572568b Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 17:53:40 +0200 Subject: [PATCH 27/29] Voice chat UI --- src/app/app.ts | 9 ++- src/app/server.ts | 8 -- src/components/page/channel/VoiceChannel.tsx | 78 +++++++++++++++++++- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/src/app/app.ts b/src/app/app.ts index de7e654..c261582 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -216,10 +216,17 @@ export default class Enclave

{ server.voiceChatUsers[msg.channel_id] = []; server.voiceChatUsers[msg.channel_id].push(msg.pubkey); + this.server?.getUsers([msg.pubkey]); return; case "Speaking": - server.voiceChatSpeaker(msg.pubkey); + clearTimeout(server.voiceChatSpeakers[msg.pubkey]); + + server.voiceChatSpeakers[msg.pubkey] = setTimeout(() => { + delete server.voiceChatSpeakers[msg.pubkey]; + this.forceRender(); + }, 800); + this.forceRender(); return; } } diff --git a/src/app/server.ts b/src/app/server.ts index 35efabf..19442f1 100644 --- a/src/app/server.ts +++ b/src/app/server.ts @@ -113,12 +113,4 @@ export default class EnclaveServer { this.websocket?.send({ method: "GetUsers", pubkeys: filteredPubkeys }); } - - public voiceChatSpeaker(pubkey: string) { - clearTimeout(this.voiceChatSpeakers[pubkey]); - - this.voiceChatSpeakers[pubkey] = setTimeout(() => { - delete this.voiceChatSpeakers[pubkey]; - }, 1000); - } } diff --git a/src/components/page/channel/VoiceChannel.tsx b/src/components/page/channel/VoiceChannel.tsx index 2d7a576..f29e13f 100644 --- a/src/components/page/channel/VoiceChannel.tsx +++ b/src/components/page/channel/VoiceChannel.tsx @@ -2,6 +2,8 @@ import Enclave from "@/app/app"; import { ChannelPageProps } from "../PageView"; import { useEffect, useRef } from "react"; import { invoke } from "@tauri-apps/api/core"; +import { cn } from "@/lib/utils"; +import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; export default function VoiceChannel({ appRef, @@ -35,13 +37,85 @@ export default function VoiceChannel({ }); }, [channel.id]); + const speakers = + appRef.current?.server?.voiceChatSpeakers && + Object.keys(appRef.current?.server?.voiceChatSpeakers); + + const users = appRef.current?.server?.voiceChatUsers[channel.id]; + + const usersLength = users?.length || 0; + + // Dynamic grid column layout based on user count + const getGridCols = () => { + if (usersLength <= 3) return "grid-cols-1"; + if (usersLength <= 6) return "grid-cols-3"; + if (usersLength <= 9) return "grid-cols-3"; + return "grid-cols-5"; + }; + + // Avatar sizing scales up for fewer participants + const getAvatarSize = () => { + if (usersLength <= 2) return "h-24 w-24 text-3xl"; + if (usersLength <= 4) return "h-20 w-20 text-2xl"; + return "h-14 w-14 text-lg"; + }; + return (

{channel.name}

-
- {JSON.stringify(appRef.current?.server?.voiceChatSpeakers)} +
+ {users?.map((pubkey) => { + const user = appRef.current?.server?.users[pubkey]; + const isSpeaking = speakers?.includes(pubkey); + const displayName = user?.displayName || "Unknown User"; + const fallbackLetter = displayName.charAt(0).toUpperCase(); + + return ( +
+ {/* Avatar Container */} +
+ + + + {fallbackLetter} + + + + {/* Mute indicator badge overlay */} + {/*{user?.isMuted && ( +
+ +
+ )}*/} +
+ +
+ {displayName} +
+
+ ); + })}
); From 6fb889840e35b5b57f3ad20697970769157adf9f Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 20:32:16 +0200 Subject: [PATCH 28/29] Proper vc leaving --- src-tauri/src/commands/audio.rs | 23 ++++- src/app/app.ts | 11 +++ src/app/protocol.ts | 8 ++ src/app/server.ts | 2 +- src/components/page/channel/VoiceChannel.tsx | 91 +++++++++++++++----- 5 files changed, 107 insertions(+), 28 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index ff2ffb3..a36c290 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -180,21 +180,22 @@ pub fn list_output_devices() -> Result, String> { } // ============================================================================ -// CONFIG UPDATES & DISCONNECT +// DISCONNECT // ============================================================================ #[tauri::command] pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { - if let Some(session) = voice_state.session.lock().unwrap().take() { - // Signal shutdown atomic to terminate sender and receiver loops + let mut lock = voice_state.session.lock().map_err(|e| e.to_string())?; + + if let Some(session) = lock.take() { session.shutdown.store(true, Ordering::SeqCst); - // Pause CPAL hardware streams explicitly to flush playback drivers immediately let _ = session.input_stream.pause(); if let Ok(output) = session.output_stream.lock() { let _ = output.pause(); } } + Ok(()) } @@ -491,3 +492,17 @@ fn build_output_stream( ) .map_err(|e| e.to_string()) } + +impl Drop for VoiceSession { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::SeqCst); + + let _ = self.input_stream.pause(); + if let Ok(output) = self.output_stream.lock() { + let _ = output.pause(); + } + + // Output hardware stream handle clean-up + eprintln!("[vc] VoiceSession dropped and audio streams paused."); + } +} diff --git a/src/app/app.ts b/src/app/app.ts index c261582..55efdec 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -219,6 +219,17 @@ export default class Enclave

{ this.server?.getUsers([msg.pubkey]); return; + case "UserLeftVoice": + if (!server.voiceChatUsers[msg.channel_id]) return; + + server.voiceChatUsers[msg.channel_id] = server.voiceChatUsers[ + msg.channel_id + ].filter((v) => v !== msg.pubkey); + + this.forceRender(); + + return; + case "Speaking": clearTimeout(server.voiceChatSpeakers[msg.pubkey]); diff --git a/src/app/protocol.ts b/src/app/protocol.ts index 93f5f49..a31272e 100644 --- a/src/app/protocol.ts +++ b/src/app/protocol.ts @@ -40,6 +40,11 @@ export type ClientMethod = channel_id: string; pubkey: string; } + | { + method: "UserLeftVoice"; + channel_id: string; + pubkey: string; + } | { method: "Speaking"; pubkey: string; @@ -89,4 +94,7 @@ export type ServerMethod = | { method: "JoinVoice"; channel_id: string; + } + | { + method: "LeaveVoice"; }; diff --git a/src/app/server.ts b/src/app/server.ts index 19442f1..a3e56d1 100644 --- a/src/app/server.ts +++ b/src/app/server.ts @@ -33,7 +33,7 @@ export default class EnclaveServer { public messages: Record>; public users: Record; - public voiceJoin?: (pin: number, channelId: string) => void; + public voiceJoin?: (pin: bigint, channelId: string) => void; public voiceChatUsers: Record; public voiceChatSpeakers: Record; diff --git a/src/components/page/channel/VoiceChannel.tsx b/src/components/page/channel/VoiceChannel.tsx index f29e13f..52a7ed0 100644 --- a/src/components/page/channel/VoiceChannel.tsx +++ b/src/components/page/channel/VoiceChannel.tsx @@ -1,9 +1,11 @@ import Enclave from "@/app/app"; import { ChannelPageProps } from "../PageView"; -import { useEffect, useRef } from "react"; +import { useEffect, useRef, useState } from "react"; import { invoke } from "@tauri-apps/api/core"; import { cn } from "@/lib/utils"; import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; +import { Button } from "@/components/ui/button"; +import { Phone, PhoneOff } from "lucide-react"; export default function VoiceChannel({ appRef, @@ -13,28 +15,49 @@ export default function VoiceChannel({ const channel = appRef.current?.page?.channel; if (!channel) return null; + const [isConnected, setIsConnected] = useState(false); const lastChannelId = useRef(null); - useEffect(() => { + // Leave current channel setup + const handleLeave = () => { + appRef.current?.server?.websocket?.send({ method: "LeaveVoice" }); + invoke("disconnect_from_vc").catch(console.error); + setIsConnected(false); + }; + + // Explicit action to join the voice channel + const handleJoin = () => { const hostname = appRef.current?.server?.hostname; - if (!hostname) return; - if (lastChannelId.current === channel.id) return; - lastChannelId.current = channel.id; - - invoke("disconnect_from_vc"); - - if (channel.kind !== "voice" || !appRef.current?.server) return; + if (!hostname || channel.kind !== "voice" || !appRef.current?.server) + return; appRef.current.server.voiceJoin = (pin, channelId) => { - invoke("connect_to_vc", { hostname, pin, channelId }).catch( - console.error, - ); + invoke("connect_to_vc", { hostname, pin, channelId }) + .then(() => setIsConnected(true)) + .catch(console.error); }; appRef.current.server.websocket?.send({ method: "JoinVoice", channel_id: channel.id, }); + + setIsConnected(true); + }; + + // Clean up when channel changes or component unmounts + useEffect(() => { + if (lastChannelId.current !== channel.id) { + if (isConnected) { + invoke("disconnect_from_vc").catch(console.error); + setIsConnected(false); + } + lastChannelId.current = channel.id; + } + + return () => { + invoke("disconnect_from_vc").catch(console.error); + }; }, [channel.id]); const speakers = @@ -42,7 +65,6 @@ export default function VoiceChannel({ Object.keys(appRef.current?.server?.voiceChatSpeakers); const users = appRef.current?.server?.voiceChatUsers[channel.id]; - const usersLength = users?.length || 0; // Dynamic grid column layout based on user count @@ -61,13 +83,19 @@ export default function VoiceChannel({ }; return ( -

-
+
+ {/* Header */} +

{channel.name}

+ + {usersLength} {usersLength === 1 ? "Participant" : "Participants"} +
+ + {/* Main Grid */}
@@ -101,15 +129,9 @@ export default function VoiceChannel({ {fallbackLetter} - - {/* Mute indicator badge overlay */} - {/*{user?.isMuted && ( -
- -
- )}*/}
+ {/* Name Plate */}
{displayName}
@@ -117,6 +139,29 @@ export default function VoiceChannel({ ); })}
+ + {/* Bottom Control Bar */} +
+ {isConnected ? ( + + ) : ( + + )} +
); } From 7436e62a49be71b86611980254018b95d4cd48a1 Mon Sep 17 00:00:00 2001 From: Klesti Selimaj Date: Thu, 27 Aug 2026 20:38:35 +0200 Subject: [PATCH 29/29] Self healing --- src-tauri/src/commands/audio.rs | 108 ++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 33 deletions(-) diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs index a36c290..b8a2ac3 100644 --- a/src-tauri/src/commands/audio.rs +++ b/src-tauri/src/commands/audio.rs @@ -26,10 +26,7 @@ const MAX_PACKET_SIZE: usize = 4096; const INITIAL_PACKET_CUSHION: usize = 3; // ~60ms cushion // --- Voice Activity Detection (VAD) Settings --- -/// RMS energy threshold to classify audio as speech (Range: 0.0 to 1.0). -/// 0.01 ≈ -40 dBFS. Adjust upward if room noise is triggering transmission. const VAD_THRESHOLD: f32 = 0.01; -/// Number of 20ms frames to keep transmitting after falling below threshold (~200ms hangover) const VAD_HANGOVER_FRAMES: usize = 10; // ============================================================================ @@ -39,7 +36,13 @@ const VAD_HANGOVER_FRAMES: usize = 10; type AudioProducer = Arc>>>>>; type AudioConsumer = Arc>>>>>; +#[derive(Clone, Default)] pub struct VoiceState { + pub inner: Arc, +} + +#[derive(Default)] +pub struct VoiceStateInner { pub session: Mutex>, } @@ -50,18 +53,19 @@ pub struct VoiceSession { pub pin: u64, pub shutdown: Arc, - // Tracked settings for diffing config changes pub current_input_device: Option, pub current_output_device: Option, - // Shared ringbuffer handles for runtime hot-swapping pub producer_in: AudioProducer, pub consumer_out: AudioConsumer, } impl VoiceSession { - /// Re-binds the input stream (microphone) without disconnecting the UDP thread - pub fn update_input_device(&mut self, device_name: Option) -> Result<(), String> { + pub fn update_input_device( + &mut self, + device_name: Option, + state_inner: Arc, + ) -> Result<(), String> { let host = cpal::default_host(); let device = match &device_name { Some(name) => host @@ -85,6 +89,8 @@ impl VoiceSession { .config(); let producer = Arc::clone(&self.producer_in); + let inner_clone = Arc::clone(&state_inner); + let new_stream = device .build_input_stream( input_config, @@ -93,7 +99,19 @@ impl VoiceSession { let _ = prod.push_slice(data); } }, - |err| eprintln!("[vc] input error: {err}"), + move |err| { + eprintln!("[vc] Input error: {err}. Attempting input stream recovery..."); + if let Ok(mut lock) = inner_clone.session.lock() { + if let Some(session) = lock.as_mut() { + let target_device = session.current_input_device.clone(); + if let Err(e) = + session.update_input_device(target_device, Arc::clone(&inner_clone)) + { + eprintln!("[vc] Input recovery failed: {e}"); + } + } + } + }, None, ) .map_err(|e| e.to_string())?; @@ -102,14 +120,16 @@ impl VoiceSession { .play() .map_err(|e| format!("Failed to play input stream: {e}"))?; - // Dropping old stream stops capturing audio hardware self.input_stream = new_stream; self.current_input_device = device_name; Ok(()) } - /// Re-binds the output stream (speakers/headphones) without interrupting UDP reception - pub fn update_output_device(&mut self, device_name: Option) -> Result<(), String> { + pub fn update_output_device( + &mut self, + device_name: Option, + state_inner: Arc, + ) -> Result<(), String> { let host = cpal::default_host(); let device = match &device_name { Some(name) => host @@ -133,7 +153,7 @@ impl VoiceSession { .config(); let consumer = Arc::clone(&self.consumer_out); - let new_stream = build_output_stream(&device, output_config, consumer)?; + let new_stream = build_output_stream(&device, output_config, consumer, state_inner)?; new_stream .play() .map_err(|e| format!("Failed to play output stream: {e}"))?; @@ -147,14 +167,6 @@ impl VoiceSession { } } -impl Default for VoiceState { - fn default() -> Self { - Self { - session: Mutex::new(None), - } - } -} - // ============================================================================ // DEVICES // ============================================================================ @@ -184,8 +196,12 @@ pub fn list_output_devices() -> Result, String> { // ============================================================================ #[tauri::command] -pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> { - let mut lock = voice_state.session.lock().map_err(|e| e.to_string())?; +pub fn disconnect_from_vc(voice_state: State<'_, VoiceState>) -> Result<(), String> { + let mut lock = voice_state + .inner + .session + .lock() + .map_err(|e| e.to_string())?; if let Some(session) = lock.take() { session.shutdown.store(true, Ordering::SeqCst); @@ -207,12 +223,13 @@ pub fn disconnect_from_vc(voice_state: State) -> Result<(), String> pub fn connect_to_vc( hostname: String, pin: u64, - config_state: State, - voice_state: State, + config_state: State<'_, ConfigState>, + voice_state: State<'_, VoiceState>, ) -> Result<(), String> { disconnect_from_vc(voice_state.clone())?; let config = config_state.0.lock().unwrap().clone(); + let state_inner = Arc::clone(&voice_state.inner); let socket = Arc::new( UdpSocket::bind("0.0.0.0:0").map_err(|e| format!("Failed to bind UDP socket: {e}"))?, @@ -266,7 +283,7 @@ pub fn connect_to_vc( // Output Ring Buffer setup let rb_out = HeapRb::::new(19200); let (producer_out, consumer_out) = rb_out.split(); - let mut producer_out = producer_out; // Handed to receiver thread + let mut producer_out = producer_out; let shared_consumer_out = Arc::new(Mutex::new(consumer_out)); let output_config = output_device @@ -278,6 +295,7 @@ pub fn connect_to_vc( &output_device, output_config, Arc::clone(&shared_consumer_out), + Arc::clone(&state_inner), )?; output_stream .play() @@ -295,6 +313,8 @@ pub fn connect_to_vc( .config(); let cb_producer = Arc::clone(&shared_producer_in); + let inner_input_err = Arc::clone(&state_inner); + let input_stream = input_device .build_input_stream( input_config, @@ -303,12 +323,24 @@ pub fn connect_to_vc( let _ = prod.push_slice(data); } }, - |err| eprintln!("[vc] input error: {err}"), + move |err| { + eprintln!("[vc] Input error: {err}. Attempting input stream recovery..."); + if let Ok(mut lock) = inner_input_err.session.lock() { + if let Some(session) = lock.as_mut() { + let target_device = session.current_input_device.clone(); + if let Err(e) = + session.update_input_device(target_device, Arc::clone(&inner_input_err)) + { + eprintln!("[vc] Input recovery failed: {e}"); + } + } + } + }, None, ) .map_err(|e| e.to_string())?; - // UDP Sender Thread (With Voice Activity Detection) + // UDP Sender Thread { let input_socket = socket.clone(); let shutdown = shutdown.clone(); @@ -323,11 +355,9 @@ pub fn connect_to_vc( if consumer_in.occupied_len() >= PACKET_SAMPLES { let _ = consumer_in.pop_slice(&mut frame_buf); - // 1. Calculate RMS energy of current audio frame let sum_squares: f32 = frame_buf.iter().map(|&s| s * s).sum(); let rms = (sum_squares / PACKET_SAMPLES as f32).sqrt(); - // 2. Check threshold and manage hangover counter let is_speaking = if rms >= VAD_THRESHOLD { hangover_counter = VAD_HANGOVER_FRAMES; true @@ -338,7 +368,6 @@ pub fn connect_to_vc( false }; - // 3. Only encode and transmit if VAD is active if is_speaking { net_packet[0..4].copy_from_slice(&sequence.to_be_bytes()); for (i, sample) in frame_buf.iter().enumerate() { @@ -433,7 +462,7 @@ pub fn connect_to_vc( .play() .map_err(|e| format!("Failed to start input: {e}"))?; - *voice_state.session.lock().unwrap() = Some(VoiceSession { + *state_inner.session.lock().unwrap() = Some(VoiceSession { input_stream, output_stream, socket, @@ -456,9 +485,11 @@ fn build_output_stream( device: &cpal::Device, config: cpal::StreamConfig, consumer: AudioConsumer, + state_inner: Arc, ) -> Result { let channels = config.channels as usize; let mut last_sample = 0.0f32; + let inner_output_err = Arc::clone(&state_inner); device .build_output_stream( @@ -487,7 +518,19 @@ fn build_output_stream( } } }, - |err| eprintln!("[vc] output error: {err}"), + move |err| { + eprintln!("[vc] Output error: {err}. Attempting output stream recovery..."); + if let Ok(mut lock) = inner_output_err.session.lock() { + if let Some(session) = lock.as_mut() { + let target_device = session.current_output_device.clone(); + if let Err(e) = session + .update_output_device(target_device, Arc::clone(&inner_output_err)) + { + eprintln!("[vc] Output recovery failed: {e}"); + } + } + } + }, None, ) .map_err(|e| e.to_string()) @@ -502,7 +545,6 @@ impl Drop for VoiceSession { let _ = output.pause(); } - // Output hardware stream handle clean-up eprintln!("[vc] VoiceSession dropped and audio streams paused."); } }