From 07dbac50c884c43497c68ecdacabf29f6e6cdec6 Mon Sep 17 00:00:00 2001 From: Leo dev Date: Mon, 5 Jan 2026 21:43:05 +0100 Subject: [PATCH 1/9] Voice testing --- public/audio-worklet.js | 12 +++++++ src/components/channel/ChannelView.tsx | 22 +++++++++++- src/components/channel/VoiceBox.tsx | 27 +++++++++++++++ src/lib/audio.ts | 47 ++++++++++++++++++++++++++ src/lib/auth.ts | 1 + src/types/protocol.ts | 13 ++++++- 6 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 public/audio-worklet.js create mode 100644 src/components/channel/VoiceBox.tsx create mode 100644 src/lib/audio.ts diff --git a/public/audio-worklet.js b/public/audio-worklet.js new file mode 100644 index 0000000..ebd7adf --- /dev/null +++ b/public/audio-worklet.js @@ -0,0 +1,12 @@ +class MicProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input.length > 0) { + const channelData = input[0]; // Float32Array + this.port.postMessage(channelData); + } + return true; + } +} + +registerProcessor("mic-processor", MicProcessor); diff --git a/src/components/channel/ChannelView.tsx b/src/components/channel/ChannelView.tsx index 8871f8f..c5d0a21 100644 --- a/src/components/channel/ChannelView.tsx +++ b/src/components/channel/ChannelView.tsx @@ -4,6 +4,8 @@ import App from "@/types/app"; import React from "react"; import PluginChannel from "./PluginChannel"; import { IndicatorContext } from "@/types/protocol"; +import VoiceBox from "./VoiceBox"; +import { sendVoice } from "@/lib/audio"; export default function ChannelView({ app, @@ -41,7 +43,25 @@ export default function ChannelView({ ); case "voice": - return null; + return ( + { + while (serverRef.current?.CONNECTING) {} + serverRef.current?.send(JSON.stringify(req)); + }} + sendVoice={(data) => { + while (!serverRef.current || serverRef.current?.CONNECTING) {} + sendVoice(serverRef.current, 0, data); + }} + indicators={indicators.filter( + (indicator) => + indicator.indicator.params.user_id !== app.profile?.id + )} + /> + ); default: return ( diff --git a/src/components/channel/VoiceBox.tsx b/src/components/channel/VoiceBox.tsx new file mode 100644 index 0000000..782dd0c --- /dev/null +++ b/src/components/channel/VoiceBox.tsx @@ -0,0 +1,27 @@ +"use client"; + +import App from "@/types/app"; +import { ClientMessage, IndicatorContext } from "@/types/protocol"; +import { Message } from "@/types/types"; +import { useEffect } from "react"; +import { startMic } from "@/lib/audio"; + +export default function VoiceBox({ + sendVoice, +}: { + app: App; + sendRequest: (req: ClientMessage) => void; + channelName?: string; + channelIcon?: string; + messages: Message[]; + indicators: IndicatorContext[]; + sendVoice: (data: Int16Array) => void; +}) { + useEffect(() => { + startMic(sendVoice); + }, []); + + return ( +
+ ); +} diff --git a/src/lib/audio.ts b/src/lib/audio.ts new file mode 100644 index 0000000..760433e --- /dev/null +++ b/src/lib/audio.ts @@ -0,0 +1,47 @@ +async function getMicStream() { + return await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); +} + +export async function startMic(onAudio: (data: Int16Array) => void) { + const stream = await getMicStream(); + + const audioCtx = new AudioContext({ sampleRate: 48000 }); + await audioCtx.audioWorklet.addModule("/audio-worklet.js"); + + const source = audioCtx.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode(audioCtx, "mic-processor"); + + source.connect(worklet); + worklet.connect(audioCtx.destination); // optional + + worklet.port.onmessage = (e) => { + const float32 = e.data as Float32Array; + const pcm16 = float32ToPCM16(float32); + onAudio(pcm16); + }; +} + +function float32ToPCM16(float32: Float32Array): Int16Array { + const pcm16 = new Int16Array(float32.length); + for (let i = 0; i < float32.length; i++) { + const s = Math.max(-1, Math.min(1, float32[i])); + pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + return pcm16; +} + +export function sendVoice(ws: WebSocket, voiceId: number, pcm: Int16Array) { + const buffer = new ArrayBuffer(2 + pcm.byteLength); + const view = new DataView(buffer); + + view.setUint16(0, voiceId, true); // little endian + new Uint8Array(buffer, 2).set(new Uint8Array(pcm.buffer)); + + ws.send(buffer); +} diff --git a/src/lib/auth.ts b/src/lib/auth.ts index a887b92..ed9f5fc 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -13,6 +13,7 @@ type AuthParams = { onNewMessage?: (m: Message) => void; messageStore: MessageStore; onIndicator?: (i: IndicatorContext) => void; + onVoice?: (userId: string, bytes: number[]) => void; }; export default async function auth({ diff --git a/src/types/protocol.ts b/src/types/protocol.ts index 052186a..577d99c 100644 --- a/src/types/protocol.ts +++ b/src/types/protocol.ts @@ -20,7 +20,9 @@ export type ClientMessage = | EnumType<"delete_message", { message_id: number }> | EnumType<"edit_message", { message_id: number; new_contents: string }> | EnumType<"load_chunk", { channel_id: string; chunk_id: number }> - | EnumType<"typing", { channel_id: string }>; + | EnumType<"typing", { channel_id: string }> + | EnumType<"join_voice", { channel_id: string }> + | EnumType<"leave_voice", { channel_id: string }>; export type ServerMessage = | EnumType<"authenticated", { uuid: string; indicators: IndicatorContext[] }> @@ -31,4 +33,13 @@ export type ServerMessage = | EnumType<"presence_update", { user_id: String; status: String }> | EnumType<"indicator", IndicatorContext> | EnumType<"shutdown", { message: string }> + | EnumType<"chunk", Message[]> + | EnumType< + "voice_join", + { channel_id: string; user_id: string; voice_id: number } + > + | EnumType< + "voice_leave", + { channel_id: string; user_id: string; voice_id: number } + > | EnumType<"chunk", Message[]>; From 1af261058f6c18cb5584fb549c1f4eaf72fbae98 Mon Sep 17 00:00:00 2001 From: Leo dev Date: Mon, 5 Jan 2026 22:34:25 +0100 Subject: [PATCH 2/9] Voice testing --- public/audio-worklet.js | 29 ++++++++++-- src/components/channel/VoiceBox.tsx | 8 +++- src/lib/audio.ts | 69 +++++++++++++++++++++++++++-- src/lib/auth.ts | 10 ++++- 4 files changed, 106 insertions(+), 10 deletions(-) diff --git a/public/audio-worklet.js b/public/audio-worklet.js index ebd7adf..b3fed3a 100644 --- a/public/audio-worklet.js +++ b/public/audio-worklet.js @@ -1,10 +1,33 @@ class MicProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.buffer = new Float32Array(960); + this.offset = 0; + } + process(inputs) { const input = inputs[0]; - if (input.length > 0) { - const channelData = input[0]; // Float32Array - this.port.postMessage(channelData); + if (!input || !input[0]) return true; + + const channel = input[0]; + + let i = 0; + while (i < channel.length) { + const remaining = this.buffer.length - this.offset; + const toCopy = Math.min(remaining, channel.length - i); + + this.buffer.set(channel.subarray(i, i + toCopy), this.offset); + + this.offset += toCopy; + i += toCopy; + + if (this.offset === this.buffer.length) { + // COPY before sending + this.port.postMessage(this.buffer.slice()); + this.offset = 0; + } } + return true; } } diff --git a/src/components/channel/VoiceBox.tsx b/src/components/channel/VoiceBox.tsx index 782dd0c..263437d 100644 --- a/src/components/channel/VoiceBox.tsx +++ b/src/components/channel/VoiceBox.tsx @@ -3,7 +3,7 @@ import App from "@/types/app"; import { ClientMessage, IndicatorContext } from "@/types/protocol"; import { Message } from "@/types/types"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; import { startMic } from "@/lib/audio"; export default function VoiceBox({ @@ -18,7 +18,11 @@ export default function VoiceBox({ sendVoice: (data: Int16Array) => void; }) { useEffect(() => { - startMic(sendVoice); + const stop = startMic(sendVoice); + + return () => { + stop.then((s) => s()); + }; }, []); return ( diff --git a/src/lib/audio.ts b/src/lib/audio.ts index 760433e..d71e39e 100644 --- a/src/lib/audio.ts +++ b/src/lib/audio.ts @@ -1,3 +1,5 @@ +"use client"; + async function getMicStream() { return await navigator.mediaDevices.getUserMedia({ audio: { @@ -10,20 +12,44 @@ async function getMicStream() { export async function startMic(onAudio: (data: Int16Array) => void) { const stream = await getMicStream(); - - const audioCtx = new AudioContext({ sampleRate: 48000 }); + const sampleRate = 48000; + const audioCtx = new AudioContext({ sampleRate }); await audioCtx.audioWorklet.addModule("/audio-worklet.js"); const source = audioCtx.createMediaStreamSource(stream); const worklet = new AudioWorkletNode(audioCtx, "mic-processor"); source.connect(worklet); - worklet.connect(audioCtx.destination); // optional + worklet.connect(audioCtx.destination); + + const buffer: number[] = []; + const chunkSize = sampleRate; worklet.port.onmessage = (e) => { const float32 = e.data as Float32Array; const pcm16 = float32ToPCM16(float32); - onAudio(pcm16); + buffer.push(...pcm16); + }; + + const interval = setInterval(() => { + if (buffer.length === 0) return; + let chunk: Int16Array; + if (buffer.length >= chunkSize) { + chunk = new Int16Array(buffer.splice(0, chunkSize)); + } else { + chunk = new Int16Array(chunkSize); + chunk.set(buffer.splice(0, buffer.length)); + } + onAudio(chunk); + }, 1000); + + // Return cleanup function + return () => { + clearInterval(interval); + worklet.disconnect(); + source.disconnect(); + audioCtx.close(); + stream.getTracks().forEach((t) => t.stop()); }; } @@ -45,3 +71,38 @@ export function sendVoice(ws: WebSocket, voiceId: number, pcm: Int16Array) { ws.send(buffer); } + +let ctx: AudioContext | null = null; + +function getAudioContext(sampleRate: number) { + if (!ctx) { + ctx = new AudioContext({ sampleRate }); + } + return ctx; +} + +export function playPCM16(u8: Uint8Array) { + const audioCtx = getAudioContext(48000); + + // Remove the first two bytes + const trimmed = u8.slice(2); + + const pcm16 = new Int16Array( + trimmed.buffer, + trimmed.byteOffset, + trimmed.byteLength / 2 + ); + + const float32 = new Float32Array(pcm16.length); + for (let i = 0; i < pcm16.length; i++) { + float32[i] = pcm16[i] / 0x8000; + } + + const buffer = audioCtx.createBuffer(1, float32.length, 48000); + buffer.getChannelData(0).set(float32); + + const src = audioCtx.createBufferSource(); + src.buffer = buffer; + src.connect(audioCtx.destination); + src.start(); +} diff --git a/src/lib/auth.ts b/src/lib/auth.ts index ed9f5fc..9f4464b 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -5,6 +5,7 @@ import { MessageStore } from "@/hooks/use-messages"; import { IndicatorContext, ServerMessage } from "@/types/protocol"; import { authRelay } from "@/actions/auth"; import { getServerById } from "@/actions/get-server"; +import { playPCM16 } from "./audio"; type AuthParams = { id: string; @@ -43,7 +44,14 @@ export default async function auth({ console.log("Connected to WebSocket:", ip); }; - ws.onmessage = (m) => { + ws.onmessage = async (m) => { + if (m.data instanceof Blob) { + const buffer = await m.data.arrayBuffer(); + const bytes = new Uint8Array(buffer); + playPCM16(bytes) + return; + } + var data = JSON.parse(m.data); console.log("Message received:", data); From abbd461e3f134fd79687b7f011972daa7e592238 Mon Sep 17 00:00:00 2001 From: Leo dev Date: Mon, 5 Jan 2026 22:45:43 +0100 Subject: [PATCH 3/9] Vc footer --- src/components/app/SidebarChannels.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/components/app/SidebarChannels.tsx b/src/components/app/SidebarChannels.tsx index 187e3ca..28bafba 100644 --- a/src/components/app/SidebarChannels.tsx +++ b/src/components/app/SidebarChannels.tsx @@ -2,8 +2,9 @@ import { ScrollArea } from "../ui/scroll-area"; import { Server, Channel } from "@/types/types"; import { Card } from "../ui/card"; -import { HashIcon, Volume2Icon } from "lucide-react"; +import { HashIcon, PhoneOff, Volume2Icon } from "lucide-react"; import App from "@/types/app"; +import { Button } from "../ui/button"; export function ChannelIcon({ kind }: { kind: "text" | "voice" }) { switch (kind) { @@ -38,7 +39,7 @@ export function ChannelItem({ channel, app }: { channel: Channel; app: App }) { export default function SidebarChannels({ app }: { app: App }) { return ( -
+

{app.server?.name}

@@ -49,6 +50,15 @@ export default function SidebarChannels({ app }: { app: App }) { ))}
+ +
+ +
); } From f6ce39a40b912ac4281cc20ae7e6f32b5300a940 Mon Sep 17 00:00:00 2001 From: Leo dev Date: Tue, 6 Jan 2026 17:52:54 +0100 Subject: [PATCH 4/9] Small fixes --- src/app/auth/CreateAccount.tsx | 1 + src/app/auth/Login.tsx | 2 ++ src/app/chat/page.tsx | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/app/auth/CreateAccount.tsx b/src/app/auth/CreateAccount.tsx index 6fa1286..8424e67 100644 --- a/src/app/auth/CreateAccount.tsx +++ b/src/app/auth/CreateAccount.tsx @@ -82,6 +82,7 @@ export default function CreateAccount() { Cookies.set("verify_temp_email", email); Cookies.set("verify_temp_password", password); router.push(`/verify`); + Cookies.set("servers", ""); } else { setFeedback({ message: response.message, kind: "error" }); } diff --git a/src/app/auth/Login.tsx b/src/app/auth/Login.tsx index d3fbc2f..6c1e61d 100644 --- a/src/app/auth/Login.tsx +++ b/src/app/auth/Login.tsx @@ -6,6 +6,7 @@ import { useState } from "react"; import { useRouter } from "next/navigation"; import { CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { loginUser } from "@/actions/user"; +import Cookies from "js-cookie"; type Feedback = { kind: "error" | "info"; message: string }; @@ -26,6 +27,7 @@ export default function Login() { message: "Login successful! Redirecting...", }); + Cookies.set("servers", ""); router.push("/chat"); } catch (err: any) { setFeedback({ diff --git a/src/app/chat/page.tsx b/src/app/chat/page.tsx index 38d0569..b9c640e 100644 --- a/src/app/chat/page.tsx +++ b/src/app/chat/page.tsx @@ -34,7 +34,7 @@ export default function ChatHub() { const activeServers: Server[] = Object.values(app.servers).slice(0, 3); const trendingServers: Server[] = [ { - id: "orus", + id: "7de3138f-e0b6-4d96-8158-8c7e866ba0ff", name: "ORUS", icon_url: "https://github.com/orus-dev.png", channels: [], From acb62df85962bbad65de1864e8aab38b574066c8 Mon Sep 17 00:00:00 2001 From: Leo dev Date: Tue, 6 Jan 2026 18:37:51 +0100 Subject: [PATCH 5/9] Improving the structure --- src/components/app/SidebarChannels.tsx | 14 +------ src/components/channel/ChannelView.tsx | 2 +- src/components/channel/VoiceBox.tsx | 51 ++++++++++++++++++++++++-- src/lib/audio.ts | 10 +---- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/src/components/app/SidebarChannels.tsx b/src/components/app/SidebarChannels.tsx index 28bafba..4c18047 100644 --- a/src/components/app/SidebarChannels.tsx +++ b/src/components/app/SidebarChannels.tsx @@ -1,10 +1,9 @@ "use client"; import { ScrollArea } from "../ui/scroll-area"; -import { Server, Channel } from "@/types/types"; +import { Channel } from "@/types/types"; import { Card } from "../ui/card"; -import { HashIcon, PhoneOff, Volume2Icon } from "lucide-react"; +import { HashIcon, Volume2Icon } from "lucide-react"; import App from "@/types/app"; -import { Button } from "../ui/button"; export function ChannelIcon({ kind }: { kind: "text" | "voice" }) { switch (kind) { @@ -50,15 +49,6 @@ export default function SidebarChannels({ app }: { app: App }) { ))}
- -
- -
); } diff --git a/src/components/channel/ChannelView.tsx b/src/components/channel/ChannelView.tsx index c5d0a21..775b3db 100644 --- a/src/components/channel/ChannelView.tsx +++ b/src/components/channel/ChannelView.tsx @@ -54,7 +54,7 @@ export default function ChannelView({ }} sendVoice={(data) => { while (!serverRef.current || serverRef.current?.CONNECTING) {} - sendVoice(serverRef.current, 0, data); + sendVoice(serverRef.current, data); }} indicators={indicators.filter( (indicator) => diff --git a/src/components/channel/VoiceBox.tsx b/src/components/channel/VoiceBox.tsx index 263437d..407272c 100644 --- a/src/components/channel/VoiceBox.tsx +++ b/src/components/channel/VoiceBox.tsx @@ -5,27 +5,72 @@ import { ClientMessage, IndicatorContext } from "@/types/protocol"; import { Message } from "@/types/types"; import { useEffect, useState } from "react"; import { startMic } from "@/lib/audio"; +import { Button } from "../ui/button"; +import { + ChevronLeftIcon, + PhoneCall, + PhoneOff, + Volume2Icon, +} from "lucide-react"; export default function VoiceBox({ sendVoice, + app, + channelName, }: { app: App; sendRequest: (req: ClientMessage) => void; channelName?: string; - channelIcon?: string; messages: Message[]; indicators: IndicatorContext[]; sendVoice: (data: Int16Array) => void; }) { + const [voiceConn, setVoiceConn] = useState(false); + useEffect(() => { + if (!voiceConn) return; + const stop = startMic(sendVoice); return () => { stop.then((s) => s()); }; - }, []); + }, [voiceConn]); return ( -
+
+ {channelName && ( +
+ app.setSidebarOpen(true)} + className="cursor-pointer flex gap-2" + > + + + {channelName} + +
+ )} + {voiceConn ? ( +
+
+ +
+
+ ) : ( + + )} +
); } diff --git a/src/lib/audio.ts b/src/lib/audio.ts index d71e39e..0823394 100644 --- a/src/lib/audio.ts +++ b/src/lib/audio.ts @@ -62,14 +62,8 @@ function float32ToPCM16(float32: Float32Array): Int16Array { return pcm16; } -export function sendVoice(ws: WebSocket, voiceId: number, pcm: Int16Array) { - const buffer = new ArrayBuffer(2 + pcm.byteLength); - const view = new DataView(buffer); - - view.setUint16(0, voiceId, true); // little endian - new Uint8Array(buffer, 2).set(new Uint8Array(pcm.buffer)); - - ws.send(buffer); +export function sendVoice(ws: WebSocket, pcm: Int16Array) { + ws.send(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength)); } let ctx: AudioContext | null = null; From 16bcf25391a178f9d272f2200c1ca99ed3cd0628 Mon Sep 17 00:00:00 2001 From: Leo dev Date: Tue, 6 Jan 2026 19:47:19 +0100 Subject: [PATCH 6/9] Both ui and join leave improvements --- src/components/channel/VoiceBox.tsx | 56 ++++++++++++++++++----------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/src/components/channel/VoiceBox.tsx b/src/components/channel/VoiceBox.tsx index 407272c..284fa46 100644 --- a/src/components/channel/VoiceBox.tsx +++ b/src/components/channel/VoiceBox.tsx @@ -8,6 +8,7 @@ import { startMic } from "@/lib/audio"; import { Button } from "../ui/button"; import { ChevronLeftIcon, + Phone, PhoneCall, PhoneOff, Volume2Icon, @@ -17,6 +18,7 @@ export default function VoiceBox({ sendVoice, app, channelName, + sendRequest, }: { app: App; sendRequest: (req: ClientMessage) => void; @@ -51,26 +53,40 @@ export default function VoiceBox({ )} - {voiceConn ? ( -
-
- -
-
- ) : ( - - )} +
+
+ {voiceConn ? ( + + ) : ( + + )} +
); } From e40884ba7fad18c374b50cb540925d1d4106a37c Mon Sep 17 00:00:00 2001 From: Leo dev Date: Tue, 6 Jan 2026 19:56:49 +0100 Subject: [PATCH 7/9] Testing speaking UI --- src/components/channel/ChannelView.tsx | 2 +- .../channel/{ => voice}/VoiceBox.tsx | 12 ++++- src/components/channel/voice/VoiceGrid.tsx | 21 +++++++++ src/components/channel/voice/VoiceTile.tsx | 45 +++++++++++++++++++ 4 files changed, 77 insertions(+), 3 deletions(-) rename src/components/channel/{ => voice}/VoiceBox.tsx (87%) create mode 100644 src/components/channel/voice/VoiceGrid.tsx create mode 100644 src/components/channel/voice/VoiceTile.tsx diff --git a/src/components/channel/ChannelView.tsx b/src/components/channel/ChannelView.tsx index 775b3db..8591a69 100644 --- a/src/components/channel/ChannelView.tsx +++ b/src/components/channel/ChannelView.tsx @@ -4,7 +4,7 @@ import App from "@/types/app"; import React from "react"; import PluginChannel from "./PluginChannel"; import { IndicatorContext } from "@/types/protocol"; -import VoiceBox from "./VoiceBox"; +import VoiceBox from "./voice/VoiceBox"; import { sendVoice } from "@/lib/audio"; export default function ChannelView({ diff --git a/src/components/channel/VoiceBox.tsx b/src/components/channel/voice/VoiceBox.tsx similarity index 87% rename from src/components/channel/VoiceBox.tsx rename to src/components/channel/voice/VoiceBox.tsx index 284fa46..da34051 100644 --- a/src/components/channel/VoiceBox.tsx +++ b/src/components/channel/voice/VoiceBox.tsx @@ -5,7 +5,7 @@ import { ClientMessage, IndicatorContext } from "@/types/protocol"; import { Message } from "@/types/types"; import { useEffect, useState } from "react"; import { startMic } from "@/lib/audio"; -import { Button } from "../ui/button"; +import { Button } from "../../ui/button"; import { ChevronLeftIcon, Phone, @@ -13,6 +13,7 @@ import { PhoneOff, Volume2Icon, } from "lucide-react"; +import VoiceGrid from "./VoiceGrid"; export default function VoiceBox({ sendVoice, @@ -53,7 +54,14 @@ export default function VoiceBox({ )} -
+