diff --git a/public/audio-worklet.js b/public/audio-worklet.js new file mode 100644 index 0000000..b3fed3a --- /dev/null +++ b/public/audio-worklet.js @@ -0,0 +1,35 @@ +class MicProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.buffer = new Float32Array(960); + this.offset = 0; + } + + process(inputs) { + const input = inputs[0]; + if (!input || !input[0]) return true; + + const channel = input[0]; + + let i = 0; + while (i < channel.length) { + const remaining = this.buffer.length - this.offset; + const toCopy = Math.min(remaining, channel.length - i); + + this.buffer.set(channel.subarray(i, i + toCopy), this.offset); + + this.offset += toCopy; + i += toCopy; + + if (this.offset === this.buffer.length) { + // COPY before sending + this.port.postMessage(this.buffer.slice()); + this.offset = 0; + } + } + + return true; + } +} + +registerProcessor("mic-processor", MicProcessor); diff --git a/src/app/auth/CreateAccount.tsx b/src/app/auth/CreateAccount.tsx index 6fa1286..8424e67 100644 --- a/src/app/auth/CreateAccount.tsx +++ b/src/app/auth/CreateAccount.tsx @@ -82,6 +82,7 @@ export default function CreateAccount() { Cookies.set("verify_temp_email", email); Cookies.set("verify_temp_password", password); router.push(`/verify`); + Cookies.set("servers", ""); } else { setFeedback({ message: response.message, kind: "error" }); } diff --git a/src/app/auth/Login.tsx b/src/app/auth/Login.tsx index d3fbc2f..6c1e61d 100644 --- a/src/app/auth/Login.tsx +++ b/src/app/auth/Login.tsx @@ -6,6 +6,7 @@ import { useState } from "react"; import { useRouter } from "next/navigation"; import { CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { loginUser } from "@/actions/user"; +import Cookies from "js-cookie"; type Feedback = { kind: "error" | "info"; message: string }; @@ -26,6 +27,7 @@ export default function Login() { message: "Login successful! Redirecting...", }); + Cookies.set("servers", ""); router.push("/chat"); } catch (err: any) { setFeedback({ diff --git a/src/app/chat/page.tsx b/src/app/chat/page.tsx index 38d0569..b9c640e 100644 --- a/src/app/chat/page.tsx +++ b/src/app/chat/page.tsx @@ -34,7 +34,7 @@ export default function ChatHub() { const activeServers: Server[] = Object.values(app.servers).slice(0, 3); const trendingServers: Server[] = [ { - id: "orus", + id: "7de3138f-e0b6-4d96-8158-8c7e866ba0ff", name: "ORUS", icon_url: "https://github.com/orus-dev.png", channels: [], diff --git a/src/app/server/[ip]/page.tsx b/src/app/server/[ip]/page.tsx index 562eea7..1d81717 100644 --- a/src/app/server/[ip]/page.tsx +++ b/src/app/server/[ip]/page.tsx @@ -28,6 +28,19 @@ export default function Server() { setServer: app.setServer, messageStore: messageStore, onIndicator: addIndicator, + onVoiceJoin: (userId, channelId, voiceId) => + app.setVoiceConns((prev) => ({ + ...prev, + [channelId]: { ...prev[channelId], [userId]: voiceId }, + })), + onVoiceLeave: (userId, channelId) => + app.setVoiceConns((prev) => { + const { [userId]: _, ...remainingUsers } = prev[channelId] || {}; + return { + ...prev, + [channelId]: remainingUsers, + }; + }), }); }, [ip, messageStore.messages], diff --git a/src/components/app/SidebarChannels.tsx b/src/components/app/SidebarChannels.tsx index 187e3ca..4c18047 100644 --- a/src/components/app/SidebarChannels.tsx +++ b/src/components/app/SidebarChannels.tsx @@ -1,6 +1,6 @@ "use client"; import { ScrollArea } from "../ui/scroll-area"; -import { Server, Channel } from "@/types/types"; +import { Channel } from "@/types/types"; import { Card } from "../ui/card"; import { HashIcon, Volume2Icon } from "lucide-react"; import App from "@/types/app"; @@ -38,7 +38,7 @@ export function ChannelItem({ channel, app }: { channel: Channel; app: App }) { export default function SidebarChannels({ app }: { app: App }) { return ( -
+

{app.server?.name}

diff --git a/src/components/channel/ChannelView.tsx b/src/components/channel/ChannelView.tsx index 8871f8f..0fb7d6b 100644 --- a/src/components/channel/ChannelView.tsx +++ b/src/components/channel/ChannelView.tsx @@ -4,6 +4,8 @@ import App from "@/types/app"; import React from "react"; import PluginChannel from "./PluginChannel"; import { IndicatorContext } from "@/types/protocol"; +import VoiceBox from "./voice/VoiceBox"; +import { sendVoice } from "@/lib/audio"; export default function ChannelView({ app, @@ -41,7 +43,26 @@ export default function ChannelView({ ); case "voice": - return null; + return ( + { + while (serverRef.current?.CONNECTING) {} + serverRef.current?.send(JSON.stringify(req)); + }} + sendVoice={(data) => { + while (!serverRef.current || serverRef.current?.CONNECTING) {} + sendVoice(serverRef.current, data); + }} + indicators={indicators.filter( + (indicator) => + indicator.indicator.params.user_id !== app.profile?.id + )} + /> + ); default: return ( diff --git a/src/components/channel/voice/VoiceBox.tsx b/src/components/channel/voice/VoiceBox.tsx new file mode 100644 index 0000000..388f661 --- /dev/null +++ b/src/components/channel/voice/VoiceBox.tsx @@ -0,0 +1,95 @@ +"use client"; + +import App from "@/types/app"; +import { ClientMessage, IndicatorContext } from "@/types/protocol"; +import { Message } from "@/types/types"; +import { useEffect, useState } from "react"; +import { startMic } from "@/lib/audio"; +import { Button } from "../../ui/button"; +import { ChevronLeftIcon, Phone, Volume2Icon } from "lucide-react"; +import VoiceGrid from "./VoiceGrid"; + +export default function VoiceBox({ + sendVoice, + app, + channelName, + sendRequest, + channelId, +}: { + channelName?: string; + app: App; + sendRequest: (req: ClientMessage) => void; + channelId: string; + messages: Message[]; + indicators: IndicatorContext[]; + sendVoice: (data: Int16Array) => void; +}) { + const [voiceConn, setVoiceConn] = useState(false); + + useEffect(() => { + if (!voiceConn) return; + + const stop = startMic(sendVoice); + + return () => { + stop.then((s) => s()); + }; + }, [voiceConn]); + + return ( +
+ {channelName && ( +
+ app.setSidebarOpen(true)} + className="cursor-pointer flex gap-2" + > + + + {channelName} + +
+ )} + {app.voiceConns[channelId] && ( + ({ + id: userId, + speaking: false, + }) + )} + app={app} + /> + )} +
+ {voiceConn ? ( + + ) : ( + + )} +
+
+ ); +} diff --git a/src/components/channel/voice/VoiceGrid.tsx b/src/components/channel/voice/VoiceGrid.tsx new file mode 100644 index 0000000..340e1ff --- /dev/null +++ b/src/components/channel/voice/VoiceGrid.tsx @@ -0,0 +1,28 @@ +import App from "@/types/app"; +import { VoiceTile, VoiceUser } from "./VoiceTile"; + +export default function VoiceGrid({ + app, + users, +}: { + app: App; + users: VoiceUser[]; +}) { + return ( +
+
+ {users.map((user) => ( + + ))} +
+
+ ); +} diff --git a/src/components/channel/voice/VoiceTile.tsx b/src/components/channel/voice/VoiceTile.tsx new file mode 100644 index 0000000..a914859 --- /dev/null +++ b/src/components/channel/voice/VoiceTile.tsx @@ -0,0 +1,50 @@ +import AsyncValue from "@/components/AsyncValue"; +import App from "@/types/app"; + +export type VoiceUser = { + id: string; + speaking: boolean; + muted?: boolean; +}; + +export function VoiceTile({ user, app }: { app: App; user: VoiceUser }) { + return ( +
+ {/* Avatar and name */} + app.getUserById(user.id)} + render={({ value }) => ( + <> + Pfp + +
+ {value?.display_name} +
+ + )} + /> + + {/* Speaking indicator */} + {user.speaking && ( +
Speaking
+ )} +
+ ); +} diff --git a/src/hooks/use-app.ts b/src/hooks/use-app.ts index 58a04d3..7f6f464 100644 --- a/src/hooks/use-app.ts +++ b/src/hooks/use-app.ts @@ -4,7 +4,6 @@ import useMessages from "./use-messages"; import { StringMap } from "@/types/typeUtils"; import useUser, { UserProfile } from "./get-user"; import { useClientSettings } from "./use-settings"; -import { get, Response } from "@/lib/request"; import { Channel, Server } from "@/types/types"; import Cookies from "js-cookie"; import { toast } from "sonner"; @@ -22,6 +21,9 @@ export default function useApp(): App { const [currentChannel, setCurrentChannel] = useState(null); const [server, setServer] = useState(); const privateMessages = useMessages(); + const [voiceConns, SetVoiceConns] = useState< + Record> + >({}); useEffect(() => { const stored = Cookies.get("servers"); @@ -38,6 +40,8 @@ export default function useApp(): App { }, []); return { + voiceConns, + setVoiceConns: SetVoiceConns, privateMessages, currentChannel, setCurrentChannel, diff --git a/src/lib/audio.ts b/src/lib/audio.ts new file mode 100644 index 0000000..1f62d94 --- /dev/null +++ b/src/lib/audio.ts @@ -0,0 +1,103 @@ +"use client"; + +async function getMicStream() { + return await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); +} + +export async function startMic(onAudio: (data: Int16Array) => void) { + const stream = await getMicStream(); + const sampleRate = 48000; + const audioCtx = new AudioContext({ sampleRate }); + await audioCtx.resume(); + await audioCtx.audioWorklet.addModule("/audio-worklet.js"); + + const source = audioCtx.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode(audioCtx, "mic-processor"); + + source.connect(worklet); + worklet.connect(audioCtx.destination); + + const buffer: number[] = []; + const chunkSize = sampleRate; + + worklet.port.onmessage = (e) => { + const float32 = e.data as Float32Array; + const pcm16 = float32ToPCM16(float32); + buffer.push(...pcm16); + }; + + const interval = setInterval(() => { + if (buffer.length === 0) return; + let chunk: Int16Array; + if (buffer.length >= chunkSize) { + chunk = new Int16Array(buffer.splice(0, chunkSize)); + } else { + chunk = new Int16Array(chunkSize); + chunk.set(buffer.splice(0, buffer.length)); + } + onAudio(chunk); + }, 1000); + + // Return cleanup function + return () => { + clearInterval(interval); + worklet.disconnect(); + source.disconnect(); + audioCtx.close(); + stream.getTracks().forEach((t) => t.stop()); + }; +} + +function float32ToPCM16(float32: Float32Array): Int16Array { + const pcm16 = new Int16Array(float32.length); + for (let i = 0; i < float32.length; i++) { + const s = Math.max(-1, Math.min(1, float32[i])); + pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + return pcm16; +} + +export function sendVoice(ws: WebSocket, pcm: Int16Array) { + ws.send(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength)); +} + +let ctx: AudioContext | null = null; + +function getAudioContext(sampleRate: number) { + if (!ctx) { + ctx = new AudioContext({ sampleRate }); + } + return ctx; +} + +export function playPCM16(u8: Uint8Array) { + const audioCtx = getAudioContext(48000); + + // Remove the first two bytes + const trimmed = u8.slice(2); + + const pcm16 = new Int16Array( + trimmed.buffer, + trimmed.byteOffset, + trimmed.byteLength / 2 + ); + + const float32 = new Float32Array(pcm16.length); + for (let i = 0; i < pcm16.length; i++) { + float32[i] = pcm16[i] / 0x8000; + } + + const buffer = audioCtx.createBuffer(1, float32.length, 48000); + buffer.getChannelData(0).set(float32); + + const src = audioCtx.createBufferSource(); + src.buffer = buffer; + src.connect(audioCtx.destination); + src.start(); +} diff --git a/src/lib/auth.ts b/src/lib/auth.ts index a887b92..d560ff9 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -5,6 +5,7 @@ import { MessageStore } from "@/hooks/use-messages"; import { IndicatorContext, ServerMessage } from "@/types/protocol"; import { authRelay } from "@/actions/auth"; import { getServerById } from "@/actions/get-server"; +import { playPCM16 } from "./audio"; type AuthParams = { id: string; @@ -13,6 +14,9 @@ type AuthParams = { onNewMessage?: (m: Message) => void; messageStore: MessageStore; onIndicator?: (i: IndicatorContext) => void; + onVoice?: (userId: string, bytes: number[]) => void; + onVoiceJoin?: (userId: string, channelId: string, voiceId: number) => void; + onVoiceLeave?: (userId: string, channelId: string, voiceId: number) => void; }; export default async function auth({ @@ -22,6 +26,8 @@ export default async function auth({ onNewMessage, messageStore, onIndicator, + onVoiceJoin, + onVoiceLeave, }: AuthParams) { console.log("Authenticating with server id:", id); @@ -42,7 +48,14 @@ export default async function auth({ console.log("Connected to WebSocket:", ip); }; - ws.onmessage = (m) => { + ws.onmessage = async (m) => { + if (m.data instanceof Blob) { + const buffer = await m.data.arrayBuffer(); + const bytes = new Uint8Array(buffer); + playPCM16(bytes); + return; + } + var data = JSON.parse(m.data); console.log("Message received:", data); @@ -92,6 +105,22 @@ export default async function auth({ if (!onIndicator) break; onIndicator(msg.params); break; + case "voice_join": + if (!onVoiceJoin) break; + onVoiceJoin( + msg.params.user_id, + msg.params.channel_id, + msg.params.voice_id + ); + break; + case "voice_leave": + if (!onVoiceLeave) break; + onVoiceLeave( + msg.params.user_id, + msg.params.channel_id, + msg.params.voice_id + ); + break; } }; diff --git a/src/types/app.ts b/src/types/app.ts index fd685bd..6fac9b5 100644 --- a/src/types/app.ts +++ b/src/types/app.ts @@ -35,4 +35,7 @@ export default interface App { currentChannel: Channel | null; setCurrentChannel: SetState; + + voiceConns: Record>; + setVoiceConns: SetState>>; } diff --git a/src/types/protocol.ts b/src/types/protocol.ts index 052186a..577d99c 100644 --- a/src/types/protocol.ts +++ b/src/types/protocol.ts @@ -20,7 +20,9 @@ export type ClientMessage = | EnumType<"delete_message", { message_id: number }> | EnumType<"edit_message", { message_id: number; new_contents: string }> | EnumType<"load_chunk", { channel_id: string; chunk_id: number }> - | EnumType<"typing", { channel_id: string }>; + | EnumType<"typing", { channel_id: string }> + | EnumType<"join_voice", { channel_id: string }> + | EnumType<"leave_voice", { channel_id: string }>; export type ServerMessage = | EnumType<"authenticated", { uuid: string; indicators: IndicatorContext[] }> @@ -31,4 +33,13 @@ export type ServerMessage = | EnumType<"presence_update", { user_id: String; status: String }> | EnumType<"indicator", IndicatorContext> | EnumType<"shutdown", { message: string }> + | EnumType<"chunk", Message[]> + | EnumType< + "voice_join", + { channel_id: string; user_id: string; voice_id: number } + > + | EnumType< + "voice_leave", + { channel_id: string; user_id: string; voice_id: number } + > | EnumType<"chunk", Message[]>;