Merge pull request #40 from AxiomChat/voice-chat

Voice chat

public/audio-worklet.js

@@ -0,0 +1,35 @@
+class MicProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.buffer = new Float32Array(960);
+    this.offset = 0;
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (!input || !input[0]) return true;
+
+    const channel = input[0];
+
+    let i = 0;
+    while (i < channel.length) {
+      const remaining = this.buffer.length - this.offset;
+      const toCopy = Math.min(remaining, channel.length - i);
+
+      this.buffer.set(channel.subarray(i, i + toCopy), this.offset);
+
+      this.offset += toCopy;
+      i += toCopy;
+
+      if (this.offset === this.buffer.length) {
+        // COPY before sending
+        this.port.postMessage(this.buffer.slice());
+        this.offset = 0;
+      }
+    }
+
+    return true;
+  }
+}
+
+registerProcessor("mic-processor", MicProcessor);

src/app/auth/CreateAccount.tsx

@@ -82,6 +82,7 @@ export default function CreateAccount() {
         Cookies.set("verify_temp_email", email);
         Cookies.set("verify_temp_password", password);
         router.push(`/verify`);
+        Cookies.set("servers", "");
       } else {
         setFeedback({ message: response.message, kind: "error" });
       }

src/app/auth/Login.tsx

@@ -6,6 +6,7 @@ import { useState } from "react";
 import { useRouter } from "next/navigation";
 import { CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import { loginUser } from "@/actions/user";
+import Cookies from "js-cookie";
 
 type Feedback = { kind: "error" | "info"; message: string };
 
@@ -26,6 +27,7 @@ export default function Login() {
         message: "Login successful! Redirecting...",
       });
 
+      Cookies.set("servers", "");
       router.push("/chat");
     } catch (err: any) {
       setFeedback({

src/app/chat/page.tsx

@@ -34,7 +34,7 @@ export default function ChatHub() {
   const activeServers: Server[] = Object.values(app.servers).slice(0, 3);
   const trendingServers: Server[] = [
     {
-      id: "orus",
+      id: "7de3138f-e0b6-4d96-8158-8c7e866ba0ff",
       name: "ORUS",
       icon_url: "https://github.com/orus-dev.png",
       channels: [],

src/app/server/[ip]/page.tsx

@@ -28,6 +28,19 @@ export default function Server() {
         setServer: app.setServer,
         messageStore: messageStore,
         onIndicator: addIndicator,
+        onVoiceJoin: (userId, channelId, voiceId) =>
+          app.setVoiceConns((prev) => ({
+            ...prev,
+            [channelId]: { ...prev[channelId], [userId]: voiceId },
+          })),
+        onVoiceLeave: (userId, channelId) =>
+          app.setVoiceConns((prev) => {
+            const { [userId]: _, ...remainingUsers } = prev[channelId] || {};
+            return {
+              ...prev,
+              [channelId]: remainingUsers,
+            };
+          }),
       });
     },
     [ip, messageStore.messages],

src/components/app/SidebarChannels.tsx

@@ -1,6 +1,6 @@
 "use client";
 import { ScrollArea } from "../ui/scroll-area";
-import { Server, Channel } from "@/types/types";
+import { Channel } from "@/types/types";
 import { Card } from "../ui/card";
 import { HashIcon, Volume2Icon } from "lucide-react";
 import App from "@/types/app";
@@ -38,7 +38,7 @@ export function ChannelItem({ channel, app }: { channel: Channel; app: App }) {
 
 export default function SidebarChannels({ app }: { app: App }) {
   return (
-    <div className="w-full flex flex-col bg-sidebar">
+    <div className="w-full h-full flex flex-col bg-sidebar">
       <div className="p-3 border-b flex items-center gap-2">
         <h2>{app.server?.name}</h2>
       </div>

src/components/channel/ChannelView.tsx

@@ -4,6 +4,8 @@ import App from "@/types/app";
 import React from "react";
 import PluginChannel from "./PluginChannel";
 import { IndicatorContext } from "@/types/protocol";
+import VoiceBox from "./voice/VoiceBox";
+import { sendVoice } from "@/lib/audio";
 
 export default function ChannelView({
   app,
@@ -41,7 +43,26 @@ export default function ChannelView({
       );
 
     case "voice":
-      return null;
+      return (
+        <VoiceBox
+          app={app}
+          channelName={app.currentChannel.name}
+          channelId={app.currentChannel.id}
+          messages={messages}
+          sendRequest={(req) => {
+            while (serverRef.current?.CONNECTING) {}
+            serverRef.current?.send(JSON.stringify(req));
+          }}
+          sendVoice={(data) => {
+            while (!serverRef.current || serverRef.current?.CONNECTING) {}
+            sendVoice(serverRef.current, data);
+          }}
+          indicators={indicators.filter(
+            (indicator) =>
+              indicator.indicator.params.user_id !== app.profile?.id
+          )}
+        />
+      );
 
     default:
       return (

src/components/channel/voice/VoiceBox.tsx

@@ -0,0 +1,95 @@
+"use client";
+
+import App from "@/types/app";
+import { ClientMessage, IndicatorContext } from "@/types/protocol";
+import { Message } from "@/types/types";
+import { useEffect, useState } from "react";
+import { startMic } from "@/lib/audio";
+import { Button } from "../../ui/button";
+import { ChevronLeftIcon, Phone, Volume2Icon } from "lucide-react";
+import VoiceGrid from "./VoiceGrid";
+
+export default function VoiceBox({
+  sendVoice,
+  app,
+  channelName,
+  sendRequest,
+  channelId,
+}: {
+  channelName?: string;
+  app: App;
+  sendRequest: (req: ClientMessage) => void;
+  channelId: string;
+  messages: Message[];
+  indicators: IndicatorContext[];
+  sendVoice: (data: Int16Array) => void;
+}) {
+  const [voiceConn, setVoiceConn] = useState(false);
+
+  useEffect(() => {
+    if (!voiceConn) return;
+
+    const stop = startMic(sendVoice);
+
+    return () => {
+      stop.then((s) => s());
+    };
+  }, [voiceConn]);
+
+  return (
+    <div className="h-svh w-full max-h-svh flex flex-col pb-5 pl-5 pr-5 gap-5">
+      {channelName && (
+        <header className="h-12 py-4 flex items-center border-b text-sm font-semibold w-full">
+          <span
+            onClick={() => app.setSidebarOpen(true)}
+            className="cursor-pointer flex gap-2"
+          >
+            <ChevronLeftIcon className="w-5 h-5 my-auto md:hidden" />
+            <Volume2Icon className="w-4 h-4 my-auto" />
+            {channelName}
+          </span>
+        </header>
+      )}
+      {app.voiceConns[channelId] && (
+        <VoiceGrid
+          users={Object.entries(app.voiceConns[channelId]).map(
+            ([userId, _]) => ({
+              id: userId,
+              speaking: false,
+            })
+          )}
+          app={app}
+        />
+      )}
+      <footer className="mt-auto h-max w-max mx-auto">
+        {voiceConn ? (
+          <Button
+            className="mx-auto my-auto h-max bg-transparent text-destructive border border-destructive hover:bg-destructive/20 rounded-full aspect-square w-12"
+            onClick={() => {
+              setVoiceConn(false);
+              sendRequest({
+                type: "leave_voice",
+                params: { channel_id: channelId },
+              });
+            }}
+          >
+            <Phone className="rotate-135" />
+          </Button>
+        ) : (
+          <Button
+            className="mx-auto my-auto h-max bg-transparent text-chart-2 border border-chart-2 hover:bg-chart-2/20 rounded-full aspect-square w-12"
+            onClick={() => {
+              setVoiceConn(true);
+              sendRequest({
+                type: "join_voice",
+                params: { channel_id: channelId },
+              });
+            }}
+          >
+            <Phone />
+          </Button>
+        )}
+      </footer>
+    </div>
+  );
+}

src/components/channel/voice/VoiceGrid.tsx

@@ -0,0 +1,28 @@
+import App from "@/types/app";
+import { VoiceTile, VoiceUser } from "./VoiceTile";
+
+export default function VoiceGrid({
+  app,
+  users,
+}: {
+  app: App;
+  users: VoiceUser[];
+}) {
+  return (
+    <div className="h-full overflow-y-auto p-4">
+      <div
+        className="
+          grid gap-4
+          grid-cols-1
+          sm:grid-cols-2
+          md:grid-cols-3
+          lg:grid-cols-4
+        "
+      >
+        {users.map((user) => (
+          <VoiceTile key={user.id} user={user} app={app} />
+        ))}
+      </div>
+    </div>
+  );
+}

src/components/channel/voice/VoiceTile.tsx

@@ -0,0 +1,50 @@
+import AsyncValue from "@/components/AsyncValue";
+import App from "@/types/app";
+
+export type VoiceUser = {
+  id: string;
+  speaking: boolean;
+  muted?: boolean;
+};
+
+export function VoiceTile({ user, app }: { app: App; user: VoiceUser }) {
+  return (
+    <div
+      className={`
+        relative flex flex-col items-center justify-center
+        aspect-square rounded-xl
+        bg-muted/40
+        transition-all
+        ${
+          user.speaking
+            ? "ring-4 ring-green-500 animate-pulse"
+            : "ring-1 ring-border"
+        }
+        ${user.muted ? "opacity-50" : ""}
+      `}
+    >
+      {/* Avatar and name */}
+      <AsyncValue
+        func={() => app.getUserById(user.id)}
+        render={({ value }) => (
+          <>
+            <img
+              src={value?.avatar_url}
+              alt="Pfp"
+              className="h-20 w-20 rounded-full"
+            />
+
+            <div className="mt-3 text-sm font-medium">
+              {value?.display_name}
+            </div>
+          </>
+        )}
+      />
+
+      {/* Speaking indicator */}
+      {user.speaking && (
+        <div className="absolute bottom-3 text-xs text-green-400">Speaking</div>
+      )}
+    </div>
+  );
+}

src/hooks/use-app.ts

@@ -4,7 +4,6 @@ import useMessages from "./use-messages";
 import { StringMap } from "@/types/typeUtils";
 import useUser, { UserProfile } from "./get-user";
 import { useClientSettings } from "./use-settings";
-import { get, Response } from "@/lib/request";
 import { Channel, Server } from "@/types/types";
 import Cookies from "js-cookie";
 import { toast } from "sonner";
@@ -22,6 +21,9 @@ export default function useApp(): App {
   const [currentChannel, setCurrentChannel] = useState<Channel | null>(null);
   const [server, setServer] = useState<Server>();
   const privateMessages = useMessages();
+  const [voiceConns, SetVoiceConns] = useState<
+    Record<string, Record<string, number>>
+  >({});
 
   useEffect(() => {
     const stored = Cookies.get("servers");
@@ -38,6 +40,8 @@ export default function useApp(): App {
   }, []);
 
   return {
+    voiceConns,
+    setVoiceConns: SetVoiceConns,
     privateMessages,
     currentChannel,
     setCurrentChannel,

src/lib/audio.ts

@@ -0,0 +1,103 @@
+"use client";
+
+async function getMicStream() {
+  return await navigator.mediaDevices.getUserMedia({
+    audio: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  });
+}
+
+export async function startMic(onAudio: (data: Int16Array) => void) {
+  const stream = await getMicStream();
+  const sampleRate = 48000;
+  const audioCtx = new AudioContext({ sampleRate });
+  await audioCtx.resume();
+  await audioCtx.audioWorklet.addModule("/audio-worklet.js");
+
+  const source = audioCtx.createMediaStreamSource(stream);
+  const worklet = new AudioWorkletNode(audioCtx, "mic-processor");
+
+  source.connect(worklet);
+  worklet.connect(audioCtx.destination);
+
+  const buffer: number[] = [];
+  const chunkSize = sampleRate;
+
+  worklet.port.onmessage = (e) => {
+    const float32 = e.data as Float32Array;
+    const pcm16 = float32ToPCM16(float32);
+    buffer.push(...pcm16);
+  };
+
+  const interval = setInterval(() => {
+    if (buffer.length === 0) return;
+    let chunk: Int16Array;
+    if (buffer.length >= chunkSize) {
+      chunk = new Int16Array(buffer.splice(0, chunkSize));
+    } else {
+      chunk = new Int16Array(chunkSize);
+      chunk.set(buffer.splice(0, buffer.length));
+    }
+    onAudio(chunk);
+  }, 1000);
+
+  // Return cleanup function
+  return () => {
+    clearInterval(interval);
+    worklet.disconnect();
+    source.disconnect();
+    audioCtx.close();
+    stream.getTracks().forEach((t) => t.stop());
+  };
+}
+
+function float32ToPCM16(float32: Float32Array): Int16Array {
+  const pcm16 = new Int16Array(float32.length);
+  for (let i = 0; i < float32.length; i++) {
+    const s = Math.max(-1, Math.min(1, float32[i]));
+    pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+  }
+  return pcm16;
+}
+
+export function sendVoice(ws: WebSocket, pcm: Int16Array) {
+  ws.send(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength));
+}
+
+let ctx: AudioContext | null = null;
+
+function getAudioContext(sampleRate: number) {
+  if (!ctx) {
+    ctx = new AudioContext({ sampleRate });
+  }
+  return ctx;
+}
+
+export function playPCM16(u8: Uint8Array) {
+  const audioCtx = getAudioContext(48000);
+
+  // Remove the first two bytes
+  const trimmed = u8.slice(2);
+
+  const pcm16 = new Int16Array(
+    trimmed.buffer,
+    trimmed.byteOffset,
+    trimmed.byteLength / 2
+  );
+
+  const float32 = new Float32Array(pcm16.length);
+  for (let i = 0; i < pcm16.length; i++) {
+    float32[i] = pcm16[i] / 0x8000;
+  }
+
+  const buffer = audioCtx.createBuffer(1, float32.length, 48000);
+  buffer.getChannelData(0).set(float32);
+
+  const src = audioCtx.createBufferSource();
+  src.buffer = buffer;
+  src.connect(audioCtx.destination);
+  src.start();
+}

src/lib/auth.ts

@@ -5,6 +5,7 @@ import { MessageStore } from "@/hooks/use-messages";
 import { IndicatorContext, ServerMessage } from "@/types/protocol";
 import { authRelay } from "@/actions/auth";
 import { getServerById } from "@/actions/get-server";
+import { playPCM16 } from "./audio";
 
 type AuthParams = {
   id: string;
@@ -13,6 +14,9 @@ type AuthParams = {
   onNewMessage?: (m: Message) => void;
   messageStore: MessageStore;
   onIndicator?: (i: IndicatorContext) => void;
+  onVoice?: (userId: string, bytes: number[]) => void;
+  onVoiceJoin?: (userId: string, channelId: string, voiceId: number) => void;
+  onVoiceLeave?: (userId: string, channelId: string, voiceId: number) => void;
 };
 
 export default async function auth({
@@ -22,6 +26,8 @@ export default async function auth({
   onNewMessage,
   messageStore,
   onIndicator,
+  onVoiceJoin,
+  onVoiceLeave,
 }: AuthParams) {
   console.log("Authenticating with server id:", id);
 
@@ -42,7 +48,14 @@ export default async function auth({
     console.log("Connected to WebSocket:", ip);
   };
 
-  ws.onmessage = (m) => {
+  ws.onmessage = async (m) => {
+    if (m.data instanceof Blob) {
+      const buffer = await m.data.arrayBuffer();
+      const bytes = new Uint8Array(buffer);
+      playPCM16(bytes);
+      return;
+    }
+
     var data = JSON.parse(m.data);
     console.log("Message received:", data);
 
@@ -92,6 +105,22 @@ export default async function auth({
         if (!onIndicator) break;
         onIndicator(msg.params);
         break;
+      case "voice_join":
+        if (!onVoiceJoin) break;
+        onVoiceJoin(
+          msg.params.user_id,
+          msg.params.channel_id,
+          msg.params.voice_id
+        );
+        break;
+      case "voice_leave":
+        if (!onVoiceLeave) break;
+        onVoiceLeave(
+          msg.params.user_id,
+          msg.params.channel_id,
+          msg.params.voice_id
+        );
+        break;
     }
   };
 

src/types/app.ts

@@ -35,4 +35,7 @@ export default interface App {
 
   currentChannel: Channel | null;
   setCurrentChannel: SetState<Channel | null>;
+
+  voiceConns: Record<string, Record<string, number>>;
+  setVoiceConns: SetState<Record<string, Record<string, number>>>;
 }

src/types/protocol.ts

@@ -20,7 +20,9 @@ export type ClientMessage =
   | EnumType<"delete_message", { message_id: number }>
   | EnumType<"edit_message", { message_id: number; new_contents: string }>
   | EnumType<"load_chunk", { channel_id: string; chunk_id: number }>
-  | EnumType<"typing", { channel_id: string }>;
+  | EnumType<"typing", { channel_id: string }>
+  | EnumType<"join_voice", { channel_id: string }>
+  | EnumType<"leave_voice", { channel_id: string }>;
 
 export type ServerMessage =
   | EnumType<"authenticated", { uuid: string; indicators: IndicatorContext[] }>
@@ -31,4 +33,13 @@ export type ServerMessage =
   | EnumType<"presence_update", { user_id: String; status: String }>
   | EnumType<"indicator", IndicatorContext>
   | EnumType<"shutdown", { message: string }>
+  | EnumType<"chunk", Message[]>
+  | EnumType<
+      "voice_join",
+      { channel_id: string; user_id: string; voice_id: number }
+    >
+  | EnumType<
+      "voice_leave",
+      { channel_id: string; user_id: string; voice_id: number }
+    >
   | EnumType<"chunk", Message[]>;