Voice testing

public/audio-worklet.js

@@ -0,0 +1,12 @@
+class MicProcessor extends AudioWorkletProcessor {
+  process(inputs) {
+    const input = inputs[0];
+    if (input.length > 0) {
+      const channelData = input[0]; // Float32Array
+      this.port.postMessage(channelData);
+    }
+    return true;
+  }
+}
+
+registerProcessor("mic-processor", MicProcessor);

src/components/channel/ChannelView.tsx

@@ -4,6 +4,8 @@ import App from "@/types/app";
 import React from "react";
 import PluginChannel from "./PluginChannel";
 import { IndicatorContext } from "@/types/protocol";
+import VoiceBox from "./VoiceBox";
+import { sendVoice } from "@/lib/audio";
 
 export default function ChannelView({
   app,
@@ -41,7 +43,25 @@ export default function ChannelView({
       );
 
     case "voice":
-      return null;
+      return (
+        <VoiceBox
+          app={app}
+          channelName={app.currentChannel.name}
+          messages={messages}
+          sendRequest={(req) => {
+            while (serverRef.current?.CONNECTING) {}
+            serverRef.current?.send(JSON.stringify(req));
+          }}
+          sendVoice={(data) => {
+            while (!serverRef.current || serverRef.current?.CONNECTING) {}
+            sendVoice(serverRef.current, 0, data);
+          }}
+          indicators={indicators.filter(
+            (indicator) =>
+              indicator.indicator.params.user_id !== app.profile?.id
+          )}
+        />
+      );
 
     default:
       return (

src/components/channel/VoiceBox.tsx

@@ -0,0 +1,27 @@
+"use client";
+
+import App from "@/types/app";
+import { ClientMessage, IndicatorContext } from "@/types/protocol";
+import { Message } from "@/types/types";
+import { useEffect } from "react";
+import { startMic } from "@/lib/audio";
+
+export default function VoiceBox({
+  sendVoice,
+}: {
+  app: App;
+  sendRequest: (req: ClientMessage) => void;
+  channelName?: string;
+  channelIcon?: string;
+  messages: Message[];
+  indicators: IndicatorContext[];
+  sendVoice: (data: Int16Array) => void;
+}) {
+  useEffect(() => {
+    startMic(sendVoice);
+  }, []);
+
+  return (
+    <div className="h-svh w-full max-h-svh flex flex-col pb-5 pl-5 gap-5" />
+  );
+}

src/lib/audio.ts

@@ -0,0 +1,47 @@
+async function getMicStream() {
+  return await navigator.mediaDevices.getUserMedia({
+    audio: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  });
+}
+
+export async function startMic(onAudio: (data: Int16Array) => void) {
+  const stream = await getMicStream();
+
+  const audioCtx = new AudioContext({ sampleRate: 48000 });
+  await audioCtx.audioWorklet.addModule("/audio-worklet.js");
+
+  const source = audioCtx.createMediaStreamSource(stream);
+  const worklet = new AudioWorkletNode(audioCtx, "mic-processor");
+
+  source.connect(worklet);
+  worklet.connect(audioCtx.destination); // optional
+
+  worklet.port.onmessage = (e) => {
+    const float32 = e.data as Float32Array;
+    const pcm16 = float32ToPCM16(float32);
+    onAudio(pcm16);
+  };
+}
+
+function float32ToPCM16(float32: Float32Array): Int16Array {
+  const pcm16 = new Int16Array(float32.length);
+  for (let i = 0; i < float32.length; i++) {
+    const s = Math.max(-1, Math.min(1, float32[i]));
+    pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+  }
+  return pcm16;
+}
+
+export function sendVoice(ws: WebSocket, voiceId: number, pcm: Int16Array) {
+  const buffer = new ArrayBuffer(2 + pcm.byteLength);
+  const view = new DataView(buffer);
+
+  view.setUint16(0, voiceId, true); // little endian
+  new Uint8Array(buffer, 2).set(new Uint8Array(pcm.buffer));
+
+  ws.send(buffer);
+}

src/lib/auth.ts

@@ -13,6 +13,7 @@ type AuthParams = {
   onNewMessage?: (m: Message) => void;
   messageStore: MessageStore;
   onIndicator?: (i: IndicatorContext) => void;
+  onVoice?: (userId: string, bytes: number[]) => void;
 };
 
 export default async function auth({

src/types/protocol.ts

@@ -20,7 +20,9 @@ export type ClientMessage =
   | EnumType<"delete_message", { message_id: number }>
   | EnumType<"edit_message", { message_id: number; new_contents: string }>
   | EnumType<"load_chunk", { channel_id: string; chunk_id: number }>
-  | EnumType<"typing", { channel_id: string }>;
+  | EnumType<"typing", { channel_id: string }>
+  | EnumType<"join_voice", { channel_id: string }>
+  | EnumType<"leave_voice", { channel_id: string }>;
 
 export type ServerMessage =
   | EnumType<"authenticated", { uuid: string; indicators: IndicatorContext[] }>
@@ -31,4 +33,13 @@ export type ServerMessage =
   | EnumType<"presence_update", { user_id: String; status: String }>
   | EnumType<"indicator", IndicatorContext>
   | EnumType<"shutdown", { message: string }>
+  | EnumType<"chunk", Message[]>
+  | EnumType<
+      "voice_join",
+      { channel_id: string; user_id: string; voice_id: number }
+    >
+  | EnumType<
+      "voice_leave",
+      { channel_id: string; user_id: string; voice_id: number }
+    >
   | EnumType<"chunk", Message[]>;