Voice agent visuals

import { createContext, useContext } from "react";
import type { SharedValue } from "react-native-reanimated";
import type { VoiceAgentState } from "stream-chat-react-native/ai-components";

/** A live voice conversation with the assistant, as the app's voice backend reports it. */
export type VoiceSession = {
  /** What the assistant is doing: idle, connecting, listening, thinking or speaking. */
  state: VoiceAgentState;
  /** The level of the user's microphone, from 0 to 1, or `null` while there is no level. */
  micLevel: SharedValue<number | null>;
  /** The level of the assistant's voice, from 0 to 1, or `null` while there is no level. */
  agentLevel: SharedValue<number | null>;
  /** Ends the conversation. */
  end: () => void;
};

const VoiceSessionContext = createContext<VoiceSession | undefined>(undefined);

/** Makes the app's voice session available to the screen that shows the panel. */
export const VoiceSessionProvider = VoiceSessionContext.Provider;

/** The voice session, or `undefined` while there is none. */
export const useVoiceSession = () => useContext(VoiceSessionContext);

import { useEffect } from "react";
import { useSharedValue } from "react-native-reanimated";

/**
 * A shared value that follows a source of audio levels from 0 to 1. `subscribe` calls `onLevel`
 * for every level and returns a function that stops it. Pass a function that doesn't change
 * between renders, such as one declared at module scope or wrapped in `useCallback()`.
 */
export const useAudioLevel = (
  subscribe: (onLevel: (level: number) => void) => () => void,
) => {
  const level = useSharedValue<number | null>(null);

  useEffect(() => {
    const unsubscribe = subscribe((next) => {
      level.value = next;
    });
    return () => {
      unsubscribe();
      level.value = null;
    };
  }, [level, subscribe]);

  return level;
};

import { Pressable, StyleSheet, Text, View } from "react-native";
import { useAnnounceOnStateChange } from "stream-chat-react-native";
import {
  StreamTheme,
  VoiceGlow,
  VoiceWave,
} from "stream-chat-react-native/ai-components";
import type {
  DeepPartial,
  Theme,
  VoiceAgentState,
} from "stream-chat-react-native/ai-components";
import type { VoiceSession } from "./VoiceSession";

const BACKDROP = "#0b0b10";
const COLORS = ["#005fff", "#7b5cff", "#00c2ff"];

// Declared once, because StreamTheme merges the theme again whenever it gets a new style.
const visualTheme: DeepPartial<Theme> = {
  voiceVisualization: { backdropColor: BACKDROP },
};

/** What the status line shows in each state. */
const statusLabels: Record<VoiceAgentState, string> = {
  connecting: "Connecting...",
  idle: "Ready",
  listening: "Listening",
  speaking: "Speaking",
  thinking: "Thinking...",
};

const inConversation =
  "Voice chat started. Talk, and the assistant answers out loud.";

/**
 * What a screen reader announces in each state. Listening, thinking and speaking can be heard, so
 * they share one sentence: it is announced once, when the conversation starts, and not on every
 * turn. Only the states with no sound get their own words.
 */
const announcements: Record<VoiceAgentState, string> = {
  connecting: "Connecting to the assistant",
  idle: "Voice chat ready",
  listening: inConversation,
  speaking: inConversation,
  thinking: inConversation,
};

/** The visuals say the state in motion only, so assistive technology skips them. */
const hiddenFromAssistiveTechnology = {
  accessibilityElementsHidden: true,
  importantForAccessibility: "no-hide-descendants",
} as const;

/**
 * A call-like strip above the message list while a voice conversation runs: the assistant as a
 * glow that moves with its state and voice, a status line, a button that ends the conversation,
 * and the user's microphone as a wave.
 */
export const VoiceAgentPanel = ({ session }: { session: VoiceSession }) => {
  const { agentLevel, end, micLevel, state } = session;
  useAnnounceOnStateChange(announcements[state]);

  const isUserTalking = state === "listening";
  const isAgentTalking = state === "speaking";

  return (
    <StreamTheme style={visualTheme}>
      <View style={styles.panel}>
        <View style={styles.row}>
          <VoiceGlow
            {...hiddenFromAssistiveTechnology}
            audioLevel={isAgentTalking ? agentLevel : undefined}
            colors={COLORS}
            state={state}
            style={styles.glow}
          />
          <View style={styles.text}>
            <Text style={styles.name}>Voice chat with Assistant</Text>
            <Text style={styles.status}>{statusLabels[state]}</Text>
          </View>
          <Pressable
            accessibilityLabel="End voice chat"
            accessibilityRole="button"
            onPress={end}
            style={styles.end}
          >
            <Text style={styles.endText}>End</Text>
          </Pressable>
        </View>
        <VoiceWave
          {...hiddenFromAssistiveTechnology}
          audioLevel={isUserTalking ? micLevel : undefined}
          colors={COLORS}
          state={isUserTalking ? "listening" : "idle"}
          style={styles.wave}
        />
      </View>
    </StreamTheme>
  );
};

const styles = StyleSheet.create({
  end: {
    borderColor: "#ff636e",
    borderRadius: 999,
    borderWidth: 1,
    paddingHorizontal: 16,
    paddingVertical: 8,
  },
  endText: { color: "#ff636e", fontWeight: "600" },
  glow: { height: 96, width: 96 },
  name: { color: "#a1a1aa", fontSize: 13 },
  panel: { backgroundColor: BACKDROP, paddingEnd: 16 },
  row: { alignItems: "center", flexDirection: "row", gap: 12 },
  status: { color: "#ffffff", fontSize: 17, fontWeight: "600" },
  text: { flex: 1 },
  wave: { alignSelf: "center", height: 112, width: 112 },
});

import type { Channel as StreamChannel } from "stream-chat";
import {
  Channel,
  MessageComposer,
  MessageList,
} from "stream-chat-react-native";
import { VoiceAgentPanel } from "./VoiceAgentPanel";
import { useVoiceSession } from "./VoiceSession";

/** The assistant's channel, with the voice panel above the messages during a voice conversation. */
export const VoiceChannelScreen = ({ channel }: { channel: StreamChannel }) => {
  const session = useVoiceSession();

  return (
    <Channel channel={channel}>
      {session ? <VoiceAgentPanel session={session} /> : null}
      <MessageList />
      <MessageComposer />
    </Channel>
  );
};

import { useEffect, useMemo, useState } from "react";
import type { PropsWithChildren } from "react";
import { useDictationAudioLevel } from "stream-chat-react-native/ai-components";
import type { VoiceAgentState } from "stream-chat-react-native/ai-components";
import { useAudioLevel } from "./useAudioLevel";
import { VoiceSessionProvider } from "./VoiceSession";

const states: VoiceAgentState[] = [
  "connecting",
  "listening",
  "thinking",
  "speaking",
];

// Stands in for the assistant's audio level from your call: a new level 15 times a second.
const subscribeToAgentLevel = (onLevel: (level: number) => void) => {
  const interval = setInterval(() => onLevel(0.2 + 0.8 * Math.random()), 66);
  return () => clearInterval(interval);
};

export const VoiceSessionHost = ({ children }: PropsWithChildren) => {
  const [step, setStep] = useState(0);
  const [isActive, setIsActive] = useState(true);
  // The microphone's level while dictation listens, and null otherwise.
  const micLevel = useDictationAudioLevel();
  const agentLevel = useAudioLevel(subscribeToAgentLevel);

  useEffect(() => {
    const interval = setInterval(() => setStep((current) => current + 1), 3000);
    return () => clearInterval(interval);
  }, []);

  const state = states[step % states.length];
  const session = useMemo(
    () =>
      isActive
        ? { agentLevel, end: () => setIsActive(false), micLevel, state }
        : undefined,
    [agentLevel, isActive, micLevel, state],
  );

  return (
    <VoiceSessionProvider value={session}>{children}</VoiceSessionProvider>
  );
};