Voice Agent

import "@stream-io/stream-agents/chat";
import type { Client } from "@stream-io/stream-agents";
import { VoiceSession } from "@stream-io/stream-agents/video";
import { useEffect, useState } from "react";
import type { Channel as ChatChannel, StreamChat } from "stream-chat";
import { Button } from "stream-chat-react";
import { VoiceGlow } from "stream-chat-react/ai-components";
import "stream-chat-react/css/index.css";
import "stream-chat-react/css/ai-components.css";

import { VoiceCall } from "./VoiceCall";

export type VoiceAgentCall = {
  voice: VoiceSession;
  chatClient: StreamChat;
  channel: ChatChannel;
};

/** Puts the agent on a new session's call, and opens the session's channel. */
const startCall = async (
  api: Client,
  agent: string,
): Promise<VoiceAgentCall> => {
  const voice = await VoiceSession.start(api.agent(agent));
  try {
    // What is said on the call is written into this channel.
    const { client, channel } = await voice.session.chat();
    const transcript = channel as ChatChannel;
    // Loads what was said so far and receives the rest live.
    await transcript.watch();
    return { voice, chatClient: client as StreamChat, channel: transcript };
  } catch (error) {
    // Without a transcript there is no call to show: end the session it opened.
    await voice.end().catch(() => undefined);
    throw error;
  }
};

// `api` is the Client from the Quickstart, with the user set.
export const VoiceAgent = ({ api, agent }: { api: Client; agent: string }) => {
  const [active, setActive] = useState<VoiceAgentCall>();
  const [isStarting, setIsStarting] = useState(false);
  const [failure, setFailure] = useState<string>();

  // Leaves the call, and ends its session, if the view goes away mid-call,
  // such as when the user navigates elsewhere. It runs when a call starts,
  // not on mount, so Strict Mode doesn't end the call it just started.
  useEffect(() => {
    if (!active) return;
    return () => void active.voice.leave().catch(() => undefined);
  }, [active]);

  const start = async () => {
    setIsStarting(true);
    setFailure(undefined);
    try {
      setActive(await startCall(api, agent));
    } catch (error) {
      setFailure(error instanceof Error ? error.message : String(error));
    } finally {
      setIsStarting(false);
    }
  };

  if (active) {
    return <VoiceCall {...active} onHangUp={() => setActive(undefined)} />;
  }
  if (isStarting) {
    return <VoiceGlow className="voice-glow" state="connecting" />;
  }
  return (
    <div className="voice-agent">
      <Button
        appearance="solid"
        onClick={() => void start()}
        size="md"
        type="button"
        variant="primary"
      >
        Talk to the agent
      </Button>
      {failure && <p role="alert">{failure}</p>}
    </div>
  );
};

import {
  ParticipantsAudio,
  StreamCall,
  StreamVideo,
  useCallStateHooks,
} from "@stream-io/video-react-sdk";
import { useEffect, useSyncExternalStore } from "react";
import type { LocalMessage } from "stream-chat";
import {
  AIStateIndicator,
  Attachment,
  Button,
  Channel,
  Chat,
  MessageList,
  WithComponents,
  type AttachmentProps,
} from "stream-chat-react";
import { VoiceGlow } from "stream-chat-react/ai-components";

import { AgentGlow } from "./AgentGlow";
import type { VoiceAgentCall } from "./VoiceAgent";

/**
 * A reply's steps are `ai_*` attachments, and a connector's sign-in also
 * writes a `connector_authorization` one for older clients. `MessageList`
 * draws neither, so leave them out of the transcript.
 */
const isAgentStep = (type: string | undefined) =>
  !!type && (type.startsWith("ai_") || type === "connector_authorization");

const TranscriptAttachment = (props: AttachmentProps) => (
  <Attachment
    {...props}
    attachments={props.attachments.filter(
      (attachment) => !("type" in attachment && isAgentStep(attachment.type)),
    )}
  />
);

// The default asks the user to send a message, which a call has no composer for.
const NothingSaidYet = () => null;

// Declared once, so WithComponents gets the same object on every render.
const overrides = {
  Attachment: TranscriptAttachment,
  EmptyStateIndicator: NothingSaidYet,
};

// `ai_generated` is declared in stream-chat.d.ts, further down this page.
const isMessageAIGenerated = (message: LocalMessage) =>
  Boolean(message.ai_generated);

/** Plays the agent: the other participant on the session's call. */
const AgentAudio = () => {
  const { useRemoteParticipants } = useCallStateHooks();
  return <ParticipantsAudio participants={useRemoteParticipants()} />;
};

export const VoiceCall = ({
  voice,
  chatClient,
  channel,
  onHangUp,
}: VoiceAgentCall & { onHangUp: () => void }) => {
  const { call, client, failure, muted } = useSyncExternalStore(
    voice.subscribe,
    voice.get,
  );

  // Joins from this browser, with the microphone on.
  // Run twice, as in Strict Mode, it joins once.
  useEffect(() => {
    // A failed join also lands in `failure`, shown below.
    voice.join().catch(() => undefined);
  }, [voice]);

  const hangUp = async () => {
    // Leaves the call, and ends the session `VoiceSession.start` opened.
    await voice.leave();
    onHangUp();
  };

  return (
    <div className="voice-call">
      {call && client ? (
        <StreamVideo client={client}>
          <StreamCall call={call}>
            <AgentGlow voice={voice} />
            <AgentAudio />
          </StreamCall>
        </StreamVideo>
      ) : (
        <VoiceGlow className="voice-glow" state="connecting" />
      )}
      {failure && <p role="alert">{failure.message}</p>}
      <WithComponents overrides={overrides}>
        <Chat client={chatClient} isMessageAIGenerated={isMessageAIGenerated}>
          <Channel channel={channel}>
            <MessageList />
            <AIStateIndicator />
            {/* No composer while the call is on. */}
          </Channel>
        </Chat>
      </WithComponents>
      <div className="voice-call__controls">
        <Button
          appearance="outline"
          onClick={() => void voice.setMuted(!muted).catch(() => undefined)}
          size="md"
          type="button"
          variant="secondary"
        >
          {muted ? "Unmute" : "Mute"}
        </Button>
        <Button
          appearance="solid"
          onClick={() => void hangUp()}
          size="md"
          type="button"
          variant="danger"
        >
          Hang up
        </Button>
      </div>
    </div>
  );
};

import type { VoiceSession } from "@stream-io/stream-agents/video";
import { useCallStateHooks } from "@stream-io/video-react-sdk";
import { useSyncExternalStore } from "react";
import {
  useAudioLevel,
  VoiceGlow,
  type VoiceAgentState,
} from "stream-chat-react/ai-components";

const statusLabels: Record<VoiceAgentState, string> = {
  connecting: "Connecting…",
  idle: "Ready",
  listening: "Listening",
  speaking: "Speaking",
  thinking: "Thinking…",
};

/** Render it inside `StreamCall`, so it can read the call's participants. */
export const AgentGlow = ({ voice }: { voice: VoiceSession }) => {
  const { status } = useSyncExternalStore(voice.subscribe, voice.get);
  const { useRemoteParticipants } = useCallStateHooks();
  // The agent is the other participant on the session's call.
  const [agent] = useRemoteParticipants();
  const agentLevel = useAudioLevel(agent?.audioStream);

  return (
    <div className="agent-glow">
      <VoiceGlow
        aria-hidden="true"
        audioLevel={status === "speaking" ? agentLevel : undefined}
        className="voice-glow"
        state={status}
      />
      <p className="agent-glow__status">{statusLabels[status]}</p>
    </div>
  );
};

// The import makes this file extend `stream-chat` instead of replacing it.
import "stream-chat";

declare module "stream-chat" {
  interface CustomMessageData {
    ai_generated?: boolean;
    generating?: boolean;
    interrupted?: boolean;
    source?: string;
    turn_id?: string;
  }
}

import type { Session } from "@stream-io/stream-agents";
import { createContext, useContext, useState } from "react";
import {
  MessageText,
  StreamedMessageText,
  useMessageContext,
  type StreamedMessageTextProps,
} from "stream-chat-react";
import { useHeardText } from "stream-chat-react/ai-components";

/** The voice session's conversation, provided around the transcript. */
export const VoiceSessionContext = createContext<Session | undefined>(
  undefined,
);

// Only a reply that is new when it mounts is paced. Older ones show in full.
const LIVE_WINDOW_MS = 10_000;

export const SpokenMessageText = (props: StreamedMessageTextProps) => {
  const session = useContext(VoiceSessionContext);
  const { message: messageFromContext } = useMessageContext();
  const message = props.message ?? messageFromContext;

  const [live] = useState(
    () => Date.now() - (message.created_at?.getTime() ?? 0) < LIVE_WINDOW_MS,
  );

  const { heardText, isPaced } = useHeardText({
    text: message.text ?? "",
    heard: (now) => session?.speech.heard(message, now),
    subscribe: session
      ? (listener) => session.speech.subscribe(listener)
      : undefined,
    generating: Boolean(message.generating),
    live,
  });

  if (!isPaced) return <StreamedMessageText {...props} />;

  return (
    <MessageText
      message={{ ...message, text: heardText }}
      renderText={props.renderText}
    />
  );
};

import { SpokenMessageText, VoiceSessionContext } from "./SpokenMessageText";

const overrides = {
  Attachment: TranscriptAttachment,
  EmptyStateIndicator: NothingSaidYet,
  StreamedMessageText: SpokenMessageText,
};

// In VoiceCall, around the transcript:
<VoiceSessionContext.Provider value={voice.session}>
  <WithComponents overrides={overrides}>
    <Chat client={chatClient} isMessageAIGenerated={isMessageAIGenerated}>
      <Channel channel={channel}>
        <MessageList />
        <AIStateIndicator />
      </Channel>
    </Chat>
  </WithComponents>
</VoiceSessionContext.Provider>;

import clsx from "clsx";
import { MessageUI, useMessageContext } from "stream-chat-react";

export const AgentMessage = () => {
  const { message } = useMessageContext();

  // Nothing to show: leave it out of the transcript.
  if (!message.text && !message.attachments?.length) return null;

  return (
    <div
      className={clsx("agent-message", {
        "agent-message--speech": message.source === "speech",
        "agent-message--generating": message.generating,
        "agent-message--interrupted": message.interrupted,
      })}
    >
      <MessageUI />
      {message.interrupted && message.text ? (
        <div className="agent-message__note">Interrupted</div>
      ) : null}
    </div>
  );
};

import type { Session } from "@stream-io/stream-agents";
import { useMemo } from "react";
import { MessageComposer, WithComponents } from "stream-chat-react";

// The submit handler sends only the text, so leave out the attachment button.
const NoAttachments = () => null;

export const SessionComposer = ({ session }: { session: Session }) => {
  // The default Stop button sends `ai_indicator.stop`, which the agent
  // doesn't act on, so stop the reply through the session.
  const overrides = useMemo(
    () => ({
      AttachmentSelector: NoAttachments,
      StopAIGenerationButton: () => (
        <button
          aria-label="Stop"
          className="str-chat__stop-ai-generation-button"
          onClick={() => session.interrupt()}
          type="button"
        />
      ),
    }),
    [session],
  );

  return (
    <WithComponents overrides={overrides}>
      <MessageComposer
        overrideSubmitHandler={async ({ message }) => {
          await session.responses.create(message.text ?? "");
        }}
      />
    </WithComponents>
  );
};

import type { Session } from "@stream-io/stream-agents";
import { useSyncExternalStore } from "react";
import { AIToolApprovalCard } from "stream-chat-react/ai-components";

/** The oldest question a tool waits on. */
export const ToolQuestion = ({ session }: { session: Session }) => {
  const { approvals } = useSyncExternalStore(
    session.conversation.subscribe,
    session.conversation.get,
  );
  const request = approvals[0];
  if (!request) return null;

  return (
    <AIToolApprovalCard
      approval={{
        title: request.approval.title,
        message: request.approval.message,
        reason: request.reason,
        allowTitle: request.approval.allowTitle,
        declineTitle: request.approval.declineTitle,
      }}
      key={request.id}
      onDecide={(allowed) => session.decide(request.id, { allowed })}
    />
  );
};