Quickstart

import "@stream-io/calypso/chat";
import type { Session } from "@stream-io/calypso";
import { useMemo } from "react";
import type { Channel as ChatChannel, StreamChat } from "stream-chat";
import {
  AIStateIndicator,
  Channel,
  Chat,
  MessageComposer,
  MessageList,
  WithComponents,
} from "stream-chat-react";
import "stream-chat-react/css/index.css";

// The agent's replies carry `ai_generated: true`.
declare module "stream-chat" {
  interface CustomMessageData {
    ai_generated?: boolean;
  }
}

/** Opens the session's channel, as the chat's views show it. */
export async function openChat(session: Session) {
  // `session.chat()` connects to Stream Chat with the user set above.
  const { client, channel } = await session.chat();
  return {
    session,
    client: client as StreamChat,
    channel: channel as ChatChannel,
  };
}

export type AgentChat = Awaited<ReturnType<typeof openChat>>;

export const AgentConversation = ({ chat }: { chat: AgentChat }) => {
  // While the agent answers, the composer shows a Stop button. Its default sends
  // `ai_indicator.stop` to the channel, which the agent does not act on, so stop
  // the reply through the session instead.
  const StopButton = useMemo(
    () => () => (
      <button
        aria-label="Stop"
        className="str-chat__stop-ai-generation-button"
        onClick={() => chat.session.interrupt()}
        type="button"
      />
    ),
    [chat.session],
  );

  return (
    <Chat
      client={chat.client}
      // Renders the agent's replies as streaming markdown.
      isMessageAIGenerated={(message) => Boolean(message.ai_generated)}
    >
      <WithComponents overrides={{ StopAIGenerationButton: StopButton }}>
        <Channel channel={chat.channel}>
          <MessageList />
          {/* Thinking and generating states, for text and voice sessions. */}
          <AIStateIndicator />
          <MessageComposer
            overrideSubmitHandler={async ({ message }) => {
              await chat.session.responses.create(message.text ?? "");
            }}
          />
        </Channel>
      </WithComponents>
    </Chat>
  );
};

import android.content.Context
import androidx.activity.compose.setContent
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.ui.Modifier
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import androidx.lifecycle.lifecycleScope
import io.getstream.chat.android.ai.compose.conversation.AIMessage
import io.getstream.chat.android.ai.compose.conversation.AIMessageRole
import io.getstream.chat.android.ai.compose.parts.AIClientIdentity
import io.getstream.chat.android.ai.compose.parts.AIMessagePart
import io.getstream.chat.android.ai.compose.ui.component.AIConversation
import io.getstream.chat.android.ai.compose.ui.component.AIToolApprover
import io.getstream.chat.android.ai.compose.ui.component.ChatComposer
import io.getstream.chat.android.client.channel.state.ChannelState
import io.getstream.chat.android.models.Message
import io.getstream.visionagents.android.core.AgentSession
import io.getstream.visionagents.android.core.AgentsException
import io.getstream.visionagents.android.core.Conversation
import io.getstream.visionagents.android.core.channelState
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.flatMapLatest
import kotlinx.coroutines.flow.flowOf
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.launch

// In your activity, with the session from step 3.
lifecycleScope.launch {
    val chat = AgentChat.open(this@MainActivity, session, lifecycleScope)
    setContent { AgentConversation(chat, Modifier.fillMaxSize()) }
}

@Composable
fun AgentConversation(chat: AgentChat, modifier: Modifier = Modifier) {
    val messages by chat.messages.collectAsStateWithLifecycle()
    val answering by chat.isAnswering.collectAsStateWithLifecycle()

    Column(modifier) {
        // Each question moves to the top, and its reply streams in below it with its
        // reasoning and tool calls.
        AIConversation(
            messages = messages,
            modifier = Modifier.weight(1f),
            // The agent is on it, and its reply isn't in the channel yet.
            isThinking = answering && messages.lastOrNull()?.role != AIMessageRole.Assistant,
            // Asks the user before a tool that asks first runs.
            approver = chat.approver,
        )
        ChatComposer(
            // The agent does not answer a message sent to the channel: ask the session.
            onSendClick = { chat.send(it.text) },
            onStopClick = chat::stop,
            // While the agent answers, the composer shows a Stop button.
            isGenerating = answering,
        )
    }
}

/** The session's channel, as the AI components show it. */
@OptIn(ExperimentalCoroutinesApi::class)
class AgentChat private constructor(
    val session: AgentSession,
    val approver: AIToolApprover,
    channel: StateFlow<ChannelState?>,
    private val scope: CoroutineScope,
) {
    /** The channel's messages, oldest first, as Stream Chat keeps them up to date. */
    val messages: StateFlow<List<AgentMessage>> = channel
        .flatMapLatest { it?.messages ?: flowOf(emptyList()) }
        .map { messages -> messages.filter { it.deletedAt == null }.map(::AgentMessage) }
        .stateIn(scope, SharingStarted.Eagerly, emptyList())

    /** Whether a reply is on its way, from the first thought to the last word. */
    val isAnswering: StateFlow<Boolean> = combine(session.conversation, messages) { conversation, messages ->
        when (conversation.state) {
            Conversation.State.Responding, is Conversation.State.Working -> true
            else -> messages.lastOrNull()?.isGenerating == true
        }
    }.stateIn(scope, SharingStarted.Eagerly, false)

    fun send(text: String) {
        scope.launch {
            try {
                session.responses.create(text)
            } catch (_: AgentsException) {
                // Not sent: the user can send it again.
            }
        }
    }

    /** Stops the reply in flight. What was written so far is kept. */
    fun stop() {
        scope.launch {
            try {
                session.interrupt()
            } catch (_: AgentsException) {
                // The reply finished first.
            }
        }
    }

    companion object {
        /** Opens the session's channel. Pass a [scope] on the main thread, such as `lifecycleScope`. */
        suspend fun open(context: Context, session: AgentSession, scope: CoroutineScope): AgentChat {
            // The user set above, who answers the calls the agent addresses to them.
            val approver = AIToolApprover("jlahey", AIClientIdentity.installId(context), scope) { call, allowed ->
                session.decide(call.id, allowed)
            }
            // Connects to Stream Chat as the user set above and watches the session's channel.
            return AgentChat(session, approver, session.channelState(context, scope), scope)
        }
    }
}

/** A message of the channel, as the AI components show it. */
data class AgentMessage(val message: Message) : AIMessage {
    override val id: String = message.id
    // The agent's replies carry `ai_generated: true`.
    override val role: AIMessageRole =
        if (message.extraData["ai_generated"] == true) AIMessageRole.Assistant else AIMessageRole.User
    override val text: String = message.text
    override val isGenerating: Boolean = message.extraData["generating"] == true
    // The reply's reasoning and tool calls, which the router writes as its attachments. Stream Chat
    // lifts `name` out of an attachment's fields, and a tool call's step needs it back.
    override val parts: List<AIMessagePart> = AIMessagePart.parts(
        message.attachments.map { it.type.orEmpty() to it.extraData + ("name" to it.name) },
    )
}

import StreamChat
import StreamChatAI
import SwiftUI
import VisionAgentsCore

/// Opens the session's channel, as the AI components show it.
@MainActor
func openChat(for session: AgentSession) async throws -> AgentChat {
    // `session.chat()` connects to Stream Chat as the user set above and opens the session's channel.
    let channel = try await session.chat()
    return AgentChat(session: session, channel: channel)
}

struct AgentConversation: View {
    @ObservedObject var chat: AgentChat
    @StateObject private var composer = AIComposerViewModel()

    var body: some View {
        VStack(spacing: 0) {
            // Each question moves to the top, and its reply streams in below it with its
            // reasoning and tool calls.
            AIConversationView(messages: chat.messages)
            AIComposerView(
                viewModel: composer,
                // The agent does not answer a message sent to the channel: ask the session.
                onMessageSend: { message in
                    Task { _ = try? await chat.session.responses.create(message.text) }
                },
                onStopGenerating: {
                    Task { try? await chat.session.interrupt() }
                }
            )
        }
        // While the agent writes, the composer shows a Stop button.
        .onReceive(chat.$isAnswering) { composer.isGenerating = $0 }
    }
}

/// The session's channel, as the AI components show it.
@MainActor
final class AgentChat: ObservableObject, ChatChannelControllerDelegate {
    let session: AgentSession
    @Published private(set) var messages: [AgentMessage] = []
    @Published private(set) var isAnswering = false

    private let channel: ChatChannelController

    init(session: AgentSession, channel: ChatChannelController) {
        self.session = session
        self.channel = channel
        channel.delegate = self
        channel.synchronize()
    }

    func channelController(
        _ channelController: ChatChannelController,
        didUpdateMessages changes: [ListChange<ChatMessage>]
    ) {
        // The channel lists its messages newest first, the conversation oldest first.
        messages = channelController.messages.reversed().map(AgentMessage.init)
        isAnswering = messages.last?.isGenerating == true
    }
}

/// A message of the channel, as the AI components show it.
struct AgentMessage: AIMessage {
    let message: ChatMessage

    var id: MessageId { message.id }
    // The agent's replies carry `ai_generated: true`.
    var role: AIMessageRole {
        message.extraData["ai_generated"]?.boolValue == true ? .assistant : .user
    }
    var text: String { message.text }
    var isGenerating: Bool { message.extraData["generating"]?.boolValue == true }
    // The reply's reasoning and tool calls, which the router writes as its attachments.
    var parts: [AIMessagePart] {
        AIMessagePart.parts(from: message.allAttachments.map { ($0.type.rawValue, $0.payload) })
    }
}

import { VoiceSession } from "@stream-io/calypso/video";
import {
  ParticipantsAudio,
  StreamCall,
  StreamVideo,
  useCallStateHooks,
} from "@stream-io/video-react-sdk";
import { useEffect, useSyncExternalStore } from "react";
import { Channel, Chat, MessageList } from "stream-chat-react";
import { VoiceGlow } from "stream-chat-react/ai-components";

/** Starts a session with voice on, and opens its transcript. Call it from a click. */
export async function startVoice() {
  // The agent joins the session's call right away.
  const voice = await VoiceSession.start(api.agent("my-agent"));
  // What is said on the call is written into the session's channel, so the chat's views show it.
  return { voice, chat: await openChat(voice.session) };
}

export const AgentVoice = ({
  voice,
  chat,
}: {
  voice: VoiceSession;
  chat: AgentChat;
}) => {
  // What the agent is doing, in the states the glow takes.
  const { call, client, muted, status } = useSyncExternalStore(
    voice.subscribe,
    voice.get,
  );

  // Joins the call from this device, with the microphone on. Run twice, it joins once.
  useEffect(() => {
    voice
      .join()
      .catch((error: unknown) =>
        console.error("Could not join the call", error),
      );
  }, [voice]);

  return (
    <div>
      <VoiceGlow state={status} style={{ width: 240, height: 240 }} />
      {call && client && (
        <StreamVideo client={client}>
          <StreamCall call={call}>
            <AgentAudio />
          </StreamCall>
        </StreamVideo>
      )}
      {/* What the user said and the agent's replies, as in the chat. */}
      <Chat
        client={chat.client}
        isMessageAIGenerated={(message) => Boolean(message.ai_generated)}
      >
        <Channel channel={chat.channel}>
          <MessageList />
        </Channel>
      </Chat>
      <button onClick={() => void voice.setMuted(!muted)}>
        {muted ? "Unmute" : "Mute"}
      </button>
      {/* Leaves the call and ends the session this page started. */}
      <button onClick={() => void voice.leave()}>Hang up</button>
    </div>
  );
};

/** Plays the agent. */
const AgentAudio = () => {
  const { useRemoteParticipants } = useCallStateHooks();
  return <ParticipantsAudio participants={useRemoteParticipants()} />;
};

import io.getstream.chat.android.ai.compose.ui.component.AIVoiceAgentState
import io.getstream.chat.android.ai.compose.ui.component.AIVoiceGlow
import io.getstream.visionagents.android.core.Conversation
import io.getstream.visionagents.android.core.SessionOptions
import io.getstream.visionagents.android.core.VoiceSession

/** Starts a session with voice on. Ask for RECORD_AUDIO before you call it. */
suspend fun startVoice(context: Context): VoiceSession =
    // The agent joins the session's call right away.
    VoiceSession.start(context, agents, SessionOptions(agent = "my-agent"))

@Composable
fun AgentVoice(voice: VoiceSession) {
    val scope = rememberCoroutineScope()
    val call by voice.call.collectAsStateWithLifecycle()
    val muted by voice.isMuted.collectAsStateWithLifecycle()
    val conversation by voice.session.conversation.collectAsStateWithLifecycle()
    // Who on the call is speaking: the agent, or the person holding the phone.
    val speakers by remember(call) { call?.state?.activeSpeakers ?: MutableStateFlow(emptyList()) }
        .collectAsStateWithLifecycle()

    // Joins the call from this device, with the microphone on and the agent on the speaker.
    LaunchedEffect(voice) { voice.join() }

    // Speaking comes from the call: a reply is written before it is spoken.
    val state = when {
        call == null -> AIVoiceAgentState.Connecting
        speakers.any { !it.isLocal } -> AIVoiceAgentState.Speaking
        speakers.any { it.isLocal } -> AIVoiceAgentState.Listening
        else -> when (conversation.state) {
            Conversation.State.Listening -> AIVoiceAgentState.Listening
            Conversation.State.Responding, is Conversation.State.Working -> AIVoiceAgentState.Thinking
            else -> AIVoiceAgentState.Idle
        }
    }

    Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(16.dp)) {
        AIVoiceGlow(state = state, modifier = Modifier.size(240.dp))
        // What the user said and the agent's replies.
        LazyColumn(Modifier.weight(1f)) {
            items(conversation.turns) { turn -> Text(turn.text) }
        }
        Row(horizontalArrangement = Arrangement.spacedBy(24.dp)) {
            Button(onClick = { voice.setMuted(!muted) }) { Text(if (muted) "Unmute" else "Mute") }
            // Leaves the call and ends the session this device started.
            Button(onClick = { scope.launch { voice.leave() } }) { Text("Hang up") }
        }
    }
}

import StreamChatAI
import StreamVideo
import SwiftUI
import VisionAgentsCore

/// Starts a session with voice on, and opens its transcript.
@MainActor
func startVoice() async throws -> (voice: VoiceSession, chat: AgentChat) {
    // The agent joins the session's call right away.
    let voice = try await VoiceSession.start(agents: agents, agent: "my-agent")
    // The session's socket, which follows what the agent is doing and runs the tools this phone offers.
    await voice.session.start()
    // What is said on the call is written into the session's channel, so the chat's views show it.
    return (voice, try await openChat(for: voice.session))
}

struct AgentVoice: View {
    @ObservedObject var voice: VoiceSession
    @ObservedObject var chat: AgentChat

    var body: some View {
        VStack(spacing: 16) {
            if let call = voice.call {
                AgentStatus(voice: voice, call: call.state)
            } else {
                AIVoiceGlowView(state: .connecting)
                    .frame(width: 240, height: 240)
            }
            // What the user said and the agent's replies, as in the chat.
            AIConversationView(messages: chat.messages)
            HStack(spacing: 24) {
                Button {
                    Task { await voice.setMuted(!voice.isMuted) }
                } label: {
                    Image(systemName: voice.isMuted ? "mic.slash.fill" : "mic.fill")
                }
                // Leaves the call and ends the session this device started.
                Button(role: .destructive) {
                    Task { await voice.leave() }
                } label: {
                    Image(systemName: "phone.down.fill")
                }
            }
            .font(.title2)
            .buttonStyle(.bordered)
        }
        // Joins the call from this device, with the microphone on and the agent on the speaker.
        .task { await voice.join() }
    }
}

/// What the agent is doing, as the glow shows it.
struct AgentStatus: View {
    @ObservedObject var voice: VoiceSession
    // The call's participants, and which of them is speaking.
    @ObservedObject var call: CallState

    var body: some View {
        AIVoiceGlowView(state: state)
            .frame(width: 240, height: 240)
    }

    private var state: AIVoiceAgentState {
        // Speaking comes from the call: a reply is written before it is spoken,
        // and is done before the agent stops talking.
        if call.remoteParticipants.contains(where: \.isSpeaking) { return .speaking }
        if call.localParticipant?.isSpeaking == true { return .listening }
        switch voice.session.state {
        case .listening: return .listening
        case .responding, .working: return .thinking
        case .idle, .ended: return .idle
        }
    }
}