Personal Assistant

name: assistant
mode: voice
llm: llm-fast
connectors:
  - gmail # each user signs in to their own Gmail

import { Tools } from "@stream-io/stream-agents";
import { z } from "zod";

export const tools = new Tools()
  .register({
    name: "get_local_context",
    description:
      "The user's current date, local time, timezone and language. Call it before " +
      "answering anything that depends on today or the time.",
    parameters: z.object({}),
    run: () =>
      `It is ${new Date().toString()}, in ` +
      `${Intl.DateTimeFormat().resolvedOptions().timeZone}. ` +
      `Their language is ${navigator.language}.`,
  })
  .register({
    name: "get_location",
    description:
      "Where the user is. Only call it when the answer depends on it, " +
      "such as the weather or places nearby.",
    parameters: z.object({
      reason: z.string().describe("why you need it, in a few words"),
    }),
    displayTitle: "Checking where you are",
    approval: {
      title: "Share your location?",
      reasonArgument: "reason",
      allowTitle: "Share",
      declineTitle: "Not now",
    },
    run: () =>
      new Promise<string>((resolve, reject) =>
        navigator.geolocation.getCurrentPosition(
          ({ coords }) =>
            resolve(
              `Latitude ${coords.latitude}, longitude ${coords.longitude}.`,
            ),
          () => reject(new Error("The browser couldn't tell where they are.")),
        ),
      ),
  });

import { api, openConversation, type AgentChat } from "./agent";
import { tools } from "./tools";

export const returnUrl = `${window.location.origin}/connected.html`;

/** Marks the conversations this page opens, for the list to find. */
export const conversationLabel = { opened_from: "assistant" };

export async function openAssistant(
  sessionId: string | undefined,
): Promise<AgentChat> {
  const assistant = api.agent("assistant");
  const session = sessionId
    ? await assistant.sessions.resume(sessionId, { tools })
    : await assistant.sessions.create({
        tools,
        returnUrl,
        custom: conversationLabel,
      });
  return openConversation(session);
}

<script type="module">
  import { completeConnectorSignIn } from "stream-chat-react/ai-components";
  completeConnectorSignIn();
</script>

import type { Schemas } from "@stream-io/stream-agents";
import { useEffect, useState } from "react";
import { Button } from "stream-chat-react";

import { api } from "./agent";
import { conversationLabel } from "./assistant";

export const ConversationList = ({
  onSelect,
  selectedId,
  version,
}: {
  onSelect: (sessionId: string | undefined) => void;
  /** The open conversation, marked in the list. */
  selectedId?: string;
  /** Changes when the list may have: a conversation created, or answered. */
  version: number;
}) => {
  const [sessions, setSessions] = useState<Schemas["Session"][]>([]);
  const [search, setSearch] = useState("");

  useEffect(() => {
    let isCurrent = true;
    const all = api.agent("assistant").sessions;
    // The conversations this page opened, typed or switched to voice.
    const filters = { limit: 50, custom: conversationLabel };
    const page = search ? all.search(search, filters) : all.query(filters);
    page.then(
      ({ items }) => {
        // An incognito session keeps no conversation to open.
        if (isCurrent)
          setSessions(items.filter((each) => each.conversation_id));
      },
      () => undefined,
    );
    return () => {
      isCurrent = false;
    };
  }, [search, version]);

  return (
    <nav className="conversations">
      <Button
        appearance="solid"
        onClick={() => onSelect(undefined)}
        size="sm"
        type="button"
        variant="primary"
      >
        New conversation
      </Button>
      <input
        className="conversations__search"
        onChange={(event) => setSearch(event.target.value)}
        placeholder="Search"
        type="search"
        value={search}
      />
      {sessions.map((session) => (
        <button
          aria-current={session.id === selectedId}
          className="conversations__item"
          key={session.id}
          onClick={() => onSelect(session.id)}
          type="button"
        >
          {session.title || "New conversation"}
        </button>
      ))}
    </nav>
  );
};

import {
  type FormEvent,
  useCallback,
  useEffect,
  useMemo,
  useRef,
  useState,
  useSyncExternalStore,
} from "react";
import { Chat } from "stream-chat-react";
import {
  AIClientIdentity,
  AIConversation,
  AIMessageComposer,
  createPopupAuthorizer,
  type AIToolApprover,
} from "stream-chat-react/ai-components";

import type { AgentChat } from "./agent";
import { openAssistant, returnUrl } from "./assistant";
import { ConversationList } from "./ConversationList";
import { useAgentMessages } from "./useAgentMessages";

const messageOf = (error: unknown) =>
  error instanceof Error ? error.message : String(error);

export const AssistantHome = () => {
  const [chat, setChat] = useState<AgentChat>();
  const [isOpening, setIsOpening] = useState(false);
  const [failure, setFailure] = useState<string>();
  // Bumped when the list may have changed, so it loads again.
  const [listVersion, setListVersion] = useState(0);
  const refreshList = useCallback(() => setListVersion((n) => n + 1), []);
  // The latest pick, so an open that a later pick overtook is dropped.
  const latestPick = useRef(0);

  // Opens from the list's click, never from an effect, so New conversation
  // creates one session per click.
  const open = async (sessionId: string | undefined) => {
    const pick = ++latestPick.current;
    setChat(undefined);
    setFailure(undefined);
    setIsOpening(true);
    try {
      const opened = await openAssistant(sessionId);
      if (pick !== latestPick.current) {
        void opened.session.stopWatching();
        return;
      }
      setChat(opened);
      if (!sessionId) refreshList();
    } catch (error) {
      if (pick === latestPick.current) setFailure(messageOf(error));
    } finally {
      if (pick === latestPick.current) setIsOpening(false);
    }
  };

  // Stops watching a conversation once another one opens, or the page goes.
  // The conversation goes on; picking it again carries it on.
  useEffect(() => {
    if (!chat) return;
    return () => void chat.session.stopWatching();
  }, [chat]);

  return (
    // `str-chat` is the SDK's theme scope: its fonts, colors and spacing.
    <div className="str-chat assistant">
      <ConversationList
        onSelect={(sessionId) => void open(sessionId)}
        selectedId={chat?.session.id}
        version={listVersion}
      />
      {chat ? (
        <AssistantConversation
          chat={chat}
          key={chat.session.id}
          onReplied={refreshList}
        />
      ) : (
        <p
          className="assistant-placeholder"
          role={failure ? "alert" : undefined}
        >
          {failure ??
            (isOpening
              ? "Opening…"
              : "Pick a conversation, or start a new one.")}
        </p>
      )}
    </div>
  );
};

const AssistantConversation = ({
  chat,
  onReplied,
}: {
  chat: AgentChat;
  /** Called when a reply ends, which is when the router titles a new conversation. */
  onReplied: () => void;
}) => {
  const { session, client, channel } = chat;
  const messages = useAgentMessages(channel);
  const [failure, setFailure] = useState<string>();
  // The router refuses a question while the agent is still answering one.
  const { kind } = useSyncExternalStore(
    session.state.subscribe,
    session.state.get,
  );
  const isAnswering = kind === "responding" || kind === "working";

  const wasAnswering = useRef(false);
  useEffect(() => {
    if (wasAnswering.current && !isAnswering) onReplied();
    wasAnswering.current = isAnswering;
  }, [isAnswering, onReplied]);

  const approver = useMemo<AIToolApprover>(
    () => ({
      userId: client.userID ?? "",
      clientId: AIClientIdentity.installId(),
      decide: async (call, allowed) => session.decide(call.id, { allowed }),
      authorizer: createPopupAuthorizer({
        returnUrl,
        decline: async (call) => session.declineLogin(call.id),
      }),
    }),
    [client, session],
  );

  const handleSubmit = (event: FormEvent<HTMLFormElement>) => {
    event.preventDefault();
    // The draft waits until the reply is done, as the send button does.
    if (isAnswering) return;
    const form = event.currentTarget;
    const text = new FormData(form).get("message");
    if (typeof text !== "string" || !text.trim()) return;
    setFailure(undefined);
    // Cleared once it's sent, so a refused question stays in the composer.
    session.responses.create(text.trim()).then(
      () => form.reset(),
      (error: unknown) => setFailure(messageOf(error)),
    );
  };

  return (
    <Chat client={client}>
      <main className="assistant-conversation">
        <div className="assistant-messages">
          <AIConversation approver={approver} messages={messages} />
        </div>
        <div className="assistant-composer">
          {failure && <p role="alert">{failure}</p>}
          <AIMessageComposer onSubmit={handleSubmit}>
            <AIMessageComposer.TextInput placeholder="Ask anything" />
            <div className="assistant-composer__actions">
              <AIMessageComposer.SubmitButton />
            </div>
          </AIMessageComposer>
        </div>
      </main>
    </Chat>
  );
};

import type { ImageSource } from "@stream-io/stream-agents";

export async function imageSource(file: File): Promise<ImageSource> {
  const bitmap = await createImageBitmap(file);
  const scale = Math.min(1, 1280 / Math.max(bitmap.width, bitmap.height));
  const canvas = document.createElement("canvas");
  canvas.width = Math.round(bitmap.width * scale);
  canvas.height = Math.round(bitmap.height * scale);
  canvas.getContext("2d")?.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
  return { url: canvas.toDataURL("image/jpeg", 0.7) };
}

import { VoiceSession } from "@stream-io/stream-agents/video";
// The call view of the Voice agent use case.
import { VoiceCall } from "./VoiceCall";

// In AssistantConversation, with its other hooks:
const [voice, setVoice] = useState<VoiceSession>();

// Off the call when the call view closes: on Hang up, or when the
// conversation closes mid-call, such as when the user opens another one.
useEffect(() => {
  if (!voice) return;
  return () => {
    void voice.leave().catch(() => undefined);
    // Takes the agent off the call. The conversation goes on in writing.
    void session.voice.stop().catch(() => undefined);
  };
}, [voice, session]);

// From the Talk button's click, since the browser plays audio and asks for the
// microphone only after one.
const talk = async () => {
  setFailure(undefined);
  try {
    await session.voice.start();
    setVoice(VoiceSession.of(session));
  } catch (error) {
    setFailure(messageOf(error));
  }
};

// And before its return, after every hook:
if (voice) {
  return (
    <VoiceCall
      channel={channel}
      chatClient={client}
      onHangUp={() => setVoice(undefined)}
      voice={voice}
    />
  );
}