Personal Assistant

name: assistant
mode: voice
llm: llm-fast
connectors:
  - name: gmail # each user signs in to their own Gmail
    tools: [search_threads, get_thread, get_message]

dependencies:
  # Asks for the calendar, the microphone and the camera on Android.
  permission_handler: ^12.0.3

import 'dart:io' show Platform;

import 'package:flutter/services.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:stream_agents_core/stream_agents_core.dart';

/// The calendar and reminders, which the app's iOS and Android code read and write.
const _calendar = MethodChannel('com.example.assistant/calendar');

/// Whether the app may use the calendar, asking the first time on Android.
Future<bool> _calendarAllowed() async =>
    !Platform.isAndroid || await Permission.calendarFullAccess.request().isGranted;

/// Calls the app's calendar code. What it couldn't do comes back as words for the model.
Future<T?> _ask<T>(String method, Map<String, Object?> arguments) async {
  try {
    return await _calendar.invokeMethod<T>(method, arguments);
  } on PlatformException catch (error) {
    throw ToolFailure(error.message ?? 'The phone could not do that just now.');
  }
}

final listEvents = AgentTool(
  name: 'list_events',
  description: "The events on the user's calendar for one day.",
  parameters: AgentTool.strings({'day': 'the day, as YYYY-MM-DD'}, required: ['day']),
  run: (arguments) async {
    if (!await _calendarAllowed()) return 'No access to the calendar.';
    final lines = await _ask<List<Object?>>('listEvents', {'day': '${arguments['day'] ?? ''}'});
    return (lines ?? const []).join('\n');
  },
);

// Android has no reminders app of its own for an app to add to, so there a reminder is an
// event in the user's calendar, with an alert when it's due.
final addReminder = AgentTool(
  name: 'add_reminder',
  description: Platform.isIOS
      ? "Adds a reminder to the user's Reminders app."
      : "Adds a reminder to the user's calendar, with an alert when it's due.",
  parameters: AgentTool.strings(
    {'title': 'what to remind them of', 'due': 'when, in ISO 8601 with the UTC offset'},
    required: ['title', 'due'],
  ),
  approval: const ToolApproval(title: 'Add this reminder?', reasonArgument: 'title'),
  run: (arguments) async {
    if (!await _calendarAllowed()) return 'No access to the calendar.';
    final due = DateTime.tryParse('${arguments['due'] ?? ''}');
    if (due == null) {
      throw const ToolFailure('The reminder needs a due time, such as 2026-10-16T18:00:00+02:00.');
    }
    await _ask<void>('addReminder', {
      'title': '${arguments['title'] ?? ''}',
      'due': due.millisecondsSinceEpoch,
    });
    return 'Added the reminder.';
  },
);

/// The tools the assistant may call, which run on this phone.
final assistantTools = [listEvents, addReminder];

/// Why a tool couldn't answer, in words the model passes on.
class ToolFailure implements Exception {
  const ToolFailure(this.message);

  final String message;

  @override
  String toString() => message;
}

import 'package:stream_agents_core/stream_agents_core.dart';

import 'agent_transport.dart';
import 'agents.dart';
import 'assistant_tools.dart';

Future<AgentTransport> openConversation(Session? session) async {
  if (session != null) {
    return AgentTransport.open(await agents.attach(session.id, tools: assistantTools));
  }
  final options = SessionOptions(tools: assistantTools, returnUrl: 'com.example.assistant:/connected');
  return AgentTransport.open(await agents.agent('assistant').chat(options));
}

import 'package:flutter/material.dart';
import 'package:stream_agents_core/stream_agents_core.dart';

import 'agents.dart';

class ConversationList extends StatefulWidget {
  const ConversationList({super.key, required this.onSelect});

  /// Called with the conversation picked, or null for a new one.
  final ValueChanged<Session?> onSelect;

  @override
  State<ConversationList> createState() => _ConversationListState();
}

class _ConversationListState extends State<ConversationList> {
  List<Session> _sessions = const [];
  String _search = '';

  /// How many reads there have been, so only the latest one's answer is shown.
  var _reads = 0;

  @override
  void initState() {
    super.initState();
    _load();
  }

  Future<void> _load() async {
    final read = ++_reads;
    final all = agents.agent('assistant').sessions;
    try {
      final page = _search.isEmpty ? await all.query() : await all.search(_search);
      if (!mounted || read != _reads) return;
      // A session that was only a call has no written conversation to open.
      setState(() => _sessions = [for (final session in page.items) if (session.conversationId.isNotEmpty) session]);
    } on Object {
      if (mounted && read == _reads) setState(() => _sessions = const []);
    }
  }

  @override
  Widget build(BuildContext context) => SafeArea(
    child: Column(
      children: [
        Padding(
          padding: const EdgeInsets.all(16),
          child: TextField(
            decoration: const InputDecoration(hintText: 'Search', prefixIcon: Icon(Icons.search)),
            onChanged: (text) {
              _search = text;
              _load();
            },
          ),
        ),
        ListTile(
          leading: const Icon(Icons.edit_outlined),
          title: const Text('New conversation'),
          onTap: () => widget.onSelect(null),
        ),
        Expanded(
          child: ListView(
            children: [
              for (final session in _sessions)
                ListTile(
                  title: Text(session.title.isEmpty ? 'New conversation' : session.title),
                  onTap: () => widget.onSelect(session),
                ),
            ],
          ),
        ),
      ],
    ),
  );
}

dependencies:
  # Opens a connector's sign-in, and brings the user back to the app.
  flutter_web_auth_2: ^5.1.0

import 'dart:async';

import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:flutter_web_auth_2/flutter_web_auth_2.dart';
import 'package:stream_chat_flutter_ai/stream_chat_flutter_ai.dart';
import 'package:stream_agents_core/stream_agents_core.dart';

import 'agent_transport.dart';
import 'conversation_list.dart';
import 'open_conversation.dart';
import 'voice_mode_screen.dart';

class AssistantHome extends StatefulWidget {
  const AssistantHome({super.key});

  @override
  State<AssistantHome> createState() => _AssistantHomeState();
}

class _AssistantHomeState extends State<AssistantHome> {
  final _scaffold = GlobalKey<ScaffoldState>();
  AgentTransport? _transport;

  /// How many conversations were opened, so only the latest one is shown.
  var _opened = 0;

  @override
  void initState() {
    super.initState();
    _open(null);
  }

  /// Opens [session], or a new conversation, and closes the one before.
  Future<void> _open(Session? session) async {
    final opening = ++_opened;
    final previous = _transport;
    setState(() => _transport = null);
    // What the previous conversation recorded is kept, to carry on later.
    if (previous != null) unawaited(_close(previous));
    try {
      final transport = await openConversation(session);
      if (!mounted || opening != _opened) {
        await _close(transport);
        return;
      }
      setState(() => _transport = transport);
    } on Object {
      // Not opened: the user can pick it again.
    }
  }

  // The conversation that showed it stops following it as it is disposed.
  static Future<void> _close(AgentTransport transport) => transport.session.close();

  @override
  void dispose() {
    final transport = _transport;
    if (transport != null) unawaited(_close(transport));
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    final transport = _transport;
    return Scaffold(
      key: _scaffold,
      appBar: AppBar(
        title: const Text('Assistant'),
        leading: IconButton(
          tooltip: 'Conversations',
          icon: const Icon(Icons.menu),
          onPressed: () => _scaffold.currentState?.openDrawer(),
        ),
        actions: [
          IconButton(
            tooltip: 'Talk',
            icon: const Icon(Icons.graphic_eq),
            onPressed: transport == null
                ? null
                : () => Navigator.of(context).push(
                    MaterialPageRoute<void>(
                      fullscreenDialog: true,
                      builder: (context) => VoiceModeScreen(session: transport.session),
                    ),
                  ),
          ),
        ],
      ),
      drawer: Drawer(
        child: ConversationList(
          onSelect: (session) {
            _scaffold.currentState?.closeDrawer();
            _open(session);
          },
        ),
      ),
      body: transport == null
          ? const Center(child: CircularProgressIndicator())
          : SafeArea(
              child: AssistantConversation(key: ValueKey(transport.session.id), transport: transport),
            ),
    );
  }
}

class AssistantConversation extends StatefulWidget {
  const AssistantConversation({super.key, required this.transport});

  final AgentTransport transport;

  @override
  State<AssistantConversation> createState() => _AssistantConversationState();
}

class _AssistantConversationState extends State<AssistantConversation> {
  late final _approver = widget.transport.approver(
    authorizer: AIToolAuthorizer(
      connect: (call, url) => _signIn(url),
      decline: (call) async => widget.transport.session.declineLogin(call.id),
    ),
  );
  late final _conversation = AIConversation(
    transport: widget.transport,
    approver: _approver,
    actions: AIConversationActions.rate,
  );

  /// Opens a connector's sign-in, and completes with whether the user connected.
  static Future<bool> _signIn(Uri url) async {
    try {
      final back = await FlutterWebAuth2.authenticate(url: '$url', callbackUrlScheme: 'com.example.assistant');
      return Uri.tryParse(back)?.queryParameters['status'] == 'connected';
    } on PlatformException catch (error) {
      // Closed before signing in.
      if (error.code == 'CANCELED') return false;
      rethrow;
    }
  }

  @override
  void dispose() {
    _conversation.dispose();
    _approver.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) => Column(
    children: [
      Expanded(child: AIConversationView.conversation(conversation: _conversation)),
      ChatComposer.conversation(conversation: _conversation),
    ],
  );
}

<activity
    android:name="com.linusu.flutter_web_auth_2.CallbackActivity"
    android:exported="true"
    android:taskAffinity="">
    <intent-filter android:label="flutter_web_auth_2">
        <action android:name="android.intent.action.VIEW" />
        <category android:name="android.intent.category.DEFAULT" />
        <category android:name="android.intent.category.BROWSABLE" />
        <data android:scheme="com.example.assistant" />
    </intent-filter>
</activity>

dependencies:
  # Shrinks the photos the composer hands over before they are sent.
  flutter_image_compress: ^2.5.1

@override
Future<void> send(AIConversationRequest<AgentMessage> request) async {
  await session.responses.create(request.text, images: await asImages(request.attachments));
}

/// The composer's photos as the session takes them: JPEG data URIs, shrunk from the
/// full-size files the picker returns so that its three fit in one request.
Future<List<AgentImage>> asImages(List<XFile> photos) async => [
  for (final photo in photos)
    if (await FlutterImageCompress.compressWithFile(
      photo.path,
      minWidth: 768,
      minHeight: 768,
      quality: 70,
    ) case final jpeg?)
      AgentImage('data:image/jpeg;base64,${base64Encode(jpeg)}'),
];

import 'dart:async';
import 'dart:io' show Platform;

import 'package:flutter/material.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:stream_agents_core/stream_agents_core.dart';
import 'package:stream_agents_rtc/stream_agents_rtc.dart';

import 'agents.dart';
import 'voice_call_screen.dart';

class VoiceModeScreen extends StatefulWidget {
  const VoiceModeScreen({super.key, required this.session});

  /// The typed conversation to switch to voice.
  final AgentSession session;

  @override
  State<VoiceModeScreen> createState() => _VoiceModeScreenState();
}

class _VoiceModeScreenState extends State<VoiceModeScreen> {
  VoiceSession? _voice;
  late final Future<void> _starting;
  var _hungUp = false;

  @override
  void initState() {
    super.initState();
    _starting = _start();
  }

  Future<void> _start() async {
    // iOS asks for the microphone by itself as the call is joined.
    if (Platform.isAndroid && !await Permission.microphone.request().isGranted) {
      _close();
      return;
    }
    try {
      // The agent joins the call with everything said so far.
      await widget.session.startVoice();
      _voice = await VoiceSession.attach(agents, widget.session.id);
      if (mounted) setState(() {});
    } on Object {
      _close();
    }
  }

  /// Leaves the call and takes the agent off it, and the conversation goes on in writing.
  Future<void> _leave() async {
    if (_hungUp) return;
    _hungUp = true;
    // Leaving a call you attached to keeps the session.
    await _voice?.leave();
    try {
      await widget.session.stopVoice();
    } on Object {
      // The call already ended.
    }
  }

  void _close() {
    if (mounted) Navigator.of(context).pop();
  }

  @override
  void dispose() {
    unawaited(_starting.whenComplete(_leave));
    super.dispose();
  }

  @override
  Widget build(BuildContext context) => switch (_voice) {
    final voice? => VoiceCallScreen(
      voice: voice,
      // The chat's session holds the tools, so their questions show over the call.
      toolSession: widget.session,
      onHangUp: () async {
        await _leave();
        _close();
      },
    ),
    null => const Scaffold(body: Center(child: CircularProgressIndicator())),
  };
}