Creating responses API

responses.create asks the agent something in writing. It works on any running session, a chat or a call, and the agent answers as though the text had been said. On a call the answer is spoken; in a chat it is written.

It returns as soon as the agent has started answering, with the id of the turn. Follow the answer on the session's events, or read the turn back from its items once it is done.

Send a text message

await session.responses.create("Where is order 1042?");

for await (const event of session.events()) {
  if (event.kind === "response_delta") process.stdout.write(event.text);
  if (event.kind === "responded") break;
}

val session = agents.agent("myagent").sessions.create(SessionOptions())
val responses = agents.sessions.responses(session.id)

val turn = responses.create("Where is order 1042?")
var status = turn.status
while (status == AgentResponse.Status.Running) {
    delay(500)
    status = responses.list().items.find { it.id == turn.id }?.status ?: status
}

val answer = responses.items(turn.id).items
    .filter { it.kind == ResponseItem.Kind.Answer }
    .joinToString("") { it.text }

let session = try await agents.chat(agent: "myagent")
await session.start()
let events = session.events()
_ = try await session.responses.create("Where is order 1042?")

for await event in events {
    if event.kind == .responseDelta { print(event.text, terminator: "") }
    if event.kind == .responded { break }
}

final session = await agents
    .agent('myagent')
    .sessions
    .create(const SessionOptions());
final responses = agents.sessions.responses(session.id);

final turn = await responses.create('Where is order 1042?');
var status = turn.status;
while (status == ResponseStatus.running) {
  await Future<void>.delayed(const Duration(milliseconds: 500));
  final turns = await responses.list();
  status = turns.items.firstWhere((each) => each.id == turn.id, orElse: () => turn).status;
}

await for (final item in responses.unwind(responseId: turn.id)) {
  if (item.kind == ItemKind.answer) print(item.text);
}

session = await agent.sessions.create()
turn = await session.responses.create("Where is order 1042?")

async for event in session.events():
    if event.kind == "response_delta":
        print(event.text, end="", flush=True)
    if event.kind == "responded":
        break

for item in await turn.items.all():
    if item.kind == "answer":
        print(item.text)

agent.chat do |session|
  turn = session.responses.create("Where is order 1042?")
  session.events do |event|
    print event.text if event.kind == "response_delta"
    break if event.kind == "responded"
  end

  turn.items.all.each { |item| puts item["text"] if item["kind"] == "answer" }
end

$session = $agent->chat();
$watch = $session->watch();
$turn = $session->responses->create('Where is order 1042?');

foreach ($watch as $event) {
    if ($event->kind === 'response_delta') {
        echo $event->text();
    }
    if ($event->kind === 'responded') {
        break;
    }
}

foreach ($turn->items->all() as $item) {
    if ($item->kind === 'answer') {
        echo $item->text, PHP_EOL;
    }
}

session, err := agent.Chat(ctx)
if err != nil {
	log.Fatal(err)
}
defer session.Close(ctx)

turn, err := session.Responses.Create(ctx, "Where is order 1042?")
if err != nil {
	log.Fatal(err)
}
for event := range session.Events() {
	if event.Kind == "response_delta" {
		fmt.Print(event.Text)
	}
	if event.Kind == "responded" {
		break
	}
}

items, err := turn.Items.All(ctx)
if err != nil {
	log.Fatal(err)
}
for _, item := range items {
	if item.Kind == acceleration.AgentResponseItemKindAnswer && item.Text != nil {
		fmt.Println(*item.Text)
	}
}

var session = await agent.ChatAsync();
var turn = await session.Responses.CreateAsync("Where is order 1042?");

await foreach (var happened in session.EventsAsync())
{
    if (happened.Kind == "response_delta")
    {
        Console.Write(happened.Text);
    }
    if (happened.Kind == "responded")
    {
        break;
    }
}

foreach (var item in await turn.Items.AllAsync())
{
    if (item.Kind == "answer")
    {
        Console.WriteLine(item.Text);
    }
}

use vision_agents::types::AgentResponseItemKind;

let session = agent.chat().await?;
let turn = session.responses.create("Where is order 1042?").await?;

while let Some(event) = session.next_event().await {
    match event.kind.as_str() {
        "response_delta" => print!("{}", event.text),
        "responded" => break,
        _ => {}
    }
}

for item in turn.items.all().await? {
    if item.kind == AgentResponseItemKind::Answer {
        println!("{}", item.text.unwrap_or_default());
    }
}

A browser or phone that opens its own session, as in the quickstart, watches it too, so events() works there. A session opened with watch: false is not watched: it reads the turn back with responses.items instead.

Send an image

Pass images next to the text. Each image is either an http(s) URL the backend can fetch, or a base64 data URI such as data:image/jpeg;base64,... for a picture you hold in memory or on disk. JPEG, PNG, WebP and GIF are accepted. detail is optional and one of auto, low or high; high costs more and reads small print better.

import { readFile } from "node:fs/promises";

const photo = (await readFile("receipt.jpg")).toString("base64");

await session.responses.create(
  "Which of these two receipts is for order 1042?",
  {
    images: [
      { url: "https://example.com/receipts/1041.png" },
      { url: `data:image/jpeg;base64,${photo}`, detail: "high" },
    ],
  },
);

val photo = Base64.encodeToString(File("receipt.jpg").readBytes(), Base64.NO_WRAP)

responses.create(
    "Which of these two receipts is for order 1042?",
    images = listOf(
        ImageSource("https://example.com/receipts/1041.png"),
        ImageSource("data:image/jpeg;base64,$photo", ImageSource.Detail.High),
    ),
)

// Two receipts the user photographed (UIImages), as JPEG data URIs. A session kept in
// Stream Chat, which is every text session a phone opens unless it's incognito, takes
// data URIs only: the router shows them on the user's message, and refuses a URL.
let receipts = [receipt1041, receipt1042].map { photo in
    "data:image/jpeg;base64," + photo.jpegData(compressionQuality: 0.8)!.base64EncodedString()
}

_ = try await session.responses.create(
    "Which of these two receipts is for order 1042?",
    images: [
        ImageSource(url: receipts[0]),
        ImageSource(url: receipts[1], detail: .high),
    ]
)

import 'dart:convert';
import 'dart:io';

final photo = base64Encode(await File('receipt.jpg').readAsBytes());

await responses.create('Which of these two receipts is for order 1042?', images: [
  const AgentImage('https://example.com/receipts/1041.png'),
  AgentImage('data:image/jpeg;base64,$photo', detail: 'high'),
]);

import base64
from pathlib import Path

photo = base64.b64encode(Path("receipt.jpg").read_bytes()).decode()

await session.responses.create(
    "Which of these two receipts is for order 1042?",
    images=[
        stream.ImageSource(url="https://example.com/receipts/1041.png"),
        stream.ImageSource(
            url=f"data:image/jpeg;base64,{photo}",
            detail=stream.ImageSourceDetail.HIGH,
        ),
    ],
)

require "base64"

photo = Base64.strict_encode64(File.binread("receipt.jpg"))

session.responses.create("Which of these two receipts is for order 1042?", images: [
  { url: "https://example.com/receipts/1041.png" },
  { url: "data:image/jpeg;base64,#{photo}", detail: "high" }
])

use GetStream\VisionAgents\Generated\ImageSource;

$photo = base64_encode(file_get_contents('receipt.jpg'));

$session->responses->create('Which of these two receipts is for order 1042?', [
    new ImageSource('https://example.com/receipts/1041.png'),
    new ImageSource("data:image/jpeg;base64,{$photo}", 'high'),
]);

data, err := os.ReadFile("receipt.jpg")
if err != nil {
	log.Fatal(err)
}

_, err = session.Responses.Create(ctx, "Which of these two receipts is for order 1042?",
	client.Image{URL: "https://example.com/receipts/1041.png"},
	client.Image{URL: "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(data), Detail: acceleration.ImageSourceDetailHigh},
)

using GetStream.VisionAgents.Models;

var photo = Convert.ToBase64String(await File.ReadAllBytesAsync("receipt.jpg"));

await session.Responses.CreateAsync("Which of these two receipts is for order 1042?",
[
    new ImageSource { Url = "https://example.com/receipts/1041.png" },
    new ImageSource { Url = $"data:image/jpeg;base64,{photo}", Detail = "high" },
]);

use base64::{Engine, engine::general_purpose::STANDARD};
use vision_agents::types::{CreateResponseRequest, ImageSource, ImageSourceDetail};

let photo = STANDARD.encode(std::fs::read("receipt.jpg")?);

session
    .responses
    .create_with(&CreateResponseRequest {
        text: "Which of these two receipts is for order 1042?".into(),
        images: Some(vec![
            ImageSource { url: "https://example.com/receipts/1041.png".into(), detail: None },
            ImageSource {
                url: format!("data:image/jpeg;base64,{photo}"),
                detail: Some(ImageSourceDetail::High),
            },
        ]),
        ..Default::default()
    })
    .await?;

The pictures go to the agent's subagent, which needs a model that can see, such as the vlm route. The conversation model gets the question and, once the subagent has looked, its findings; it never sees the pixels. So the turn takes a little longer than a text-only one. In a conversation kept in Stream Chat, the router also uploads the pictures to the channel and shows them on the person's message.

Send a video

Pass recorded clips next to the text, such as a customer's video of a parcel that arrived damaged. A response takes at most two clips. Each clip is either an http(s) URL the backend can fetch, or a base64 data URI such as data:video/mp4;base64,... for a file on disk. MP4, MOV, 3GP, MKV and WebM are accepted, up to 50 MB each. The agent sees 8 frames spread evenly across each clip; set max_frames up to 32 for a longer one.

$STREAM_ACCELERATION_URL is your router's URL, $SERVER_TOKEN is a server token, and $SESSION_ID is the ID of the session that answers.

clip, err := client.ClipFile("unboxing.mp4")
if err != nil {
	log.Fatal(err)
}

_, err = session.Responses.Create(ctx, "Which of these clips shows order 1042 arriving damaged?",
	client.Clip{URL: "https://example.com/returns/1042.mp4", MaxFrames: 16},
	clip,
)

use base64::{Engine, engine::general_purpose::STANDARD};
use vision_agents::types::{CreateResponseRequest, VideoSource};

let clip = STANDARD.encode(std::fs::read("unboxing.mp4")?);

session
    .responses
    .create_with(&CreateResponseRequest {
        text: "Which of these clips shows order 1042 arriving damaged?".into(),
        videos: Some(vec![
            VideoSource { url: "https://example.com/returns/1042.mp4".into(), max_frames: Some(16) },
            VideoSource { url: format!("data:video/mp4;base64,{clip}"), max_frames: None },
        ]),
        ..Default::default()
    })
    .await?;

curl -X POST "$STREAM_ACCELERATION_URL/v1/agents/sessions/$SESSION_ID/responses" \
  -H "api_key: $STREAM_API_KEY" -H "stream-auth-type: jwt" \
  -H "Authorization: Bearer $SERVER_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{"text": "Which of these clips shows order 1042 arriving damaged?", "videos": [{"url": "https://example.com/returns/1042.mp4", "max_frames": 16}]}'

Info:

The Go and Rust SDKs take videos. From the others, send videos in the body of POST /v1/agents/sessions/{id}/responses, as in the cURL tab.

The backend samples each clip into frames, each with the moment it was taken. It shows the frames to the agent's subagent when the agent has one, and otherwise to its conversation model. That model needs to see images, such as gemini/gemini-3.8-flash.

Send a voice note

Pass a recording of what the person asked, such as a voice message they held the mic button for, as audio. The router transcribes it before the turn starts, and the agent answers what was said, after any text sent with it, so the text may be empty. A response takes one voice note. It is either an http(s) URL the backend can fetch, or a base64 data URI such as data:audio/mp4;base64,... for a recording in memory or on disk, at most 8 MiB either way. The usual recording formats are accepted, such as M4A, WebM, Ogg, MP3 and WAV. The router reads the kind from the bytes, so a WebM or MP4 recording counts as audio even when it is labelled as video.

This is the body of POST /v1/agents/sessions/{id}/responses with a voice note as a data URI:

{
  "text": "",
  "audio": {
    "url": "data:audio/mp4;base64,AAAAIGZ0eXBNNEEgAAAAAE00QSBtcDQyaXNvbQ..."
  },
  "request_id": "9b2f6c1e-4d7a-4b8e-a5f3-2c1d0e9f8a7b"
}

text is required, and may be the empty string when the voice note carries the whole question.
audio is one object with a single url, not an array.
request_id is one fresh id per question, of letters, digits, - and _. A conversation kept in Stream Chat needs it, and with it the router uploads the voice note to the channel and shows it on the person's message. A session not kept in Stream Chat ignores it. The SDKs send one for you.

note, err := os.ReadFile("question.m4a")
if err != nil {
	log.Fatal(err)
}

_, err = session.Responses.Create(ctx, "",
	client.VoiceNote{URL: "data:audio/mp4;base64," + base64.StdEncoding.EncodeToString(note)},
)

use base64::{Engine, engine::general_purpose::STANDARD};
use vision_agents::types::{AudioSource, CreateResponseRequest};

let note = STANDARD.encode(std::fs::read("question.m4a")?);

session
    .responses
    .create_with(&CreateResponseRequest {
        text: String::new(),
        audio: Some(AudioSource { url: format!("data:audio/mp4;base64,{note}") }),
        ..Default::default()
    })
    .await?;

printf '{"text": "", "audio": {"url": "data:audio/mp4;base64,%s"}, "request_id": "%s"}' \
  "$(base64 < question.m4a | tr -d '\n')" "$(uuidgen)" > voice-note.json

curl -X POST "$STREAM_ACCELERATION_URL/v1/agents/sessions/$SESSION_ID/responses" \
  -H "api_key: $STREAM_API_KEY" -H "stream-auth-type: jwt" \
  -H "Authorization: Bearer $SERVER_TOKEN" \
  -H "Content-Type: application/json" \
  --data-binary @voice-note.json

The cURL tab writes the body to a file first, because a recording of a minute or more is too long to pass on the command line. $STREAM_ACCELERATION_URL, $SERVER_TOKEN and $SESSION_ID are as in Send a video.

Info:

The Go and Rust SDKs take a voice note from tag 0.2.0 of Vision-Agents. The JavaScript SDK's responses.create doesn't take one yet in @stream-io/calypso 0.3.1. From JavaScript and the other SDKs, send audio in the body of POST /v1/agents/sessions/{id}/responses, as in the cURL tab.

The note is transcribed in the agent's languages, and the question the agent answers is the typed text, then what was said. In a conversation kept in Stream Chat, the person's message carries the transcript and shows the voice note. A note that isn't audio, is over 8 MiB or has no words in it is a 400, and no turn starts. So is a voice note to an agent in a region where the deployment transcribes nothing.

Send what the person did in a reply's UI

A reply can be UI the agent writes, such as a plan with a map and a form to change it. When the person does something in it that's for the agent, such as tapping a place or submitting the form, ask with their next message, in their words, and pass what they did as uiAction. It names the reply it came from (message_id, the id of the reply's Stream Chat message), the UI's format and the action (name), with what it carries (payload), in at most 8 KB as JSON.

The client SDKs make it for you: in React, aiUIActionJSON(action) from stream-chat-react/ai-components turns the AIUIAction the person's tap made into it, and on iOS, StreamChatAI's AIUIAction has it as json, as does the AIUIAction of stream-chat-android-ai-compose on Android. Use the action's label as the text.

import { aiUIActionJSON } from "stream-chat-react/ai-components";

// In the AIUIHost's send, with the AIUIAction of the person's tap.
await session.responses.create(action.label, {
  uiAction: aiUIActionJSON(action),
});

// In a coroutine launched from the AIUIHost's send, with the AIUIAction of the person's tap.
session.responses.create(action.label, uiAction = action.json)

// In the AIUIHost's send, with the AIUIAction of the person's tap.
_ = try await session.responses.create(action.label, uiAction: action.json)

// replyID is the Stream Chat message id of the reply whose UI the person used.
_, err := session.Responses.Create(ctx, "Make the plan more relaxed",
	client.UIAction{
		"message_id": replyID,
		"format":     "a2ui",
		"name":       "update_plan",
		"payload":    map[string]any{"pace": "relaxed"},
	},
)

// Needs serde_json in your Cargo.toml: `cargo add serde_json`.
use serde_json::json;
use vision_agents::types::CreateResponseRequest;

// reply_id is the Stream Chat message id of the reply whose UI the person used.
let action = json!({
    "message_id": reply_id,
    "format": "a2ui",
    "name": "update_plan",
    "payload": { "pace": "relaxed" },
});

session
    .responses
    .create_with(&CreateResponseRequest {
        text: "Make the plan more relaxed".into(),
        ui_action: action.as_object().cloned().unwrap_or_default(),
        ..Default::default()
    })
    .await?;

Info:

The JavaScript SDK takes uiAction from @stream-io/calypso 0.3.0. The Swift, Go and Rust SDKs take it on the accelerate branch of Vision-Agents, from tag 0.2.0, and the Kotlin SDK from io.getstream:calypso-core:0.1.0-202611041811-SNAPSHOT. From the others, send ui_action in the body of POST /v1/agents/sessions/{id}/responses.

The model is told what the person did after their words. In a conversation kept in Stream Chat, the person's message carries the action as custom.ai_ui_action, and the model is also told the opening words of the reply it was in, so the agent knows which reply they acted on when it showed the same UI more than once. An action without a message_id, format or name, or over 8 KB, is a 400, and the question isn't asked. The JavaScript, Go and Rust SDKs leave out an empty action, so the question is asked without one.

Set the length and reasoning

Cap one answer with max_output_tokens, or change how long the model thinks before it with reasoning_effort. Both apply to this answer alone, including the replies that finish it after its tools, and the session's own settings come back for the next question.

await session.responses.create("Plan my week around these meetings", {
  maxOutputTokens: 800,
  reasoningEffort: "high",
});

session.responses.create(
    "Plan my week around these meetings",
    maxOutputTokens = 800,
    reasoningEffort = "high",
)

_ = try await session.responses.create(
    "Plan my week around these meetings",
    maxOutputTokens: 800,
    reasoningEffort: "high"
)

_, err := session.Responses.Create(ctx, "Plan my week around these meetings",
	client.MaxOutputTokens(800),
	client.ReasoningEffort("high"),
)

Info:

The JavaScript, Kotlin, Dart, Swift and Go SDKs take these. From the others, send max_output_tokens and reasoning_effort in the body of POST /v1/agents/sessions/{id}/responses.

max_output_tokens counts reasoning too, so leave room for it when you raise the effort. The efforts depend on the model, such as none, low, medium, high or max. One the session's model doesn't take is a 400, and the question isn't asked.

Reading Responses and History

A session keeps a record of each turn, called a response, and what happened inside it, called its items. These are read over HTTP, so they work during the conversation, after it has ended, and for a session your process is not holding.

Method	What it returns
`responses.create(text)`	Asks something and returns the turn as soon as it has started. See Send a text message
`responses.list()`	The turns, oldest first, a page at a time as `{ items, has_more, next_cursor }`: what was asked, the status (`running`, `completed`, ...) and how long it took
`responses.items`	What happened, turn by turn: `said`, `thought`, `answer`, `blocked`, `error` and tool calls
`responses.rewind(response)`	Carries on from the end of that response, as though nothing after it was said

const { items: turns } = await session.responses.list({ limit: 50 });

for (const item of await session.responses.items.all()) {
  console.log(item.kind, item.text);
}

const old = agent.sessions.responses("96f20176bfc7b3c051b759abf0bcae94");
const lastWeek = (await old.list()).items;

val turns = responses.list(limit = 50).items

responses.unwind().collect { println("${it.kind} ${it.text}") }

val old = agent.sessions.responses("96f20176bfc7b3c051b759abf0bcae94")
val lastWeek = old.list().items

let turns = try await session.responses.list(limit: 50).items

for try await item in session.responses.unwind() {
    print(item.kind, item.text)
}

let old = agent.sessions.responses("96f20176bfc7b3c051b759abf0bcae94")
let lastWeek = try await old.list().items

final turns = (await session.responses.list(limit: 50)).items;

await for (final item in session.responses.unwind()) {
  print('${item.kind} ${item.text}');
}

final old = agents.sessions.responses('96f20176bfc7b3c051b759abf0bcae94');
final lastWeek = (await old.list()).items;

turns = (await session.responses.list(limit=50)).items

for item in await session.responses.items.all():
    print(item.kind, item.text)

old = agent.sessions.responses("96f20176bfc7b3c051b759abf0bcae94")
last_week = (await old.list()).items

turns = session.responses.list(limit: 50)["items"]

session.responses.items.all.each do |item|
  puts "#{item["kind"]} #{item["text"]}"
end

old = agent.sessions.responses("96f20176bfc7b3c051b759abf0bcae94")
last_week = old.list["items"]

$turns = $session->responses->list(limit: 50)->items;

foreach ($session->responses->items->all() as $item) {
    echo "{$item->kind} {$item->text}\n";
}

$old = $agent->sessions->responses('96f20176bfc7b3c051b759abf0bcae94');
$lastWeek = $old->list()->items;

page, err := session.Responses.List(ctx, 50, "")
if err != nil {
	log.Fatal(err)
}
turns := page.Items

items, err := session.Responses.Items.All(ctx)
if err != nil {
	log.Fatal(err)
}
for _, item := range items {
	fmt.Println(item.Kind, item.Text)
}
old := agent.Sessions.Responses("96f20176bfc7b3c051b759abf0bcae94")
lastWeek, err := old.List(ctx, 0, "")

var turns = (await session.Responses.ListAsync(limit: 50)).Items;

foreach (var item in await session.Responses.Items.AllAsync())
{
    Console.WriteLine($"{item.Kind} {item.Text}");
}
var old = agent.Sessions.Responses("96f20176bfc7b3c051b759abf0bcae94");
var lastWeek = (await old.ListAsync()).Items;

let turns = session.responses.list(Some(50), None).await?.items;

for item in session.responses.items.all().await? {
    println!("{} {}", item.kind, item.text.unwrap_or_default());
}

let old = agent.sessions.responses("96f20176bfc7b3c051b759abf0bcae94");
let last_week = old.list(None, None).await?.items;

Items leave out the answer's fragments, which are on the events; they are the shape of the conversation rather than its keystrokes. An incognito session has no items, because it records nothing.

Limits

A conversation a user opens in their own name, from their device, takes text only for now. Send pictures to a session your backend opened.
An image, video or voice note URL must be public: the backend fetches it without credentials, and refuses one that resolves to a private or loopback address.
A video URL points at one file. A stream playlist, such as HLS (.m3u8), is refused.