Android AI Voice Assistant Tutorial

[versions]
agp = "8.8.1"
kotlin = "2.0.0"
coreKtx = "1.15.0"
lifecycleRuntimeKtx = "2.8.7"
activityCompose = "1.10.0"
composeBom = "2024.04.01"
stream = "<latest_version>"
retrofit = "2.11.0"
accompanist = "0.33.1-alpha"
kotlinxSerializationJson="1.6.0"

[libraries]
androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" }
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycleRuntimeKtx" }
androidx-lifecycle-runtime-compose = {group = "androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycleRuntimeKtx"}
androidx-lifecycle-viewmodel-compose = {group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycleRuntimeKtx"}

androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" }
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityCompose" }
androidx-compose-bom = { group = "androidx.compose", name = "compose-bom", version.ref = "composeBom" }

androidx-ui = { group = "androidx.compose.ui", name = "ui" }
androidx-ui-graphics = { group = "androidx.compose.ui", name = "ui-graphics" }
androidx-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview" }
androidx-material3 = { group = "androidx.compose.material3", name = "material3" }

getstream-video-android-ui-core = { group = "io.getstream", name = "stream-video-android-ui-core", version.ref = "stream"}
retrofit = {group = "com.squareup.retrofit2", name = "retrofit", version.ref = "retrofit" }
retrofit-gson = {group = "com.squareup.retrofit2", name = "converter-gson", version.ref = "retrofit" }
accompanist-permissions = {group = "com.google.accompanist", name = "accompanist-permissions", version.ref = "accompanist" }
kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name="kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }

package io.getstream.ai.audiodemo

import io.getstream.ai.audiodemo.Credentials
import okhttp3.ResponseBody
import retrofit2.Retrofit
import retrofit2.converter.gson.GsonConverterFactory
import retrofit2.http.GET
import retrofit2.http.POST
import retrofit2.http.Path

interface ApiService {
    @GET("credentials")
    suspend fun getCredentials(): Credentials

    @POST("{callType}/{channelId}/connect")
    suspend fun connectAI(
        @Path("callType", encoded = true) callType: String,
        @Path("channelId", encoded = true) channelId: String): ResponseBody
}

object RetrofitInstance {
    private val BASE_URL = "http://10.0.2.2:3000/"

    val api: ApiService by lazy {
        Retrofit.Builder()
            .baseUrl(BASE_URL)
            .addConverterFactory(GsonConverterFactory.create())
            .build()
            .create(ApiService::class.java)
    }
}

class MainViewModel(val app: Application) : AndroidViewModel(app) {

    var callUiState = MutableStateFlow(CallUiState.IDLE)
    val credentials = MutableStateFlow<DataLayerResponse<Credentials>>(DataLayerResponse.Initial("Initial State"))
    var client: StreamVideo? = null
    var call: Call? = null

    fun initCredentials() {
        viewModelScope.launch {
            try {
                credentials.emit(DataLayerResponse.Initial("Getting Credentials"))
                val response = RetrofitInstance.api.getCredentials()
                createStreamClient(response)
                credentials.emit(DataLayerResponse.Success(response))
            } catch (ex: Exception) {
                credentials.emit(DataLayerResponse.Error(ex.message))
            }
        }
    }

    private fun createStreamClient(credentials: Credentials) {
        val userId = credentials.userId

        val user = User(
            id = userId, 
            name = "Tutorial", 
            image = "https://bit.ly/2TIt8NR",
        )

        client = StreamVideoBuilder(
            context = app.applicationContext,
            apiKey = credentials.apiKey,
            geo = GEO.GlobalEdgeNetwork,
            user = user,
            token = credentials.token,
        ).build()
    }

    fun joinCall() {

        client?.let { client->
            val credentials = (credentials.value as DataLayerResponse.Success).data
            call = client.call(credentials.callType, credentials.callId)
            connectAi(call!!, credentials.callId, credentials.callType)
        }

    }

    private fun connectAi(call: Call, channelId: String, callType: String) {
        viewModelScope.launch {
            try {
                callUiState.emit(CallUiState.JOINING)
                val encodedChannelId = URLEncoder.encode(channelId, StandardCharsets.UTF_8.toString())
                val encodedCallType = URLEncoder.encode(callType, StandardCharsets.UTF_8.toString())
                val response = RetrofitInstance.api.connectAI(encodedCallType, encodedChannelId)
                call.join(create = true)
                callUiState.emit(CallUiState.ACTIVE)
            }catch (ex: Exception) {
                ex.printStackTrace()
                callUiState.emit(CallUiState.ERROR)
            }

        }
    }

    fun disconnect(){
        viewModelScope.launch {
            call?.end()
            callUiState.emit(CallUiState.IDLE)
        }
    }
}

sealed class DataLayerResponse<T> {
    class Success<T>(val data: T): DataLayerResponse<T>()
    class Error<T>(val message: String?): DataLayerResponse<T>()
    class Initial<T>(val message: String): DataLayerResponse<T>()
}

package io.getstream.ai.audiodemo

@OptIn(ExperimentalPermissionsApi::class)
@Composable
fun MicrophonePermissionScreen(onPermissionGranted: () -> Unit) {
    val microphonePermissionState =
        rememberPermissionState(android.Manifest.permission.RECORD_AUDIO)

    LaunchedEffect(Unit) {
        if (!microphonePermissionState.status.isGranted) {
            microphonePermissionState.launchPermissionRequest()
        }
    }

    LaunchedEffect(microphonePermissionState.status.isGranted) {
        if (microphonePermissionState.status.isGranted) {
            onPermissionGranted()
        }
    }

    Column(
        modifier = Modifier
            .fillMaxSize()
            .padding(16.dp),
        horizontalAlignment = Alignment.CenterHorizontally,
        verticalArrangement = Arrangement.Center
    ) {
        when {
            microphonePermissionState.status.isGranted -> {}
            microphonePermissionState.status.shouldShowRationale -> {
                Column(horizontalAlignment = Alignment.CenterHorizontally) {
                    Text(
                        text = "Microphone access is required to proceed.",
                        fontSize = 18.sp,
                        color = Color.White
                    )
                    Spacer(modifier = Modifier.height(8.dp))
                    Button(onClick = { microphonePermissionState.launchPermissionRequest() }) {
                        Text(text = "Grant Permission", color = Color.White)
                    }
                }
            }

            else -> {
                Column(horizontalAlignment = Alignment.CenterHorizontally) {
                    Text(
                        text = "Microphone permission is needed to proceed.",
                        fontSize = 18.sp,
                        color = Color.White
                    )
                    Spacer(modifier = Modifier.height(8.dp))
                    Button(onClick = { microphonePermissionState.launchPermissionRequest() }) {
                        Text(text = "Request Permission", color = Color.White)
                    }
                }
            }
        }
    }
}

package io.getstream.ai.audiodemo

data class Credentials(
     val apiKey: String,
     val token: String,
     val callId: String,
     val callType: String,
     val userId: String
)

enum class CallUiState {
    IDLE, JOINING, ACTIVE, ERROR
}

fun initCredentials() {
    viewModelScope.launch {
        try {
            credentials.emit(DataLayerResponse.Initial("Getting Credentials"))
            val response = RetrofitInstance.api.getCredentials()
            createStreamClient(response)
            credentials.emit(DataLayerResponse.Success(response))
        } catch (ex: Exception) {
            credentials.emit(DataLayerResponse.Error(ex.message))
        }
  }

private fun createStreamClient(credentials: Credentials) {
      val userId = credentials.userId
      
      val user = User(
          id = userId, 
          name = "Tutorial", 
          image = "https://bit.ly/2TIt8NR",
      )

      client = StreamVideoBuilder(
          context = app.applicationContext,
          apiKey = credentials.apiKey,
          geo = GEO.GlobalEdgeNetwork,
          user = user,
          token = credentials.token,
      ).build()
}

class MainActivity : ComponentActivity() {

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        enableEdgeToEdge()

        setContent {
            var isPermissionGranted by remember { mutableStateOf(false) }
            if (isPermissionGranted) {
                Body()
            } else {
                MicrophonePermissionScreen(onPermissionGranted = {
                    isPermissionGranted = true
                })
            }
        }
    }
}

@Composable
fun Body() {
    val viewModel: MainViewModel = viewModel(key = "MainViewModel")
    LaunchedEffect(Unit) {
        viewModel.initCredentials()
    }
    val creds = viewModel.credentials.collectAsStateWithLifecycle()
    when (creds.value) {
        is DataLayerResponse.Initial -> {
            val message = (creds.value as DataLayerResponse.Initial<Credentials>).message
            Box(
                Modifier
                    .fillMaxSize()
                    .background(Color.Black),
                contentAlignment = Alignment.Center
            ) {
                Text(message, color = Color.White)
            }
        }

        is DataLayerResponse.Success -> {
            Box(
                modifier = Modifier
                    .fillMaxSize()
                    .background(Color.Black)
            ) {
                AiContentUi()
            }

        }

        is DataLayerResponse.Error -> {
            Box(
                Modifier
                    .fillMaxSize()
                    .background(Color.Black),
                contentAlignment = Alignment.Center
            ) {
                Text("Getting Credentials Failed", color = Color.White)
            }
        }
    }
}

@Composable
fun AiContentUi() {
    val viewModel: MainViewModel = viewModel(key = "MainViewModel")
    val callUiState by viewModel.callUiState.collectAsStateWithLifecycle()

    when (callUiState) {
        CallUiState.IDLE -> {
            Box(Modifier.fillMaxSize(), contentAlignment = Alignment.Center) {
                Button(colors = ButtonDefaults.buttonColors()
                    .copy(contentColor = Color.White, containerColor = Color.Black),
                    onClick = {
                        viewModel.joinCall()
                    }) {
                    Text("Click to talk to Ai", color = Color.White)
                }
            }

        }

        CallUiState.JOINING -> {
            Box(Modifier.fillMaxSize(), contentAlignment = Alignment.Center) {
                Row(
                    horizontalArrangement = Arrangement.Center,
                    verticalAlignment = Alignment.CenterVertically
                ) {
                    Text(
                        "Waiting for AI Agent to Join...",
                        color = Color.White,
                        modifier = Modifier.padding(bottom = 12.dp)
                    )
                    Spacer(Modifier.width(8.dp))
                    CircularProgressIndicator(
                        modifier = Modifier.width(24.dp),
                        color = MaterialTheme.colorScheme.secondary,
                        trackColor = MaterialTheme.colorScheme.surfaceVariant,
                    )
                }
            }
        }

        CallUiState.ACTIVE -> {
            Box(Modifier.fillMaxSize()) {
                AISpeakingView(viewModel.call?.state!!)
                CallEndButton(modifier = Modifier
                    .align(Alignment.BottomEnd)
                    .padding(12.dp), onClick = {
                    viewModel.disconnect()
                })
            }
        }

        CallUiState.ERROR -> {
            Box(Modifier.fillMaxSize(), contentAlignment = Alignment.Center) {
                Column(horizontalAlignment = Alignment.CenterHorizontally) {
                    Text("Something went wrong, please re-try", color = Color.White)
                    Button(colors = ButtonDefaults.buttonColors()
                        .copy(contentColor = Color.White, containerColor = Color.Black),
                        onClick = {
                            viewModel.joinCall()
                        }) {
                        Text("Connect to Ai Agent", color = Color.White)
                    }
                }
            }
        }
    }
}

@Composable
fun CallEndButton(modifier: Modifier, onClick: () -> Unit) {
    IconButton(
        onClick = onClick,
        modifier = modifier
            .size(56.dp)
            .clip(CircleShape)
            .background(Color.Red)
    ) {
        Icon(
            painter = painterResource(id = R.drawable.call_end),
            contentDescription = "End call",
            tint = Color.White
        )
    }
}

enum class CallUiState {
    IDLE, JOINING, ACTIVE, ERROR
}

@Composable
fun AISpeakingView(callState: CallState) {
	//TODO
}

enum class SpeakerState {
    AI_SPEAKING,
    USER_SPEAKING,
    IDLE
}

fun SpeakerState.gradientColors(): List<Color> {
    return when (this) {
        SpeakerState.USER_SPEAKING -> listOf(
            Color.Red,
            Color.Red.copy(alpha = 0f)
        )
        else -> listOf(
            Color(0f, 0.976f, 1f),
            Color(0f, 0.227f, 1f, 0f)
        )
    }
}

@Composable
fun AISpeakingView(callState: CallState) {
    val agentId = "lucy"

    var amplitude by remember { mutableFloatStateOf(0f) }
    var audioLevels by remember { mutableStateOf(listOf<Float>()) }
    var speakerState by remember { mutableStateOf(SpeakerState.IDLE) }

    GlowView(amplitude, speakerState)

    LaunchedEffect(Unit) {

        callState.activeSpeakers.collectLatest { speakers->
            speakers.forEach { speaker->
                speaker.audioLevels.collectLatest { audioLevel->
                    if(speaker.userId.value.contains("lucy")) {
                        if(speakerState!=SpeakerState.AI_SPEAKING) {
                            speakerState = SpeakerState.AI_SPEAKING
                        }
                    } else {
                        if(speakerState!=SpeakerState.USER_SPEAKING) {
                            speakerState = SpeakerState.USER_SPEAKING
                        }
                    }
                    audioLevels = audioLevel
                    amplitude =
                        computeSingleAmplitude(audioLevels) * getRandomFloatInRange(1f, 2f)
                }
            }
        }
    }

    LaunchedEffect(Unit) {

        callState.activeSpeakers.collectLatest { speaker->
            val aiSpeaker = speaker.find { it.userId.value.contains(agentId) }

            // Find the local user speaking
            val localSpeaker = speaker.find { it.userId.value == callState.me.value!!.userId.value }
            if(aiSpeaker == null && localSpeaker == null){
                speakerState = SpeakerState.IDLE
                audioLevels = emptyList()
                amplitude = 0f
            }
        }
    }
}

fun computeSingleAmplitude(levels: List<Float>): Float {
    val normalized = normalizePeak(levels)
    if (normalized.isEmpty()) return 0f

    return normalized.average().toFloat()
}

fun normalizePeak(levels: List<Float>): List<Float> {
    val maxLevel = levels.maxOfOrNull { abs(it) } ?: return levels
    return if (maxLevel > 0) levels.map { it / maxLevel } else levels
}

@Composable
fun GlowView(
    amplitude: Float,
    speakerState: SpeakerState = SpeakerState.AI_SPEAKING
) {
    // Animated amplitude for smooth transitions
    val animatedAmplitude = remember { Animatable(0f) }

    // Update animated amplitude when input amplitude changes
    LaunchedEffect(amplitude) {
        animatedAmplitude.animateTo(
            targetValue = amplitude,
            animationSpec = tween(600, easing = EaseInOut)
        )
    }

    // Continuous time value for wave animation
    var time by remember { mutableStateOf(0f) }

    // Rotation animation
    val infiniteRotation = rememberInfiniteTransition()
    val rotationAngle by infiniteRotation.animateFloat(
        initialValue = 0f,
        targetValue = 360f,
        animationSpec = infiniteRepeatable(
            animation = tween(10000, easing = LinearEasing),
            repeatMode = RepeatMode.Restart
        )
    )

    // Time progression for wave effect
    LaunchedEffect(Unit) {
        while (true) {
            time = (time + 0.005f) % 1.0f
            delay(16L)
        }
    }

    val gradientColors = speakerState.gradientColors()

    Box(
        modifier = Modifier
            .fillMaxSize()
            .background(Color.Black)
    ) {
        Canvas(modifier = Modifier.fillMaxSize()) {
            val center = Offset(size.width / 2, size.height / 2)

            // Outer Layer
            drawGlowLayer(
                center = center,
                baseRadiusMin = size.minDimension * 0.30f,
                baseRadiusMax = size.minDimension * 0.50f,
                blurRadius = 60f,
                baseOpacity = 0.35f,
                scaleRange = 0.3f,
                waveRangeMin = 0.2f,
                waveRangeMax = 0.02f,
                time = time,
                amplitude = animatedAmplitude.value,
                rotationAngle = rotationAngle,
                gradientColors = gradientColors
            )

            // Middle Layer
            drawGlowLayer(
                center = center,
                baseRadiusMin = size.minDimension * 0.20f,
                baseRadiusMax = size.minDimension * 0.30f,
                blurRadius = 40f,
                baseOpacity = 0.55f,
                scaleRange = 0.3f,
                waveRangeMin = 0.15f,
                waveRangeMax = 0.03f,
                time = time,
                amplitude = animatedAmplitude.value,
                rotationAngle = rotationAngle,
                gradientColors = gradientColors
            )

            // Inner Core Layer
            drawGlowLayer(
                center = center,
                baseRadiusMin = size.minDimension * 0.10f,
                baseRadiusMax = size.minDimension * 0.20f,
                blurRadius = 20f,
                baseOpacity = 0.9f,
                scaleRange = 0.5f,
                waveRangeMin = 0.35f,
                waveRangeMax = 0.05f,
                time = time,
                amplitude = animatedAmplitude.value,
                rotationAngle = rotationAngle,
                gradientColors = gradientColors
            )
        }
    }
}

private fun androidx.compose.ui.graphics.drawscope.DrawScope.drawGlowLayer(
    center: Offset,
    baseRadiusMin: Float,
    baseRadiusMax: Float,
    blurRadius: Float,
    baseOpacity: Float,
    scaleRange: Float,
    waveRangeMin: Float,
    waveRangeMax: Float,
    time: Float,
    amplitude: Float,
    rotationAngle: Float,
    gradientColors: List<Color>
) {
    // Calculate the actual radius based on amplitude
    val baseRadius = lerp(baseRadiusMin, baseRadiusMax, amplitude)

    // Calculate wave range (inverse relationship to amplitude)
    val waveRange = lerp(waveRangeMax, waveRangeMin, 1 - amplitude)

    // Calculate the scale factors
    val shapeWaveSin = sin(2 * PI * time).toFloat()
    val shapeWaveCos = cos(2 * PI * time).toFloat()

    // Scale from amplitude
    val amplitudeScale = 1.0f + scaleRange * amplitude

    // Final x/y scale = amplitude scale + wave
    val xScale = (amplitudeScale + waveRange * shapeWaveSin)
    val yScale = (amplitudeScale + waveRange * shapeWaveCos)

    // Draw the oval with gradient
    drawIntoCanvas { canvas ->
        val paint = androidx.compose.ui.graphics.Paint().asFrameworkPaint().apply {
            shader = RadialGradient(
                center.x, center.y, baseRadius,
                intArrayOf(
                    gradientColors[0].copy(alpha = 0.9f).toArgb(),
                    gradientColors[1].toArgb()
                ),
                floatArrayOf(0f, 1f),
                Shader.TileMode.CLAMP
            )
            alpha = (baseOpacity * 255).toInt()
        }

        // Apply blur
        paint.maskFilter = BlurMaskFilter(blurRadius, BlurMaskFilter.Blur.NORMAL)

        // Save the current state, rotate, draw, and restore
        canvas.nativeCanvas.save()
        canvas.nativeCanvas.rotate(rotationAngle, center.x, center.y)

        // Draw the oval with calculated dimensions
        canvas.nativeCanvas.drawOval(
            center.x - baseRadius * xScale,
            center.y - baseRadius * yScale,
            center.x + baseRadius * xScale,
            center.y + baseRadius * yScale,
            paint
        )

        canvas.nativeCanvas.restore()
    }
}

fun getRandomFloatInRange(min: Float, max: Float): Float {
    return Random.nextFloat() * (max - min) + min
}