Agora Conversational AI Voice UI for React

pnpm add orb-ui@0.10.0 agora-agent-client-toolkit@2.10.0 agora-rtc-sdk-ng@4.24.8 agora-rtm@2.3.0

import { useEffect, useMemo } from 'react'
import AgoraRTC, {
  type IAgoraRTCRemoteUser,
  type ILocalAudioTrack,
  type IRemoteAudioTrack,
} from 'agora-rtc-sdk-ng'
import AgoraRTM, { type RTMClient } from 'agora-rtm'
import { AgoraVoiceAI, TranscriptHelperMode } from 'agora-agent-client-toolkit'
import { Orb } from 'orb-ui'
import { createAgoraAdapter, type AgoraSession } from 'orb-ui/adapters'

type SessionGrant = {
  id: string
  appId: string
  channel: string
  uid: string
  agentUserId: string
  rtcToken: string
  rtmToken: string
}

async function jsonRequest<T>(url: string, signal: AbortSignal, method = 'POST'): Promise<T> {
  const response = await fetch(url, {
    method,
    signal,
    credentials: 'same-origin',
    cache: 'no-store',
  })
  if (!response.ok) throw new Error('Your application could not authorize the Agora session.')
  return response.json()
}

async function createSession(signal: AbortSignal): Promise<AgoraSession> {
  const rtc = AgoraRTC.createClient({ mode: 'rtc', codec: 'vp8' })
  let microphone: ILocalAudioTrack | undefined
  let remoteAudio: IRemoteAudioTrack | undefined
  let rtm: RTMClient | undefined
  let toolkit: AgoraVoiceAI | undefined
  let grant: SessionGrant | undefined
  let closed = false
  let playbackError: unknown
  let closing: Promise<void> | undefined

  const check = () => {
    if (closed || signal.aborted) throw new DOMException('Session cancelled.', 'AbortError')
  }
  const renew = async () => {
    try {
      check()
      const tokens = await jsonRequest<{ rtcToken: string; rtmToken: string }>(
        `/api/agora/sessions/${grant!.id}/tokens`,
        signal,
      )
      check()
      await Promise.all([rtc.renewToken(tokens.rtcToken), rtm!.renewToken(tokens.rtmToken)])
    } catch (error) {
      playbackError = error
    }
  }
  const published = async (
    user: IAgoraRTCRemoteUser,
    mediaType: 'audio' | 'video' | 'datachannel',
  ) => {
    if (mediaType !== 'audio' || String(user.uid) !== grant?.agentUserId) return
    try {
      await rtc.subscribe(user, 'audio')
      check()
      remoteAudio = user.audioTrack
      remoteAudio?.play()
    } catch (error) {
      user.audioTrack?.stop()
      if (!closed && !signal.aborted) playbackError = error
    }
  }
  const unpublished = (user: IAgoraRTCRemoteUser, mediaType: 'audio' | 'video') => {
    if (mediaType === 'audio' && String(user.uid) === grant?.agentUserId) {
      remoteAudio?.stop()
      remoteAudio = undefined
    }
  }
  const close = (): Promise<void> => {
    if (closing) return closing
    closed = true
    signal.removeEventListener('abort', cancelCapture)
    microphone?.close()
    remoteAudio?.stop()
    rtc.off('user-published', published)
    rtc.off('user-unpublished', unpublished)
    rtc.off('token-privilege-will-expire', renew)
    rtm?.removeEventListener('tokenPrivilegeWillExpire', renew)
    toolkit?.unsubscribe()
    toolkit?.destroy()
    // No AbortSignal here: cancellation must still stop the backend agent.
    closing = Promise.allSettled([
      rtc.leave(),
      rtm?.logout(),
      grant
        ? fetch(`/api/agora/sessions/${grant.id}`, {
            method: 'DELETE',
            credentials: 'same-origin',
            keepalive: true,
          }).then((response) => {
            if (!response.ok) throw new Error('Agent shutdown was not confirmed.')
          })
        : undefined,
    ]).then((results) => {
      const failure = results.find((result) => result.status === 'rejected')
      if (failure?.status === 'rejected') throw failure.reason
    })
    return closing
  }
  const cancelCapture = () => {
    closed = true
    microphone?.close()
    remoteAudio?.stop()
  }
  signal.addEventListener('abort', cancelCapture, { once: true })

  try {
    // Start capture before any network await. Late permission results are closed.
    const capture = AgoraRTC.createMicrophoneAudioTrack({ encoderConfig: 'speech_standard' }).then(
      (track) => {
        microphone = track
        if (closed || signal.aborted) {
          track.close()
          check()
        }
        return track
      },
    )
    const values = await Promise.all([
      jsonRequest<SessionGrant>('/api/agora/sessions', signal).then((value) => {
        grant = value
        return value
      }),
      capture,
    ])
    grant = values[0]
    check()
    rtm = new AgoraRTM.RTM(grant.appId, grant.uid)
    await rtm.login({ token: grant.rtmToken })
    check()
    toolkit = await AgoraVoiceAI.init({
      rtcEngine: rtc,
      rtmEngine: rtm,
      renderMode: TranscriptHelperMode.TEXT,
    })
    check()
    rtc.on('user-published', published)
    rtc.on('user-unpublished', unpublished)
    rtc.on('token-privilege-will-expire', renew)
    rtm.addEventListener('tokenPrivilegeWillExpire', renew)
    await rtc.join(grant.appId, grant.channel, grant.rtcToken, grant.uid)
    check()
    await rtc.publish(microphone!)
    toolkit.subscribeMessage(grant.channel)
    await jsonRequest(`/api/agora/sessions/${grant.id}/start`, signal)
    check()
    return {
      toolkit,
      rtc,
      agentUserId: grant.agentUserId,
      close,
      getMicrophoneTrack: () => microphone,
      getAgentAudioTrack: () => {
        if (playbackError) throw playbackError
        return remoteAudio
      },
    }
  } catch (error) {
    await close().catch(() => undefined)
    throw error
  }
}

export function AgoraVoice() {
  const adapter = useMemo(() => createAgoraAdapter({ createSession }), [])
  useEffect(
    () => () => {
      void adapter.stop().catch(() => undefined)
    },
    [adapter],
  )
  return <Orb adapter={adapter} theme="circle" aria-label="Start or stop your Agora assistant" />
}