Build an ElevenLabs Conversational AI Agent in React

React → your POST /api/conversation-token → ElevenLabs token endpoint
React + conversation token → ElevenLabs WebRTC session

npm create vite@latest elevenlabs-react-agent -- --template react-ts
cd elevenlabs-react-agent
npm install
npm install @elevenlabs/client@1.9.0 express

ELEVENLABS_API_KEY=replace_with_your_server_key
ELEVENLABS_AGENT_ID=replace_with_your_private_agent_id

import express from 'express'

const apiKey = process.env.ELEVENLABS_API_KEY
const agentId = process.env.ELEVENLABS_AGENT_ID
if (!apiKey || !agentId) throw new Error('Set both ElevenLabs environment variables')

const app = express()
app.post('/api/conversation-token', async (req, res) => {
  res.set('Cache-Control', 'no-store')
  // This server is for the local tutorial only. Reject other browser origins.
  if (req.get('origin') !== 'http://localhost:5173') {
    return res.status(403).json({ error: 'Origin not allowed' })
  }

  try {
    const url = new URL('https://api.elevenlabs.io/v1/convai/conversation/token')
    url.searchParams.set('agent_id', agentId)
    const upstream = await fetch(url, {
      headers: { 'xi-api-key': apiKey },
      signal: AbortSignal.timeout(10_000),
    })
    if (!upstream.ok) throw new Error('Token request failed')
    const body = await upstream.json()
    if (typeof body.token !== 'string' || !body.token) {
      throw new Error('Missing conversation token')
    }
    return res.json({ token: body.token })
  } catch {
    // Do not forward provider responses or credentials to the browser.
    return res.status(502).json({ error: 'Unable to start a conversation' })
  }
})

app.listen(3001, '127.0.0.1', () => {
  console.log('Local token server listening on port 3001')
})

import { defineConfig } from 'vite'
import react from '@vitejs/plugin-react'

export default defineConfig({
  plugins: [react()],
  server: {
    host: 'localhost',
    port: 5173,
    strictPort: true,
    proxy: { '/api': 'http://127.0.0.1:3001' },
  },
})

import { Conversation } from '@elevenlabs/client'

type Session = Awaited<ReturnType<typeof Conversation.startSession>>
type Phase = 'idle' | 'connecting' | 'connected' | 'stopping'
type Events = {
  phase: (value: Phase) => void
  mode: (value: 'speaking' | 'listening') => void
  message: (value: string) => void
  error: (value: string) => void
}
type Attempt = {
  cancelled: boolean
  ended: boolean
  starting: boolean
  abort: AbortController
  session?: Session
}

export function createVoice(events: Events) {
  let current: Attempt | undefined
  const live = (attempt: Attempt) => current === attempt && !attempt.cancelled
  const finish = (attempt: Attempt) => {
    if (current !== attempt) return
    current = undefined
    events.phase('idle')
  }
  const end = async (session: Session) => {
    try {
      await session.endSession()
    } catch {
      events.error('Could not confirm session cleanup. Reload before reconnecting.')
      // Keep the controller locked if cleanup could not be confirmed.
      throw new Error('Session cleanup failed')
    }
  }

  return {
    async start() {
      if (current) return
      const attempt: Attempt = {
        cancelled: false,
        ended: false,
        starting: true,
        abort: new AbortController(),
      }
      current = attempt
      events.error('')
      events.phase('connecting')
      try {
        const response = await fetch('/api/conversation-token', {
          method: 'POST',
          signal: attempt.abort.signal,
        })
        if (!response.ok) throw new Error('Unable to obtain a conversation token')
        const { token } = await response.json()
        if (typeof token !== 'string' || !token) throw new Error('Invalid token response')
        if (!live(attempt)) return
        const session = await Conversation.startSession({
          conversationToken: token,
          connectionType: 'webrtc',
          onMessage: ({ role, message }) => {
            if (live(attempt)) events.message(`${role}: ${message}`)
          },
          onModeChange: ({ mode }) => {
            if (live(attempt)) events.mode(mode)
          },
          onInterruption: () => {
            if (live(attempt)) events.mode('listening')
          },
          onError: (message) => {
            if (live(attempt)) events.error(message)
          },
          onDisconnect: () => {
            attempt.ended = true
            if (!attempt.starting && !attempt.cancelled) finish(attempt)
          },
        })
        attempt.session = session
        if (live(attempt) && !attempt.ended) {
          events.phase('connected')
        }
      } catch (error) {
        attempt.ended = true
        if (live(attempt)) {
          events.error(error instanceof Error ? error.message : 'Connection failed')
        }
      } finally {
        attempt.starting = false
        if (attempt.cancelled || attempt.ended) {
          events.phase('stopping')
          try {
            if (attempt.session) await end(attempt.session)
            finish(attempt)
          } catch {
            /* stay locked */
          }
        }
      }
    },
    async stop() {
      const attempt = current
      if (!attempt || attempt.cancelled) return
      attempt.cancelled = true
      attempt.abort.abort()
      events.phase('stopping')
      if (attempt.session && !attempt.starting) {
        try {
          await end(attempt.session)
          finish(attempt)
        } catch {
          /* stay locked */
        }
      }
    },
  }
}

import { useEffect, useRef, useState } from 'react'
import { createVoice } from './voice'

export default function App() {
  const voice = useRef<ReturnType<typeof createVoice> | null>(null)
  const [phase, setPhase] = useState('idle')
  const [mode, setMode] = useState('listening')
  const [error, setError] = useState('')
  const [messages, setMessages] = useState<string[]>([])

  useEffect(() => {
    let mounted = true
    const controller = createVoice({
      phase: (value) => {
        if (mounted) setPhase(value)
      },
      mode: (value) => {
        if (mounted) setMode(value)
      },
      error: (value) => {
        if (mounted) setError(value)
      },
      message: (value) => {
        if (mounted) setMessages((items) => [...items.slice(-49), value])
      },
    })
    voice.current = controller
    return () => {
      mounted = false
      voice.current = null
      void controller.stop()
    }
  }, [])

  function start() {
    setMessages([])
    setMode('listening')
    void voice.current?.start()
  }

  return (
    <main style={{ maxWidth: 720, margin: '3rem auto', padding: 24 }}>
      <h1>Talk to your ElevenLabs agent</h1>
      <p>Start enables your microphone and sends audio to the agent.</p>
      <p role="status">{phase === 'connected' ? mode : phase}</p>
      <button disabled={phase !== 'idle'} onClick={start}>
        Start conversation
      </button>
      <button
        disabled={phase === 'idle' || phase === 'stopping'}
        onClick={() => void voice.current?.stop()}
      >
        Stop
      </button>
      {error && <p role="alert">{error}</p>}
      <h2>Live text events</h2>
      <ol>
        {messages.map((message, index) => (
          <li key={index}>{message}</li>
        ))}
      </ol>
    </main>
  )
}

node --env-file=.env server.mjs

npm run dev