Why Doesn’t My Outbound Twilio Call Terminate Automatically When Using WebSocket Media Streams?

I'm developing an application using Twilio's Programmable Voice API coupled with WebSockets to stream media for an outbound call with real-time AI processing. Although the call initiates and the AI processes the media stream via WebSocket, the call does not end automatically when the conversation concludes; it remains active until I manually hang up.

Expected Behavior

• The call should terminate automatically when a predefined condition is met (e.g., after a closing phrase).
• The WebSocket should close once the call is completed.

Current Implementation

I implemented call handling with Twilio’s API and Media Streams. Below is a simplified version of my implementation:

Call Initiation

PYTHONCopied
import os
import json
import asyncio
import re
import logging
from dotenv import load_dotenv
from fastapi import FastAPI, WebSocket
from fastapi.websockets import WebSocketDisconnect, WebSocketState
from twilio.rest import Client
import websockets
import uvicorn
from twilio.twiml.voice_response import VoiceResponse, Connect, Hangup

load_dotenv()

TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID")
TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
PHONE_NUMBER_FROM = os.getenv("PHONE_NUMBER_FROM")
DOMAIN = os.getenv("DOMAIN", "")
PORT = int(os.getenv("PORT", 6060))

twilio_client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)

app = FastAPI()

async def make_call(phone_number: str):
    response = VoiceResponse()
    connect = Connect()
    connect.stream(url=f"wss://{DOMAIN}/media-stream", timeout="10", maxLength="1800")
    response.append(connect)
    response.append(Hangup())

    try:
        call = await asyncio.to_thread(
            twilio_client.calls.create,
            from_=PHONE_NUMBER_FROM,
            to=phone_number,
            twiml=str(response)
        )
        return call.sid
    except Exception as e:
        logging.error(f"Failed to initiate call: {e}")
        raise

---

WebSocket Handling for Media Stream

PYTHONCopied
@app.websocket("/media-stream")
async def handle_media_stream(websocket: WebSocket):
    await websocket.accept()
    logging.info("WebSocket connected")

    try:
        async with websockets.connect("wss://api.example.com/ai-processing") as ai_ws:
            await handle_conversation(websocket, ai_ws)
    except Exception as e:
        logging.error(f"WebSocket error: {e}")
        await websocket.close(code=1011)

async def handle_conversation(websocket: WebSocket, ai_ws):
    async def receive_from_twilio():
        try:
            async for message in websocket.iter_text():
                data = json.loads(message)
                if data.get("event") == "media" and ai_ws.open:
                    await ai_ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": data["media"]["payload"]}))
        except WebSocketDisconnect:
            logging.info("Twilio WebSocket disconnected")

    async def send_to_twilio():
        try:
            async for ai_message in ai_ws:
                response = json.loads(ai_message)

                if response.get("type") == "response.audio.delta":
                    audio_payload = base64.b64encode(base64.b64decode(response["delta"])).decode("utf-8")
                    await websocket.send_json({
                        "event": "media",
                        "streamSid": "12345",
                        "media": {"payload": audio_payload}
                    })
                elif response.get("type") == "function_call" and response["function"]["name"] == "end_call":
                    await end_call()
        except Exception as e:
            logging.error(f"Error in send_to_twilio: {e}")

    await asyncio.gather(receive_from_twilio(), send_to_twilio())

---

Function to End the Call

PYTHONCopied
async def end_call():
    global active_call_sid
    if active_call_sid:
        try:
            logging.info(f"Ending call {active_call_sid}")
            await asyncio.to_thread(
                twilio_client.calls(active_call_sid).update,
                status="completed"
            )
            logging.info(f"Call {active_call_sid} ended successfully")
            active_call_sid = None
        except Exception as e:
            logging.error(f"Error ending call: {e}")
            active_call_sid = None

Open AI Tool

PYTHONCopied
from typing import Dict, Any

end_call_tool_definition = {
    "type": "function",
    "name": "end_call",
    "description": "Ends the current interview call after saying goodbye.",
    "parameters": {
        "type": "object",
        "properties": {
            "reason": {
                "type": "string",
                "description": "The reason for ending the call (e.g., 'interview_complete', 'user_request')"
            }
        },
        "required": ["reason"]
    }
}

async def end_call_tool(active_call_sid: str, twilio_client) -> Dict[str, Any]:
    """
    Tool to terminate the Twilio call
    """
    try:
        if active_call_sid:
            await twilio_client.calls(active_call_sid).update(status="completed")
            return {
                "status": "success",
                "message": f"Call {active_call_sid} terminated successfully"
            }
    except Exception as e:
        return {
            "status": "error",
            "message": f"Failed to terminate call: {str(e)}"
        }

---

Issue

• The call does not end even after executing the end_call function.
• The WebSocket connection remains open, and Twilio does not disconnect the media stream.
• The end_call function logs a success message, yet the call stays active until manually hung up.

---

What I Have Tried

• Verified that end_call is executed and logs a success message.
• Ensured that active_call_sid is correctly assigned and maintained.
• Checked Twilio logs for API errors related to call termination.
• Tried using <Hangup/> in TwiML, but the issue remains.
• Implemented proper WebSocket closure after call termination.
• Tested using AI tool calls as referenced in the blog post: https://www.twilio.com/en-us/blog/outbound-calls-python-openai-realtime-api-voice

Any insights into why the call remains active and how to ensure that both the call and WebSocket connection close automatically would be greatly appreciated.