Snapshot 218453
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2024-04-04T16:46:51.717-07:00",
"description": "Retell AI APIs\r\nhttps://docs.retellai.com/api-references/",
"group_id": null,
"id": "lwx818h8c19p",
"name": "API",
"page_id": "8x029ww7t6mg",
"position": 1,
"showcase": true,
"start_date": "2024-04-03T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-05-07T21:49:03.396-07:00"
},
{
"components": [
"zp9mcws9qkbb",
"8v2910yllbb6"
],
"created_at": "2024-05-06T11:34:07.321-07:00",
"description": null,
"group_id": null,
"id": "0f986dpn56cp",
"name": "End to End Calling",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-07-04T17:16:43.312-07:00"
},
{
"components": [
"qmpvts88k0rb",
"czxjkbtz30j3"
],
"created_at": "2026-03-16T12:42:33.302-07:00",
"description": null,
"group_id": null,
"id": "m3jpzvxpgsd2",
"name": "Chat",
"page_id": "8x029ww7t6mg",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-16T12:43:28.217-07:00"
},
{
"components": [
"mvdl46wg23p8",
"81w3mcrc7rws",
"wl14y03nkypc",
"j2ddm7zfhv36",
"5dz4gpjsdwz3",
"pg144yx7w9sm"
],
"created_at": "2026-03-16T12:40:03.276-07:00",
"description": null,
"group_id": null,
"id": "m5c22pk2ywc7",
"name": "Features",
"page_id": "8x029ww7t6mg",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-16T12:43:28.226-07:00"
},
{
"created_at": "2024-04-04T16:46:51.742-07:00",
"description": "Frontend playground and dashboard for users\r\nhttps://beta.retellai.com/dashboard",
"group_id": null,
"id": "cgjlf3jyd8tk",
"name": "Dashboard",
"page_id": "8x029ww7t6mg",
"position": 5,
"showcase": true,
"start_date": "2024-04-03T17:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-05-25T14:40:19.336-07:00"
}
],
"incidents": [
{
"components": [],
"created_at": "2026-10-25T16:51:19.000-07:00",
"id": "t31pxwhd727k",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "Between 1:45 PM PDT Oct 24th and 4:50 PDT Oct 25th, a permission issue with openAI speech-to-speech model caused calls using speech-to-speech agents to fail or go silent. Access has been restored and speech-to-speech calls are working normally. Agents using the standard (cascading) setup were not affected.",
"created_at": "2026-10-25T16:51:19.528-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-25T16:51:19.000-07:00",
"id": "fb4jc9fbnzl5",
"incident_id": "t31pxwhd727k",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-25T16:55:36.138-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Speech-to-speech calls failing due to a provider permission issue",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-25T16:51:19.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/zvbgjdc1ytzw",
"started_at": "2026-10-25T16:51:19.000-07:00",
"status": "resolved",
"updated_at": "2026-10-25T16:55:36.264-07:00"
},
{
"components": [
{
"created_at": "2024-04-04T17:42:39.276-07:00",
"description": "End to end calling between AI and human",
"group_id": "0f986dpn56cp",
"id": "zp9mcws9qkbb",
"name": "Web Call",
"page_id": "8x029ww7t6mg",
"position": 1,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.817-07:00"
},
{
"created_at": "2024-05-06T11:34:44.759-07:00",
"description": null,
"group_id": "0f986dpn56cp",
"id": "8v2910yllbb6",
"name": "Phone Call",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.835-07:00"
}
],
"created_at": "2026-09-16T18:15:44.034-07:00",
"id": "0lypqpmy2zjw",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The issue has been found and resolved. About 22% of calls may have been affected and been ended early, or have no audio.\n\nA postmortem will be published by EOD 09-17.",
"created_at": "2026-09-16T18:35:40.872-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T18:35:40.000-07:00",
"id": "q2sf3y5h0vdl",
"incident_id": "0lypqpmy2zjw",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-17T00:03:18.834-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are aware of issues with some calls and are investigating.",
"created_at": "2026-09-16T18:15:44.123-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T18:15:44.123-07:00",
"id": "z58nynl7k7zg",
"incident_id": "0lypqpmy2zjw",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T18:15:44.123-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Call service disruption",
"page_id": "8x029ww7t6mg",
"postmortem_body": "**Status: resolved.** All times are PST.\n\n## Summary\n\nOn September 16 at 5:37 PM we deployed a planned improvement to call handling. The improvement was not meant to be active yet, but the deployment turned it on, and it began to assign live calls to workers.\n\nThis change unintentionally let two workers take the same call, and each worker then started it. The two starts interfered with each other, and the call failed. About 22% of calls were affected from 5:37 PM to 6:20 PM.\n\nWe stopped the new workers, rolled the deployment back, and saw the issue resolve.\n\n## Impact\n\nAn affected call shows one of two signs:\n\n* The call ends as soon as it connects. The duration is zero or close to zero seconds.\n* The call connects. The agent does not speak. The call ends after a silence.\n\nA call that ran correctly is not affected. Its recording, its transcript, and its billing record are correct.\n\n## Root cause\n\nWe were making an improvement to our system for call handling. As part of this change, we made a new queue for calls and a new set of workers, which was meant to be inactive. Our old servers were writing to both queues as a test, which did not cause issues as the new workers were not active.\n\nA deployment replaces servers one group at a time, so when the deployment began at 5:37PM, some servers ran the old code and some ran the new code. The old servers still wrote each call to both queues, and the new workers now read the new queue. The two systems therefore handled the same call, once each, and undid each other's work.\n\n## Timeline \\(PST\\)\n\n| Time | Event |\n| --- | --- |\n| 5:37 PM | We deploy the change. |\n| 5:40 PM | Call errors increase. |\n| 5:43 PM | Our call monitor reports the first failures. |\n| 5:47 PM | The system pages the on-call engineer. |\n| 6:14 PM | We stop the new call workers. Double handling drops sharply. |\n| 6:18 PM | We start the rollback, which removes the change itself. |\n| 6:20 PM | The last affected call. Call errors return to normal. |\n| 7:54 PM | The rollback finishes. |\n\n## Prevention\n\n1. \\(IN PROGRESS\\) **Improve rollout process checks** We will improve our process checks for large changes to ensure that future deployments roll out in the intended state, and explicitly call out deployment time checks.\n2. \\(IN PROGRESS\\) **Let the monitor stop a deployment.** We will connect our call monitor to our deployment system. Repeated failures will then stop a deployment.\n\n## Closing\n\nWe are sorry for the disruption. A failed call is a bad experience for you, and for the person you called.",
"postmortem_body_last_updated_at": "2026-09-17T00:21:51.177-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-17T00:19:38.374-07:00",
"reminder_intervals": null,
"resolved_at": "2026-09-16T18:35:40.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/khgqtfwb8dth",
"started_at": "2026-09-16T18:35:40.000-07:00",
"status": "resolved",
"updated_at": "2026-09-17T00:23:56.967-07:00"
},
{
"components": [
{
"created_at": "2024-04-04T17:42:39.276-07:00",
"description": "End to end calling between AI and human",
"group_id": "0f986dpn56cp",
"id": "zp9mcws9qkbb",
"name": "Web Call",
"page_id": "8x029ww7t6mg",
"position": 1,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.817-07:00"
},
{
"created_at": "2024-05-06T11:34:44.759-07:00",
"description": null,
"group_id": "0f986dpn56cp",
"id": "8v2910yllbb6",
"name": "Phone Call",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.835-07:00"
}
],
"created_at": "2026-09-05T16:10:39.000-07:00",
"id": "3fvmq7mlt1wq",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The issue affecting web, outbound, and inbound calls has been resolved. All calling services are operating normally.",
"created_at": "2026-09-05T17:19:31.605-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-05T17:19:31.605-07:00",
"id": "s6ct0q78487m",
"incident_id": "3fvmq7mlt1wq",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-05T17:19:31.605-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "Starting at 4:10 PM, web and outbound calls began failing, while inbound calls experienced connection delays. We are actively investigating the issue and will provide an update as soon as more information is available.",
"created_at": "2026-09-05T16:49:39.301-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-05T16:10:39.000-07:00",
"id": "spqn9fzy1s4d",
"incident_id": "3fvmq7mlt1wq",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-05T17:19:59.498-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Web Calls and Phone Calls Disruption",
"page_id": "8x029ww7t6mg",
"postmortem_body": "### Summary\n\nOn September 5, a rare race condition in our custom LLM call path caused call scheduling queues across our fleet to stall. Because the failure also made containers appear underutilized, autoscaling never triggered. Between roughly 10:00 and 16:00 PST the impact grew until most containers were stuck and new calls could not be placed.\n\n### Impact\n\n* **Web calls, outbound calls, and inbound calls** were all affected.\n* Inbound calls were the least affected, since they hold the highest priority in our scheduling queue.\n* Peak impact occurred around 16:00 PST.\n\n### Root Cause\n\nThe failure required a specific sequence of events:\n\n1. **Stalled initialization.** If a call ended before its custom LLM websocket was established, the call's initialization never completed. The scheduling queue waited indefinitely for that initialization to finish.\n2. **Queue exhaustion.** Each container allows a fixed number of concurrent dequeue operations. Enough stalled initializations consumed all available slots, leaving the container unable to pick up new calls.\n3. **Misleading scaling signals.** The call-end path still ran and decremented the container's concurrency counter. As a result, stuck containers reported low concurrency and low resource utilization — so no scale-out was triggered.\n\nWith every container blocked and autoscaling idle, new calls had nowhere to go.\n\nThis pattern is only reachable through that exact ordering, which is why it went undetected until it occurred at scale.\n\n### Timeline \\(PST\\)\n\n* 10AM: A customer's custom LLM calls begin dropping before initialization completes.\n* Afternoon: The customer scales up traffic; the number of stalled initializations grows.\n* ~16:00: Nearly all containers are blocked on dequeue slots. Autoscaling does not trigger. Call placement is broadly impacted.\n* After 16:00: Mitigation begins: manual scale-out of containers and clearing of stuck concurrency for the affected customer. Service recovers.\n\n### Prevention\n\n1. \\(COMPLETED\\): Fix the race condition in custom LLM call initialization\n2. \\(IN PROGRESS\\): Add monitoring and alerting on call scheduling queues to detect stuck states\n3. \\(IN PROGRESS\\): Decouple the scheduling queue from call initialization so a slow or failed init cannot block dequeue\n4. \\(IN PROGRESS\\): Add call scheduling queue pressure as an input to our autoscaling policies\n\n### Closing\n\nWe apologize for the disruption. We are treating this as an opportunity to harden our scheduling and scaling infrastructure, and we will ship the remaining action items within a few days.",
"postmortem_body_last_updated_at": "2026-09-06T09:45:28.005-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-05T20:36:03.494-07:00",
"reminder_intervals": null,
"resolved_at": "2026-09-05T17:19:31.605-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/xvzvn7qv5hhv",
"started_at": "2026-09-05T16:10:39.000-07:00",
"status": "resolved",
"updated_at": "2026-09-06T09:45:28.009-07:00"
},
{
"components": [
{
"created_at": "2024-05-06T11:34:44.759-07:00",
"description": null,
"group_id": "0f986dpn56cp",
"id": "8v2910yllbb6",
"name": "Phone Call",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.835-07:00"
}
],
"created_at": "2026-08-07T14:55:52.984-07:00",
"id": "d1rdm0kcdmcg",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-08-07T15:22:13.205-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-07T15:22:13.205-07:00",
"id": "g0l341qlny9y",
"incident_id": "d1rdm0kcdmcg",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-07T15:22:13.205-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Service should be back to normal. We are continuing to monitor to ensure stability.",
"created_at": "2026-08-07T15:16:41.710-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-07T15:16:41.710-07:00",
"id": "7wss1v5tqlj3",
"incident_id": "d1rdm0kcdmcg",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-07T15:16:41.710-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Due to issues with an upstream provider, we are seeing connection issues for a portion of inbound calls. This is not widespread.",
"created_at": "2026-08-07T14:55:53.074-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-07T14:55:53.074-07:00",
"id": "2ysknny4w4h1",
"incident_id": "d1rdm0kcdmcg",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-07T14:55:53.074-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-08-07T15:16:41.710-07:00",
"name": "Some inbound calls failing to connect",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-08-07T15:22:13.205-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/70k58fhxg60t",
"started_at": "2026-08-07T14:55:52.972-07:00",
"status": "resolved",
"updated_at": "2026-08-07T15:22:13.226-07:00"
},
{
"components": [
{
"created_at": "2024-05-06T11:34:44.759-07:00",
"description": null,
"group_id": "0f986dpn56cp",
"id": "8v2910yllbb6",
"name": "Phone Call",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.835-07:00"
}
],
"created_at": "2026-07-29T07:55:42.000-07:00",
"id": "b3bdg8661m8q",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-07-29T08:42:37.301-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-29T08:42:37.301-07:00",
"id": "f77wnqjlqvh8",
"incident_id": "b3bdg8661m8q",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-29T08:42:37.301-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "This has been fixed, we are monitoring.",
"created_at": "2026-07-29T08:39:31.068-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-29T08:39:31.068-07:00",
"id": "v5wlsmclpfhz",
"incident_id": "b3bdg8661m8q",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-29T08:39:31.068-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "The issue was identified as a datacenter issue from underlying telephony provider, where 2 out of 3 datacenters failed to deliver webhook to Retell backend server.",
"created_at": "2026-07-29T08:35:15.608-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-29T08:35:15.608-07:00",
"id": "zdqjb1t4d524",
"incident_id": "b3bdg8661m8q",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-29T08:35:15.608-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We identified that certain number's inbound is not connecting, investigating. Outbound is not impacted.",
"created_at": "2026-07-29T08:26:42.300-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-29T07:55:42.000-07:00",
"id": "372w82plm0zb",
"incident_id": "b3bdg8661m8q",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-29T08:36:28.092-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-29T08:39:31.068-07:00",
"name": "Inbound calling not connecting",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-07-29T08:42:37.301-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/tsnvgr35lywv",
"started_at": "2026-07-29T07:00:42.000-07:00",
"status": "resolved",
"updated_at": "2026-07-29T08:42:37.321-07:00"
},
{
"components": [],
"created_at": "2026-07-14T08:42:34.971-07:00",
"id": "7b3nsqw6snpk",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2026-07-14T09:32:13.964-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-14T09:32:13.964-07:00",
"id": "16slxh8h3c1t",
"incident_id": "7b3nsqw6snpk",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-14T09:32:13.964-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are continuing to investigate this issue.",
"created_at": "2026-07-14T09:32:03.501-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-14T09:32:03.501-07:00",
"id": "kbp3wb2t27bg",
"incident_id": "7b3nsqw6snpk",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-14T09:32:03.501-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Outbound batch calls are failing. We are investigating the issue",
"created_at": "2026-07-14T08:42:35.012-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-14T08:42:35.012-07:00",
"id": "83xc27778808",
"incident_id": "7b3nsqw6snpk",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-14T08:42:35.012-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Batch calls failing",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-07-14T09:32:13.964-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/phj7psflrn78",
"started_at": "2026-07-14T08:42:34.963-07:00",
"status": "resolved",
"updated_at": "2026-07-14T09:32:13.982-07:00"
},
{
"components": [],
"created_at": "2026-07-01T14:45:25.042-07:00",
"id": "0yvkspq0rb1d",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2026-07-01T15:53:36.731-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-01T15:53:36.731-07:00",
"id": "xg1b9znv19f6",
"incident_id": "0yvkspq0rb1d",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-01T15:53:36.731-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-07-01T14:45:25.075-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-01T14:45:25.075-07:00",
"id": "hbg08zp55k37",
"incident_id": "0yvkspq0rb1d",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-01T14:45:25.075-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "TTS provider 11labs v3 model is down",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-07-01T15:53:36.731-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/lc4lsxy8fdfv",
"started_at": "2026-07-01T14:45:25.036-07:00",
"status": "resolved",
"updated_at": "2026-07-01T15:53:36.744-07:00"
},
{
"components": [
{
"created_at": "2024-05-06T11:34:44.759-07:00",
"description": null,
"group_id": "0f986dpn56cp",
"id": "8v2910yllbb6",
"name": "Phone Call",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.835-07:00"
}
],
"created_at": "2026-06-24T09:31:50.048-07:00",
"id": "by5mld4lvssl",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-06-24T11:05:47.582-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-24T11:05:47.582-07:00",
"id": "dy6r5k14xxl5",
"incident_id": "by5mld4lvssl",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-24T11:05:47.582-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-06-24T09:31:50.144-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-24T09:31:50.144-07:00",
"id": "thll8py10jjl",
"incident_id": "by5mld4lvssl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-24T09:31:50.144-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-06-24T09:31:50.144-07:00",
"name": "partial outage affecting incoming and ongoing call connections",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-06-24T11:05:47.582-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8s9w522ksqf7",
"started_at": "2026-06-24T09:31:50.035-07:00",
"status": "resolved",
"updated_at": "2026-06-24T11:05:47.600-07:00"
},
{
"components": [
{
"created_at": "2024-04-04T17:42:39.276-07:00",
"description": "End to end calling between AI and human",
"group_id": "0f986dpn56cp",
"id": "zp9mcws9qkbb",
"name": "Web Call",
"page_id": "8x029ww7t6mg",
"position": 1,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.817-07:00"
},
{
"created_at": "2026-03-16T12:42:33.325-07:00",
"description": null,
"group_id": "m3jpzvxpgsd2",
"id": "qmpvts88k0rb",
"name": "SMS chat",
"page_id": "8x029ww7t6mg",
"position": 1,
"showcase": true,
"start_date": "2026-03-16T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-06-22T02:03:49.633-07:00"
},
{
"created_at": "2024-05-06T11:34:44.759-07:00",
"description": null,
"group_id": "0f986dpn56cp",
"id": "8v2910yllbb6",
"name": "Phone Call",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.835-07:00"
},
{
"created_at": "2026-03-16T12:42:44.635-07:00",
"description": null,
"group_id": "m3jpzvxpgsd2",
"id": "czxjkbtz30j3",
"name": "API chat",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2026-03-16T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-06-22T02:03:49.662-07:00"
}
],
"created_at": "2026-06-22T00:00:01.000-07:00",
"id": "3szrf44r1kyz",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "qmpvts88k0rb",
"name": "Chat - SMS chat",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "czxjkbtz30j3",
"name": "Chat - API chat",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The incident has been resolved",
"created_at": "2026-06-22T02:03:49.699-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-22T02:03:49.699-07:00",
"id": "sbmgjhl6cszj",
"incident_id": "3szrf44r1kyz",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-22T02:03:49.699-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "qmpvts88k0rb",
"name": "Chat - SMS chat",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "czxjkbtz30j3",
"name": "Chat - API chat",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A fix is being deployed; we are actively monitoring.",
"created_at": "2026-06-22T01:16:21.716-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-22T01:16:21.716-07:00",
"id": "th1qk8ck410g",
"incident_id": "3szrf44r1kyz",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-22T01:16:21.716-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "qmpvts88k0rb",
"name": "Chat - SMS chat",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "czxjkbtz30j3",
"name": "Chat - API chat",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "Due to a recent regression, AI calls and chats powered by GPT-4o and GPT-4o mini are currently unavailable.\n\nOur team is working to restore service as quickly as possible.",
"created_at": "2026-06-22T01:01:01.055-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-22T00:00:01.000-07:00",
"id": "wlb4bzfbpfj3",
"incident_id": "3szrf44r1kyz",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-22T01:02:09.891-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-06-22T01:16:21.716-07:00",
"name": "Chats and calls unavailable for GPT-4o and 4o-mini",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-06-22T02:03:49.699-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bgf91wl04wn6",
"started_at": "2026-06-22T00:00:01.000-07:00",
"status": "resolved",
"updated_at": "2026-06-22T02:03:49.713-07:00"
},
{
"components": [
{
"created_at": "2024-04-04T17:42:39.276-07:00",
"description": "End to end calling between AI and human",
"group_id": "0f986dpn56cp",
"id": "zp9mcws9qkbb",
"name": "Web Call",
"page_id": "8x029ww7t6mg",
"position": 1,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.817-07:00"
},
{
"created_at": "2024-05-06T11:34:44.759-07:00",
"description": null,
"group_id": "0f986dpn56cp",
"id": "8v2910yllbb6",
"name": "Phone Call",
"page_id": "8x029ww7t6mg",
"position": 2,
"showcase": true,
"start_date": "2024-04-04T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-16T18:35:40.835-07:00"
}
],
"created_at": "2026-06-12T00:30:50.000-07:00",
"id": "sp8cz3hf2lnn",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The issue is resolved now",
"created_at": "2026-06-12T09:44:47.516-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-12T09:44:47.516-07:00",
"id": "tyh5d7wcjjkp",
"incident_id": "sp8cz3hf2lnn",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T09:44:47.516-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zp9mcws9qkbb",
"name": "End to End Calling - Web Call",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "8v2910yllbb6",
"name": "End to End Calling - Phone Call",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "The platform voice, Grace and Nico, are not working",
"created_at": "2026-06-12T09:31:50.765-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-12T00:30:50.000-07:00",
"id": "tvwjq4kqpwgc",
"incident_id": "sp8cz3hf2lnn",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T09:32:30.736-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Platform voice Grace and Nico not working",
"page_id": "8x029ww7t6mg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-06-12T09:44:47.516-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/qsdqh4dhzcdx",
"started_at": "2026-06-12T00:30:50.000-07:00",
"status": "resolved",
"updated_at": "2026-06-12T09:44:47.530-07:00"
}
],
"page": {
"id": "8x029ww7t6mg",
"name": "Retell AI",
"url": "https://status.retellai.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}