Snapshot 258474
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2022-02-18T18:42:41.790-08:00",
"description": "LiveKit Cloud API and media backend",
"group_id": null,
"id": "w6vlj3798sr9",
"name": "Global Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 1,
"showcase": true,
"start_date": "2022-02-18T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-19T10:11:40.674-07:00"
},
{
"created_at": "2022-03-14T14:26:39.493-07:00",
"description": "LiveKit Cloud Dashboard (analytics, settings, and billing)",
"group_id": null,
"id": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"page_id": "1l77jzt47dnl",
"position": 2,
"showcase": true,
"start_date": "2022-03-14T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-04T20:49:10.078-08:00"
},
{
"created_at": "2024-04-26T13:45:56.283-07:00",
"description": "LiveKit Cloud Egress",
"group_id": null,
"id": "tqbt7d0pxglt",
"name": "Global Egress",
"page_id": "1l77jzt47dnl",
"position": 3,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-09-15T11:03:07.889-07:00"
},
{
"created_at": "2024-04-09T13:39:18.122-07:00",
"description": "LiveKit Cloud Ingress",
"group_id": null,
"id": "xxpt24rv03gb",
"name": "Global Ingress",
"page_id": "1l77jzt47dnl",
"position": 4,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-09-15T11:03:07.904-07:00"
},
{
"created_at": "2024-04-09T13:34:13.943-07:00",
"description": "LiveKit Cloud SIP",
"group_id": null,
"id": "721pl69wg5js",
"name": "Global SIP",
"page_id": "1l77jzt47dnl",
"position": 5,
"showcase": true,
"start_date": "2024-03-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-09-15T11:03:07.916-07:00"
},
{
"created_at": "2025-08-08T16:51:06.523-07:00",
"description": "LiveKit Cloud Agents",
"group_id": null,
"id": "sby4t3xn3f13",
"name": "Global Cloud Agents",
"page_id": "1l77jzt47dnl",
"position": 6,
"showcase": true,
"start_date": "2025-08-08T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2025-10-07T14:25:18.755-07:00"
},
{
"created_at": "2025-11-10T10:58:43.733-08:00",
"description": "LiveKit Inference",
"group_id": null,
"id": "9vhqhvqj8w2f",
"name": "Global Inference",
"page_id": "1l77jzt47dnl",
"position": 7,
"showcase": true,
"start_date": "2025-10-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-08-06T14:08:41.462-07:00"
},
{
"components": [
"lc7bgprzlp3l",
"4nkzh05vycmp",
"tl22wqghm6pb",
"8ncb6xyswtqz",
"c4m3xm3cwt6h",
"v5gf807mvj7b",
"vtht1l05qwwh",
"4mwwf6ttmdb4",
"17x2nqd992p7",
"9lgc3vgdkw8m",
"bhmbfl3bsj1c",
"hskhhq2dylq3",
"nrltzplnclyb",
"nmmxjspd37t9",
"g3swm83zqq5b"
],
"created_at": "2023-08-15T16:12:19.586-07:00",
"description": "Status of RTC services in each data center. This is informational and does not impact end users. When individual data centers have outages, traffic is automatically rerouted to healthy data centers.",
"group_id": null,
"id": "wnz36vpsr8r6",
"name": "Regional Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 8,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-11-10T11:06:47.983-08:00"
},
{
"components": [
"hvk3pdr6718l",
"2tbbklyb4hyg",
"dqt24j3vbfs2",
"xlh8b38zdsqy",
"pqnbk13z9t9w",
"qlbhfg915ty7",
"4p8nhdn2hnzf",
"3kqys4tr4cm0",
"r9wfj2c3gxfd",
"1dx8y43fvzxq",
"qgjt5xhqk19x",
"y6208vsg87rm",
"x68rs848r6l4",
"8wnsdj48gl79",
"1wdbm64pwvn2"
],
"created_at": "2023-08-15T16:12:20.254-07:00",
"description": "LiveKit RTC Proxy",
"group_id": null,
"id": "6c8qxblzzb28",
"name": "Regional TURN",
"page_id": "1l77jzt47dnl",
"position": 9,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-11-10T11:06:47.998-08:00"
},
{
"components": [
"7qfmhthz007l",
"slslgb76xtrs",
"kxrg82356fw7",
"y73p98f7tjcj",
"r0dtyvvp7rlf",
"8x9mw06n1gc1",
"m209vs5lqfb5",
"dh6h9x7b4hjg",
"t5jvz91wjkbl",
"g4hkvbj4fpjn",
"rgkzhrmy9djg",
"z5x79lgc1ysv",
"4h15kx4656x6",
"d1rwg8qm7tdh",
"pxry48sd81cn"
],
"created_at": "2023-08-15T16:12:19.479-07:00",
"description": "LiveKit analytics ingestion service",
"group_id": null,
"id": "sgn9505ymgt1",
"name": "Regional Analytics Ingestion",
"page_id": "1l77jzt47dnl",
"position": 10,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-11-10T11:06:48.011-08:00"
},
{
"components": [
"hl33tvh2rsyv"
],
"created_at": "2026-06-12T19:08:50.148-07:00",
"description": null,
"group_id": null,
"id": "vft5zs1w67gn",
"name": "Regional Phone Numbers",
"page_id": "1l77jzt47dnl",
"position": 11,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-12T19:09:21.451-07:00"
},
{
"components": [
"39v8x79rxsz0",
"n1bl6t93gxjp",
"r5d2n1576yc9",
"j67f6nrhkbyr",
"pmpy136qfygt",
"h8sxjmx739sn",
"qt4ft6wpk1z2",
"mw34774gztnn",
"rkq0ws59cp9c",
"dwrnqmtt4j9v",
"9cp7ypprb299",
"ljrsf4q5zsp1",
"dg0thvt4cs4c",
"g0q45y8b17gt",
"95y3tyskl647"
],
"created_at": "2024-04-09T13:31:09.214-07:00",
"description": "LiveKit Cloud SIP",
"group_id": null,
"id": "10pgb4w5dktp",
"name": "Regional SIP",
"page_id": "1l77jzt47dnl",
"position": 12,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-12T19:09:21.465-07:00"
},
{
"components": [
"0dffphcxdpms",
"fdvz9r3d3zxq",
"p1v76tf148q0",
"xflmkytqgjqq",
"n1m21rzckvpx",
"vnfx4cjv20ms",
"42dm2xz6rv5f",
"tj3fwnx1tn1d",
"fpx2pyfvqkzb",
"hvs203d19slt",
"z70tw064kw5v",
"qzm7yxbsqyrv",
"9nw07mbwvjt9",
"3tq2xx5rn6np",
"9yzgz1mb16sl"
],
"created_at": "2024-04-09T13:41:44.617-07:00",
"description": "LiveKit Cloud Ingress",
"group_id": null,
"id": "yp6lxpz96my7",
"name": "Regional Ingress",
"page_id": "1l77jzt47dnl",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-12T19:09:21.480-07:00"
},
{
"components": [
"3nk8yqm24tcz",
"h4y3ycbkk44g",
"4jphsby1js10",
"lbqmjtz83mz6",
"q9y5js232qj3",
"l901z012dksj",
"kws59sqph3gk",
"rjyfbb7nwsj9",
"s278rcs69fg4",
"754jl79gk2xw",
"cy46qgxpbvfn",
"j2yrmy5t5f25",
"y0q1x3v80p5z",
"4j4xgjkzmb76",
"wh2x4r3bvp7z"
],
"created_at": "2024-04-26T13:45:56.312-07:00",
"description": "LiveKit Cloud Egress",
"group_id": null,
"id": "kq12w94wnd91",
"name": "Regional Egress",
"page_id": "1l77jzt47dnl",
"position": 14,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-12T19:09:21.495-07:00"
},
{
"components": [
"rg84mxb77y57",
"23msrmqcyflg",
"m8jjnmhf79pr",
"2dtfkd6xd7fc"
],
"created_at": "2025-08-08T16:51:06.801-07:00",
"description": "LiveKit Cloud Agents",
"group_id": null,
"id": "jnv1myhtwskd",
"name": "Regional Cloud Agents",
"page_id": "1l77jzt47dnl",
"position": 15,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-12T19:09:21.508-07:00"
},
{
"components": [
"y3by6kk36ks2",
"xynf5bz37k2m",
"pbbbbcfwhym2",
"d5gbv778ltcc",
"h65p2dtrkg13",
"y8frvwlk98fj",
"6kjbcj2s4fmd",
"9t9lhjsvtd51",
"2cjzj98zztyq",
"fzspscjl06dy",
"byzl2ztt3ktk",
"fhfggxc9twlt",
"9g8spx72gqvm",
"2yt1wn4f1zgq",
"8pq6kq7yyywr"
],
"created_at": "2025-11-10T11:02:29.702-08:00",
"description": "LiveKit Inference",
"group_id": null,
"id": "tfzhwknxcr0l",
"name": "Regional Inference",
"page_id": "1l77jzt47dnl",
"position": 16,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-12T19:09:21.523-07:00"
}
],
"incidents": [
{
"components": [],
"created_at": "2026-11-11T12:30:00.000-08:00",
"id": "wfwdx9wrq319",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "Between approximately 18:15 and 20:30 UTC on November 11, some requests to Gemma 4 31B (google/gemma-4-31b-it) on LiveKit Inference stalled or timed out due to an issue at an upstream model provider. The provider has mitigated the issue and Gemma is operating normally.",
"created_at": "2026-11-11T13:40:21.809-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-11T12:30:00.000-08:00",
"id": "n6cwb9ddzdz4",
"incident_id": "wfwdx9wrq319",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T13:41:57.012-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Elevated timeouts on LiveKit Inference Gemma 4 31B on low-priority requests",
"page_id": "1l77jzt47dnl",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-11T12:30:00.000-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/0ymy6fhvphf1",
"started_at": "2026-11-11T12:30:00.000-08:00",
"status": "resolved",
"updated_at": "2026-11-11T13:41:57.321-08:00"
},
{
"components": [
{
"created_at": "2022-03-14T14:26:39.493-07:00",
"description": "LiveKit Cloud Dashboard (analytics, settings, and billing)",
"group_id": null,
"id": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"page_id": "1l77jzt47dnl",
"position": 2,
"showcase": true,
"start_date": "2022-03-14T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-04T20:49:10.078-08:00"
}
],
"created_at": "2026-11-03T12:03:00.283-08:00",
"id": "52cbdz5f83yn",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-11-04T20:49:10.149-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-04T20:49:10.149-08:00",
"id": "5k8wmcm0w9wl",
"incident_id": "52cbdz5f83yn",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-04T20:49:10.149-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
}
],
"body": "Verification is currently in progress",
"created_at": "2026-11-04T20:38:52.125-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-04T20:38:52.125-08:00",
"id": "064nbfsn89v9",
"incident_id": "52cbdz5f83yn",
"status": "verifying",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-04T20:38:52.125-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-11-04T20:00:08.837-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-04T20:00:08.837-08:00",
"id": "cpvvfh437qg9",
"incident_id": "52cbdz5f83yn",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-04T20:00:08.837-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Maintenance will begin as scheduled in 60 minutes.",
"created_at": "2026-11-04T19:00:30.088-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-04T19:00:30.088-08:00",
"id": "s6rbzjfdzj5g",
"incident_id": "52cbdz5f83yn",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-04T19:00:30.088-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be upgrading critical backend components supporting Cloud Analytics.\n\nDuring this operation, there may be minimal downtime or brief interruptions to analytics services. We will make every effort to keep the Cloud Dashboard and Analytics APIs available throughout the upgrade.\n\nNo data will be lost, and real-time traffic will remain unaffected.\n\nThank you for your patience while we perform these upgrades.",
"created_at": "2026-11-03T12:03:00.380-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T12:03:00.380-08:00",
"id": "3mydwq8zgl4z",
"incident_id": "52cbdz5f83yn",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T12:03:00.380-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-04T20:38:52.125-08:00",
"name": "Cloud Analytics – Infrastructure Upgrade Notice",
"page_id": "1l77jzt47dnl",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-04T20:49:10.149-08:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-11-04T20:00:00.000-08:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": "2026-11-04T19:00:30.378-08:00",
"scheduled_until": "2026-11-04T21:00:00.000-08:00",
"shortlink": "https://stspg.io/02sc593k12mv",
"started_at": "2026-11-03T12:03:00.277-08:00",
"status": "completed",
"updated_at": "2026-11-04T20:49:10.194-08:00"
},
{
"components": [
{
"created_at": "2024-04-09T13:39:17.990-07:00",
"description": "LiveKit Cloud Ingress",
"group_id": "yp6lxpz96my7",
"id": "0dffphcxdpms",
"name": "US East - Ingress",
"page_id": "1l77jzt47dnl",
"position": 3,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-31T09:15:05.444-07:00"
},
{
"created_at": "2024-04-09T13:39:18.302-07:00",
"description": "LiveKit Cloud Ingress",
"group_id": "yp6lxpz96my7",
"id": "fdvz9r3d3zxq",
"name": "US West - Ingress",
"page_id": "1l77jzt47dnl",
"position": 4,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-31T09:15:10.992-07:00"
},
{
"created_at": "2023-08-15T16:10:06.164-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "8ncb6xyswtqz",
"name": "US West - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 7,
"showcase": true,
"start_date": "2023-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-31T09:13:44.329-07:00"
},
{
"created_at": "2024-08-26T15:05:04.869-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "vtht1l05qwwh",
"name": "US Central - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 12,
"showcase": true,
"start_date": "2024-08-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:13:59.073-07:00"
},
{
"created_at": "2024-09-05T15:08:46.949-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "17x2nqd992p7",
"name": "UAE - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 15,
"showcase": true,
"start_date": "2024-09-05T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2025-03-17T23:20:47.000-07:00"
},
{
"created_at": "2025-02-14T04:10:01.468-08:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "9lgc3vgdkw8m",
"name": "Israel - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 16,
"showcase": true,
"start_date": "2025-02-14T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:36.913-07:00"
},
{
"created_at": "2025-04-22T11:50:10.358-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "bhmbfl3bsj1c",
"name": "Saudi Arabia - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 17,
"showcase": true,
"start_date": "2025-04-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:46.039-07:00"
},
{
"created_at": "2025-07-23T07:55:28.905-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "hskhhq2dylq3",
"name": "Europe South - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 18,
"showcase": true,
"start_date": "2025-07-23T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:53.757-07:00"
},
{
"created_at": "2025-07-23T07:56:34.258-07:00",
"description": "LiveKit Cloud Ingress",
"group_id": "yp6lxpz96my7",
"id": "3tq2xx5rn6np",
"name": "US Central - Ingress",
"page_id": "1l77jzt47dnl",
"position": 19,
"showcase": true,
"start_date": "2025-07-23T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:15:19.220-07:00"
},
{
"created_at": "2025-07-23T07:55:29.462-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "nmmxjspd37t9",
"name": "US East - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 20,
"showcase": true,
"start_date": "2025-07-23T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:10.269-07:00"
},
{
"created_at": "2025-07-23T07:55:29.532-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "g3swm83zqq5b",
"name": "Europe Central - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 21,
"showcase": true,
"start_date": "2025-07-23T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:59.837-07:00"
}
],
"created_at": "2026-10-31T00:43:54.000-07:00",
"id": "ltw4070n1sm6",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "8ncb6xyswtqz",
"name": "Regional Real Time Communication - US West - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "17x2nqd992p7",
"name": "Regional Real Time Communication - UAE - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9lgc3vgdkw8m",
"name": "Regional Real Time Communication - Israel - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "bhmbfl3bsj1c",
"name": "Regional Real Time Communication - Saudi Arabia - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hskhhq2dylq3",
"name": "Regional Real Time Communication - Europe South - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "g3swm83zqq5b",
"name": "Regional Real Time Communication - Europe Central - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0dffphcxdpms",
"name": "Regional Ingress - US East - Ingress",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "fdvz9r3d3zxq",
"name": "Regional Ingress - US West - Ingress",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "3tq2xx5rn6np",
"name": "Regional Ingress - US Central - Ingress",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved. Between 07:10 and 07:43 UTC on October 31, a software bug caused media servers in several regions to restart repeatedly. \n\nDuring this window:\n- Participants connected to affected servers were disconnected. Impact was heaviest in US East, US Central, US West, Canada and Mexico, where about 18% of participants were disconnected at least once. Many were disconnected more than once, as automatic reconnects landed on servers that then restarted as well. A small share of participants in Europe Central, Europe South, UAE, Saudi Arabia and Israel were also disconnected. Most disconnected participants reconnected automatically, about half within 5 seconds, but each reconnect started a new session rather than resuming the existing one, so participants may have seen repeated audio and video interruptions. Some participants did not reconnect. Participants joining during this window may also have taken longer to connect or failed to join.\n- Most agents reconnected and continued their conversations, but a small number of agent sessions ended or restarted. Some SIP call transfers also failed.\n- About 1.3% of Room Service API requests failed during this period. ListParticipants and GetParticipant were the most affected. Some WHIP ingress update and delete requests also failed.\n\nThe bug was triggered when a specific type of data message was relayed between servers hosting the same multi-region room. We stopped the trigger at 07:43 UTC, and there were no further restarts after that. A fix was deployed to all regions by 08:49 UTC.\n\nApologies for the impact. We will follow up with a postmortem sharing more details on impact and RCA as soon as possible.",
"created_at": "2026-10-31T09:05:54.444-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T00:43:54.000-07:00",
"id": "ndb9nxz9d28y",
"incident_id": "ltw4070n1sm6",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T09:11:10.494-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Disconnections and failed connections in multiple regions",
"page_id": "1l77jzt47dnl",
"postmortem_body": "**Summary**\n\nOn October 31, 2026, between 07:10 and 07:43 UTC, an uncommon data pattern surfaced a latent software bug which caused media servers in several regions to restart repeatedly. A small percentage of participants were disconnected and the vast majority automatically reconnected. Impact was concentrated in North America. We stopped the trigger at 07:43 UTC and deployed a fix to all regions by 08:49 UTC.\n\nWe apologize for the disruption this caused.\n\n**Root Cause**\n\nApplications can open their own data channel and send raw data over it, outside of LiveKit's standard data message format. Media servers forward this raw data to other participants in the room, wrapping it with the sender's identity so that receivers know who sent it. When a room spans multiple regions, the media servers hosting it also forward this data to each other. That server-to-server step sent the raw data without the sender's identity, and the receiving server assumed the identity would be present, which caused it to crash. This gap in our server-to-server protocol had been present for some time and was not introduced by a recent deployment.\n\nStarting at 07:10 UTC, one application sent raw data over its own data channel in a room that spanned multiple regions. Every server that received the forwarded message crashed at the same time, while the server the sender was connected to kept running and kept sending. Participants on the crashed servers automatically reconnected to other servers. Those servers then began hosting the room, received the next message, and crashed as well. This cycle repeated until we stopped the trigger at 07:43 UTC. In total, media servers in 12 of our data centers restarted, some of them up to 10 times.\n\n**Timeline \\(UTC\\)**\n\n2026-10-31 07:10 - An application begins sending raw data over its own data channel in a multi-region room. Media servers in US East and Canada restart within seconds. \n07:13 - Our monitoring detects the restarts and an incident is opened automatically. \n07:10 to 07:43 - Media servers in 12 data centers restart repeatedly as participants reconnect to new servers. \n07:43 - We block the traffic triggering the bug. There are no further restarts after this point, and customer impact ends. \n07:46 - A fix for the bug is ready. \n08:04 - The fix begins rolling out, starting with a small set of regions. \n08:49 - The fix is deployed to all regions.\n\n**Scope of Impact**\n\nRealtime sessions:\n\n* Across all regions, about 1.5% were disconnected more than once within five minutes.\n* In US East, US Central, US West, Canada and Mexico, about 6% were disconnected more than once within five minutes.\n* The median time from disconnect to rejoin was about 3 seconds.\n\nAgents:\n\n* Agents were disconnected and reconnected the same way as other participants, with the same median reconnect time of about 3 seconds. An agent that reconnects keeps its conversation state, so most conversations continued after a short gap. Fewer than 1% of active agent sessions did not recover.\n\nIngress, Egress and SIP:\n\n* Most active WHIP ingress streams were interrupted, and reconnects typically took about 40 seconds. Some WHIP ingress update and delete requests also failed.\n* About 5% of active egress sessions were disconnected. Most reconnected within a few seconds.\n* About 1% of SIP participants were disconnected, and half of them rejoined in under a second. Some SIP call transfers also failed.\n\nAPIs:\n\n* About 1.3% of Room Service API requests failed during the window. ListParticipants and GetParticipant were the most affected.\n\nHow to tell if you were affected: participants in affected sessions saw a reconnecting state, or a brief loss of audio and video. In session analytics, an affected participant appears to leave and rejoin the same room with the same identity between 07:10 and 07:43 UTC.\n\n**Detection and Response**\n\nOur monitoring detected the server restarts about 3 minutes after they began, and an incident was opened automatically. Identifying the specific room and traffic triggering the restarts took about 30 minutes. Once we found it, we blocked the trigger at 07:43 UTC and the restarts stopped immediately. We deployed a fix to all regions by 08:49 UTC.\n\nWe did not post to our status page while the incident was in progress, and published a notice only after it was resolved. Our original post mistakenly sent notifications as we initially over-estimated the scope of impact.\n\n**Mitigations and Follow-ups**\n\n* We fixed the bug. Media servers now handle forwarded data that arrives without the sender's identity instead of crashing. This fix was deployed to all regions by 08:49 UTC.\n* Raw data forwarded between media servers now includes the sender's identity, so receiving servers always know who sent it.\n\nWe appreciate your understanding and are committed to continuously improving our platform's reliability. If you have any questions, please reach out to our support team.",
"postmortem_body_last_updated_at": "2026-11-03T05:59:47.433-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-03T05:59:47.433-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-31T00:43:54.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/x6ny2r0ttg2b",
"started_at": "2026-10-31T00:43:54.000-07:00",
"status": "resolved",
"updated_at": "2026-11-03T05:59:47.458-08:00"
},
{
"components": [
{
"created_at": "2022-03-14T14:26:39.493-07:00",
"description": "LiveKit Cloud Dashboard (analytics, settings, and billing)",
"group_id": null,
"id": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"page_id": "1l77jzt47dnl",
"position": 2,
"showcase": true,
"start_date": "2022-03-14T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-04T20:49:10.078-08:00"
},
{
"created_at": "2024-04-09T13:31:08.657-07:00",
"description": "LiveKit Cloud SIP",
"group_id": "10pgb4w5dktp",
"id": "n1bl6t93gxjp",
"name": "US East - SIP",
"page_id": "1l77jzt47dnl",
"position": 2,
"showcase": true,
"start_date": "2024-03-02T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-28T18:35:22.181-07:00"
},
{
"created_at": "2025-02-12T04:43:36.321-08:00",
"description": "LiveKit Cloud SIP",
"group_id": "10pgb4w5dktp",
"id": "qt4ft6wpk1z2",
"name": "US Central - SIP",
"page_id": "1l77jzt47dnl",
"position": 9,
"showcase": true,
"start_date": "2025-02-12T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-28T18:35:52.889-07:00"
},
{
"created_at": "2024-08-26T15:05:04.869-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "vtht1l05qwwh",
"name": "US Central - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 12,
"showcase": true,
"start_date": "2024-08-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:13:59.073-07:00"
},
{
"created_at": "2025-07-23T07:55:29.462-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "nmmxjspd37t9",
"name": "US East - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 20,
"showcase": true,
"start_date": "2025-07-23T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:10.269-07:00"
}
],
"created_at": "2026-10-28T16:47:43.000-07:00",
"id": "vtwkvgmyjzmm",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved. Between 23:30 and 23:58 UTC, a network disruption at an upstream infrastructure provider affected our US East region. During this window:\n\nInbound SIP calls in US East failed at elevated rates, with affected calls rejected with 500 or 503 responses. Answered inbound calls were about 65% below normal across the window, and up to 90% below normal between 23:45 and 23:55 UTC.\nAbout 10% of outbound SIP call requests (CreateSIPParticipant) in US East failed, some returning 408 deadline_exceeded.\n\nBetween 23:28 and 23:58 UTC, there were elevated rates of Room Service, Egress, and Ingress API request server errors in US East.\nSome participants saw failed room joins or increased packet loss.\n\nThe LiveKit Cloud dashboard was inaccessible for part of this window.\n\nNetwork connectivity recovered at approximately 23:58 UTC, and error rates and call volumes returned to normal by 00:20 UTC. \n\nA related network issue also affected our much smaller Mexico region from 23:29 UTC, disconnecting active sessions and degrading service until traffic was moved to nearby regions at approximately 01:01 UTC. \n\nSincere apologies for the impact. We will follow up with a postmortem.",
"created_at": "2026-10-28T18:19:20.477-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T18:19:20.000-07:00",
"id": "q3m5mkpv1mtc",
"incident_id": "vtwkvgmyjzmm",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-01T08:28:36.370-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We've applied a mitigation for our Mexico region and services have returned to baseline. We are continuing to monitor and will update again within 15 minutes.",
"created_at": "2026-10-28T17:41:01.127-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T17:41:01.127-07:00",
"id": "hxsh35bvnjyw",
"incident_id": "vtwkvgmyjzmm",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T17:41:01.127-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "While the US regions still appear to be stable, we are tracking degraded services including SIP, RTC, and APIs in our Mexico region. We will update again as soon as possible.",
"created_at": "2026-10-28T17:36:48.492-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T17:36:48.492-07:00",
"id": "kth8vsdlqhjd",
"incident_id": "vtwkvgmyjzmm",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T17:36:48.492-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We believe the mitigation was successful and are seeing services return to normal. The dashboard is accessible and error rates have gone down across services. We are continuing to monitor to make sure that the situation is resolved. We will update again as soon as possible.",
"created_at": "2026-10-28T17:16:41.176-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T17:16:41.176-07:00",
"id": "kqpgzj1vv7yn",
"incident_id": "vtwkvgmyjzmm",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T17:16:41.176-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are still determining the full scope of impact, but we believe that the Room Service has also been impacted. We will update again within 15 minutes.",
"created_at": "2026-10-28T17:09:42.153-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T17:09:42.153-07:00",
"id": "j2vclz26xbtd",
"incident_id": "vtwkvgmyjzmm",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T17:09:42.153-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are currently attempting a mitigation. We will follow up with more information in the next 15 minutes.",
"created_at": "2026-10-28T17:00:03.232-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T17:00:03.232-07:00",
"id": "89f2g89p1cng",
"incident_id": "vtwkvgmyjzmm",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T17:00:03.232-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We believe there is also impact to inbound calls in the affected regions. We are working on identifying the cause and will update again within 15 minutes.",
"created_at": "2026-10-28T16:57:34.153-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T16:57:34.153-07:00",
"id": "3818h2nnw1rs",
"incident_id": "vtwkvgmyjzmm",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T16:57:34.153-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The LiveKit cloud dashboard is currently inaccessible. We are investigating this and will provide another update within 15 minutes.",
"created_at": "2026-10-28T16:51:41.083-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T16:51:41.083-07:00",
"id": "8h8xvx1hqz8s",
"incident_id": "vtwkvgmyjzmm",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T16:51:41.083-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n1bl6t93gxjp",
"name": "Regional SIP - US East - SIP",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "qt4ft6wpk1z2",
"name": "Regional SIP - US Central - SIP",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating an increased rate of failing outbound calls in US East. We will post another update with more information as soon as possible.",
"created_at": "2026-10-28T16:47:43.582-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T16:47:43.000-07:00",
"id": "wvr65gvnsbgt",
"incident_id": "vtwkvgmyjzmm",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-01T08:27:15.161-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-28T17:16:41.176-07:00",
"name": "Investigating failing calls and RoomService API failures in US East and dashboard issues",
"page_id": "1l77jzt47dnl",
"postmortem_body": "## Summary\n\nOn October 28, 2026, a network disruption at our cloud provider affected our US East data center. Starting at 23:28 UTC, services in US East lost the ability to reach our database and certain external services, such as model providers. Connections to end users and to our other regions kept working, so realtime media continued to flow.\n\nThis caused widespread failures across SIP calls, Room Service, Egress and Ingress APIs, and Cloud Agents in US East. A related issue affected our Mexico region for longer. Connectivity in US East came back at around 23:58 UTC. Mexico recovered at 01:01 UTC after we moved its traffic to nearby regions.\n\nOur automatic failover should have moved traffic away from US East, and it didn't. That's the main thing we're fixing.\n\nWe sincerely apologize to customers whose calls and agents were disrupted.\n\n## Impact\n\n**US East \\(23:28 to 23:58 UTC\\)**\n\n* **SIP inbound:** a large share of inbound calls failed, rejected with 500 or 503 responses. Answered inbound calls were about 65% below normal across the window, and up to 90% below normal between 23:45 and 23:55.\n* **SIP outbound:** about 10% of `CreateSIPParticipant` requests in US East failed, some returning 408 `deadline_exceeded`.\n* **APIs:** significant server errors on Room Service, Egress, and Ingress APIs in US East. Recent versions of our server SDKs automatically retry failed API requests in other regions, which reduced the impact for customers on those versions.\n* **Cloud Agents:** agents in US East couldn't reach some of the model providers and other services they depend on, so agent sessions failed or degraded. Cloud Agents currently run in a single region, so they couldn't be quickly moved.\n* **Realtime:** some participants in US East saw failed room joins or increased packet loss. Sessions that were already connected kept working.\n* **Dashboard:** the LiveKit Cloud dashboard was inaccessible for part of the window.\n\n**Mexico \\(23:29 to 01:01 UTC\\)**\n\nActive sessions were disconnected and service was degraded until we moved traffic to nearby regions.\n\n## Root Cause\n\nStarting at around 23:28 UTC, our cloud provider had a networking incident in its US East data center. It was a partial outage: our servers could still reach end users and our other regions, but lost access to our database and to certain external services, such as model providers.\n\nAnything that depended on those started failing. API requests and SIP call setup timed out, and agents couldn't reach the model providers they need to respond.\n\nOur automatic failover moves traffic away from a region when it becomes unhealthy. It didn't kick in here, for two reasons:\n\n1. **Our health checks mostly look at our own software.** None of it failed. Every server in US East was up and responding.\n2. **We could see that US East couldn't reach external services, but nothing acted on that automatically.** Those signals paged engineers, but they didn't feed into failover.\n\nAs far as failover was concerned, US East looked healthy, so we kept sending traffic to a region that couldn't do much of the work.\n\nCloud Agents had a separate gap. They currently run only in the data center specified in its deployment \\(i.e. US East\\) and we did not have a process to automatically migrate them quickly in case of a data center failure.\n\nConnectivity came back at around 23:58 UTC, once the provider mitigated the issue on their side.\n\n## Timeline\n\n\\(all times in UTC, starting October 28\\)\n\n* **23:28**: Requests from US East to our database and certain external services start timing out. API errors follow shortly after.\n* **23:35**: Alerts fire for API errors in US East and an incident is opened.\n* **23:42**: Incident call starts. We initially focus on the API timeouts.\n* **23:45**: We remove an affected server to rule out a single bad instance. Failures continue. Since SIP, APIs, and agents are all affected, we start treating it as a regional network problem rather than a software bug.\n* **23:56**: Incident escalated. We start draining US East.\n* **23:58**: The provider's fix restores connectivity in US East. Impact in US East ends.\n* **00:36**: We identify degraded SIP, realtime, and API service in our Mexico region.\n* **01:01**: Mexico traffic moved to nearby regions, and service there recovers.\n\n## Corrective Actions \u0026 Prevention\n\n* **Fail over based on external connectivity.** We're adding checks for whether a region can reach the services it depends on, and failover will act on them. If a region can't do its job, it should be drained automatically, even when every server in it looks healthy.\n* **Deploy Agents to multiple data centers.** We'll run agents in multiple data centers to have sufficient redundancy.\n* **Faster failover in agent dispatch.** If a server or an entire data center stops responding, agent dispatch will hand the job to another agent within seconds.\n* **Multi-cloud redundancy.** We're adding capacity with more than one cloud provider, so an outage at one provider can't take down a region.",
"postmortem_body_last_updated_at": "2026-11-01T07:53:55.431-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-01T07:53:55.431-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-28T18:19:20.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/rxfmxyfndqvq",
"started_at": "2026-10-28T16:47:43.000-07:00",
"status": "resolved",
"updated_at": "2026-11-01T08:28:36.422-08:00"
},
{
"components": [
{
"created_at": "2024-08-26T15:05:04.869-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "vtht1l05qwwh",
"name": "US Central - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 12,
"showcase": true,
"start_date": "2024-08-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:13:59.073-07:00"
},
{
"created_at": "2025-07-23T07:55:29.462-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "nmmxjspd37t9",
"name": "US East - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 20,
"showcase": true,
"start_date": "2025-07-23T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:10.269-07:00"
}
],
"created_at": "2026-10-20T16:09:18.000-07:00",
"id": "32478ky44148",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Between 21:30 and 23:17 UTC, roughly 10% of all WebRTC connections in our US East and US Central regions took 5 seconds or longer to establish. We applied a mitigation at 23:15 UTC, and join latencies have returned to normal by 23:17 UTC.\n\nWe will follow up with more details on the root cause and preventive actions in a postmortem. We apologize for the impact and appreciate your patience.",
"created_at": "2026-10-20T16:50:48.222-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T16:50:48.000-07:00",
"id": "27y2b5tqzm4l",
"incident_id": "32478ky44148",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T17:15:32.019-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We identified connection drops at load balancers in our US Central and US East regions that caused a subset of connection latencies to take 5 seconds or longer. We applied a mitigation at 23:15 UTC, and we are seeing connection speeds return to normal. \n\nWe are continuing to monitor before marking this resolved.",
"created_at": "2026-10-20T16:25:07.688-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-20T16:25:07.000-07:00",
"id": "wrwjbqqy754h",
"incident_id": "32478ky44148",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T17:13:27.354-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "nmmxjspd37t9",
"name": "Regional Real Time Communication - US East - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating higher than normal join latencies affecting a small percentage of participants in our US East and US Central regions. The latency increase appears to have begun at 21:30 UTC.\n\nAffected participants may notice increased join latencies higher than 5 seconds. The SDK retries automatically, and most of these joins succeed on retry. If your application stops waiting for an agent before then, those calls may time out.\n\nWe will share another update as we learn more.",
"created_at": "2026-10-20T16:09:18.287-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-20T16:09:18.000-07:00",
"id": "514l8mmkqvf3",
"incident_id": "32478ky44148",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T17:10:47.895-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-20T16:25:07.000-07:00",
"name": "Increased join latency in US East and US Central",
"page_id": "1l77jzt47dnl",
"postmortem_body": "**Root Cause** \nBetween 2026-10-16 and 2026-10-23, some new connections to LiveKit Cloud in US Central, US East and EU Central got no response. The cause was a fault in our cloud provider's network load balancers. Some load balancers kept sending new connections to servers that were no longer in service. Most clients connected on a retry, so the main effect was a slower join. Some clients waited much longer. A routine server replacement and one of our recovery actions on October 20 also closed open connections from self-hosted agent workers. This caused agent dispatch failures for some customers.\n\n**Technical Details** \nEach LiveKit Cloud region has network load balancers from our cloud provider. A load balancer sends new connections only to servers that pass its health check. When servers are added or removed, the provider updates the health-check configuration of the load balancer.\n\nOur cloud provider found these causes:\n\n1. A burst of health-check configuration updates overloaded the provider's health-check service.\n2. Some of these updates failed.\n3. A bug in the automatic recovery left the affected health checks inactive after the overload stopped.\n4. As a result, the load balancers kept old health data. They continued to send new connections to servers that were no longer listening. Those connections got no response.\n\nThe fault affected only some load balancers. Other load balancers with the same servers worked correctly. The load balancers reported no errors.\n\n**Agent dispatch failures.** Each self-hosted agent worker keeps one long-lived connection to LiveKit Cloud. The worker gets job offers on this connection. Two events on October 20 closed these connections:\n\n1. At 16:14, a routine server replacement in US Central stopped one of our ingress servers. This closed about 485 agent worker connections at the same time. At that time, new traffic did not go to US Central, so these workers reconnected to US East. The workers of each affected customer were then split between two regions. Until about 17:40, agent dispatch failed or was slow for some customers.\n2. At 23:15, we deleted the faulty load balancer in US Central. About 600 agent worker connections still used it. The workers reconnected within about 1 minute. But our servers did not detect that the old connections were closed. They continued to offer jobs on these connections until 23:32, when the connections timed out. Those job offers failed, and some jobs were not dispatched.\n\n**Timeline \\(UTC\\)** \n2026-10-16 06:50 - Connection timeouts started in EU Central \n2026-10-16 10:52 - We moved traffic away from the affected EU Central servers; the timeouts stopped. We opened a case with our cloud provider \n2026-10-17 - We added probes that test each load balancer address from several regions \n2026-10-20 08:00 - Slow agent joins started in US Central \n2026-10-20 15:28 - We moved traffic away from US Central; join times went back to normal \n2026-10-20 16:14 - A routine server replacement in US Central closed about 485 agent worker connections; agent dispatch failed or was slow for some customers until about 17:40 \n2026-10-20 21:30 - After traffic returned to US Central, slow joins started in US Central and US East \n2026-10-20 22:30 - We moved traffic away from US Central again \n2026-10-20 23:13 - We moved traffic away from US East \n2026-10-20 23:15 - We deleted the faulty load balancer in US Central; about 600 agent worker connections still used it \n2026-10-20 23:32 - The old agent worker connections timed out; agent dispatch went back to normal \n2026-10-20 23:41 - We replaced the faulty load balancer in US Central and moved traffic back \n2026-10-21 00:55 - We replaced the faulty load balancers in US East and moved traffic back \n2026-10-21 03:13 - The fault occurred on another load balancer in US Central; we moved traffic away \n2026-10-21 06:12 - We replaced that load balancer and moved traffic back to US Central \n2026-10-21 07:02 - We replaced the faulty load balancers in EU Central and moved traffic back \n2026-10-21 - Our cloud provider restored the correct health state on the 5 affected load balancers \n2026-10-23 15:15 - The fault occurred at a lower rate on one load balancer in US East \n2026-10-23 22:35 - Our probes detected it \n2026-10-23 23:10 - We moved traffic off that load balancer; customer impact ended \n2026-10-27 - Our cloud provider sent the root cause analysis\n\n**Scope of Impact** \nOn each affected load balancer, 5% to 25% of new connections got no response. The load balancer fault did not affect connections that were already open. But a routine server replacement and one of our recovery actions on October 20 closed open connections from self-hosted agent workers \\(see **Technical Details**\\).\n\n* Oct 16 06:50–10:52 - Region: EU Central - About 5% of new connections timed out. This included connections for SIP calls.\n* Oct 20 08:00 – Oct 21 06:15 - Region: US Central - A share of new connections timed out.\n* Oct 20 16:14–17:40 - Region: US Central, US East - Some self-hosted agents did not get jobs or joined late. Some inbound calls were not answered.\n* Oct 20 21:30 – Oct 21 00:55 - Region: US East - 5% to 10% of new connections timed out.\n* Oct 20 23:16–23:32 - Region: US Central - Job offers to some self-hosted agent workers timed out. Some jobs were not dispatched.\n* Oct 23 15:15–18:15 and 20:45–23:10 - Region: US East - About 0.3–0.5% of new connections to the region timed out.\n\nThe LiveKit SDKs stop a signal connection attempt after 5 seconds and try again. Most participants and agents joined on the retry, about 5 seconds later than usual.\n\nClients with a longer connection timeout, or with no timeout, waited longer. Some self-hosted agents reported room joins that took 60–70 seconds.\n\nAPI calls to the affected regions could also time out or be slow.\n\n**Detection and Response** \nCustomers reported the EU Central and US Central timeouts before our monitoring found them. Our checks tested each region as a whole, so a fault on one load balancer address was hidden. Most connections in a region still worked. On October 17, we added probes that test each load balancer address from several regions. These probes detected the October 23 fault. In each event, we moved traffic away from the affected region, replaced the faulty load balancers, and then moved traffic back.\n\n**Mitigations and Follow-ups**\n\n* Our cloud provider fixed the bug in a new release. With the fix, a failed health-check update is retried until it succeeds. The provider plans to start the deployment of this release in mid-November. We are working with the provider on when the fix reaches each region.\n* Each load balancer now sends connections only to the servers that run our ingress servers. Before, it included a larger group of servers, so each server that we added or removed caused a health-check update. Now the health checks change only when an ingress server moves. This change is live in most regions. We are completing it in the remaining regions, including US Central and US East.\n* Our probes now test each load balancer address and alert when the same connections fail again and again.\n* We changed how we replace a faulty load balancer. We now add a new load balancer, test it, and move traffic to it. The faulty load balancer stays in service until its open connections end, so agent workers on it are not cut off.\n* If an agent dispatch times out in one region, LiveKit Cloud now tries the next region.\n* We are working on a change that moves agent worker connections out of a region when we move traffic away from it. Then all agent workers of a customer reconnect to the same region.\n\nWe sincerely apologize for the disruption this caused. We appreciate your understanding and are committed to continuously improving our platform's reliability. If you have any questions, please reach out to our support team.",
"postmortem_body_last_updated_at": "2026-11-05T14:35:53.950-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-05T14:27:29.983-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-20T16:50:48.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/1jh7l7w7vk2z",
"started_at": "2026-10-20T16:09:18.000-07:00",
"status": "resolved",
"updated_at": "2026-11-05T14:35:53.955-08:00"
},
{
"components": [
{
"created_at": "2024-08-26T15:05:04.869-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "vtht1l05qwwh",
"name": "US Central - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 12,
"showcase": true,
"start_date": "2024-08-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:13:59.073-07:00"
}
],
"created_at": "2026-10-19T10:02:08.891-07:00",
"id": "5bckjlg3md8d",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "vtht1l05qwwh",
"name": "Regional Real Time Communication - US Central - Real Time Communication",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Between 16:35-17:03 UTC, Chicago region experienced about 11% failure rates in API errors, failed agent dispatches, and issues connecting to rooms. We faced another wave of the same issue between 17:47-17:53 UTC where the failure rates were less than 10%. Room Service APIs called from other regions targeting to a room hosted in Chicago region were also affected during this period. \nAPI requests made with recent server SDK versions are automatically retried in other regions, so customers on those versions should not have seen these errors.\nThe last related errors occurred at 17:53 UTC, and all affected services are operating normally since. We will share a detailed RCA with more information soon.",
"created_at": "2026-10-19T10:34:00.655-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-19T10:34:00.000-07:00",
"id": "121wh3m84514",
"incident_id": "5bckjlg3md8d",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T11:20:35.273-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "w6vlj3798sr9",
"name": "Global Real Time Communication",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented as of 17:00 UTC and we are seeing the errors return back to baseline. We are continuing to monitor before marking this resolved.",
"created_at": "2026-10-19T10:11:40.722-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-19T10:11:40.722-07:00",
"id": "8wnzkk66r152",
"incident_id": "5bckjlg3md8d",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T10:11:40.722-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "w6vlj3798sr9",
"name": "Global Real Time Communication",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating reports of elevated API errors across our services in the Chicago region.",
"created_at": "2026-10-19T10:02:09.151-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-19T10:02:09.000-07:00",
"id": "d4w8v8drmkqp",
"incident_id": "5bckjlg3md8d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T10:13:55.295-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-19T10:11:40.722-07:00",
"name": "Investigating reports of elevated API failures in Chicago region",
"page_id": "1l77jzt47dnl",
"postmortem_body": "## Summary\n\nOn October 19, 2026, between 16:34 and 17:03 UTC and again between 17:47 and 17:54 UTC, some room management, agent dispatch and SIP API requests failed for rooms hosted in our US Central \\(Chicago\\) region. Requests from other regions that targeted those rooms also failed. During these windows some rooms could not be created or joined, and some agents were not dispatched.\n\nThe cause was CPU exhaustion on the node running one of our message bus instances in US Central. A recently deployed overlay network used more CPU than it requested, and because our monitoring tracked CPU per pod rather than per node, we didn't see the node run out. That message bus instance began dropping messages, and a room-coordination service that depended on it stopped completing requests.\n\nWe have moved the message bus in US Central onto larger nodes and resized other regions at similar risk. Node-level CPU alerting and automatic recovery for the coordination service are in progress.\n\n## Impact\n\nBetween 16:34 and 17:03 UTC, and again from 17:47 to 17:54 UTC, room management \\(RoomService\\), agent dispatch \\(AgentDispatchService\\) and SIP APIs failed for some rooms hosted in US Central. Because every region relies on the same coordination step, requests from other US regions that targeted those rooms failed as well. Affected requests returned a 503.\n\nAgent dispatch was the most affected. During the first window, 26.5% of rooms that requested an agent did not receive one. During the second window, 17.5% did not.\n\nRooms hosted in other regions, and requests that did not target US Central rooms, were not affected.\n\n## Root Cause\n\nChanges to room state, such as creating a room or dispatching an agent, must be confirmed by several servers before they take effect. A coordination service gathers those confirmations over our internal messaging system. If a quorum doesn't arrive within one second, the change fails.\n\nWe recently rolled out a new overlay network for traffic between regions. It is more resilient to packet loss, but uses more CPU than its predecessor. In US Central, the message bus had little CPU headroom, and traffic was uneven: one of the three instances carried more than the other two combined, and its load had nearly doubled over the preceding hours.\n\nThe overlay runs on every node and carries each message the message bus sends or receives. To guard against packet loss, it sends each message over several relay paths, so its CPU use climbs with message traffic. So that time-sensitive packet delivery is never throttled, the overlay can use more CPU than it requests, and it runs at a higher scheduling priority than the message bus. On the busiest instance's node, the two together pushed CPU above 90% and starved the message bus.\n\nAt 16:34 UTC, routine node maintenance moved another messaging server to a new host, and its clients reconnected to the remaining servers. Shortly after, one coordination instance stopped getting replies through the overloaded server. New requests still reached it, but the responses it needed to finish them did not. Within four minutes it had stopped completing requests entirely.\n\nThe coordination service is built to tolerate the loss of an instance: when one fails, another takes over its work. But this instance had not failed outright. Some messages still reached it, so it kept reporting itself healthy, and callers kept waiting on it instead of moving on. They retried, and request volume to it rose more than tenfold. Its unfinished updates also blocked changes to the same rooms in other US regions.\n\nWe ended the first window by replacing the instance manually. The second window began when the instance that took over its busiest work stalled the same way, because the message bus was still short of CPU.\n\nOur monitoring did not catch the saturation. We monitored CPU for each pod against its own request, and relied on Kubernetes to keep nodes from being overloaded. That works only while every workload stays within its request. The message bus never exceeded its request, and the overlay's extra use did not show up in pod-level monitoring, so the node's saturation went unseen. Our error-rate alerts fired within four minutes, but nothing pointed to the node.\n\n## Timeline \\(UTC\\)\n\n* 16:34 - Routine node maintenance moves a messaging server in US Central to a new host. API failures begin for rooms hosted in the region.\n* 16:38 - Automated monitoring detects elevated error rates across several US regions, and our team begins investigating.\n* 16:55 - The affected room-coordination instance is identified and replaced.\n* 17:03 - Error rates return to baseline.\n* 17:35 - We mark the incident resolved on this status page. The root cause is not yet known.\n* 17:47 - Errors return as the instance that took over the busiest work degrades in the same way.\n* 17:52 - That instance is replaced. Error rates return to baseline by 17:54.\n* 17:59 - A further instance showing early signs of the same problem is replaced.\n* 18:14 - New rooms are placed in other regions while the investigation continues.\n* 20:53 - CPU exhaustion on the messaging host is identified as the root cause.\n* 21:36 - Messaging servers in US Central are moved to larger hosts.\n* 22:51 - US Central resumes hosting new rooms.\n\n## Corrective Actions \u0026 Prevention\n\n**Done**\n\n* **Increased CPU for the message bus in US Central.** The region's message bus now runs on larger nodes.\n* **Other regions resized.** We checked peak utilization across every region and moved messaging servers in regions at similar risk onto larger hosts.\n* **Automatic diagnostic capture.** The room-coordination service now records detailed diagnostics if a stall recurs.\n\n**Underway**\n\n* **Node-level CPU alerting.** We are adding CPU alerts for each node that carries internal traffic, alongside our pod-level monitoring, and alerts on CPU scheduling delays.\n* **Deadline-miss alerting.** A room-coordination instance that keeps missing its deadline will page our team.\n* **Automatic recovery for room coordination.** An instance that stops completing requests will hand off its work, and callers will route away from unhealthy instances.\n* **Retry limits.** Room-coordination requests will use backoff and a retry budget, so a stalled instance can no longer multiply request volume across regions.\n* **Messaging client reconnects.** Clients will reconnect automatically when a subscription stops receiving messages.\n* **Reserved CPU for the network overlay.** Hosts that run our messaging servers will reserve CPU for the overlay, so it cannot starve the servers it carries.\n\nWe sincerely apologize for the disruption. If you have questions or were affected in ways not described here, please reach out to our support team.",
"postmortem_body_last_updated_at": "2026-10-21T17:52:42.641-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-21T17:52:42.641-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-19T10:34:00.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/7tjjp1kgpkss",
"started_at": "2026-10-19T10:02:09.000-07:00",
"status": "resolved",
"updated_at": "2026-10-21T17:52:42.662-07:00"
},
{
"components": [
{
"created_at": "2024-04-09T13:31:08.432-07:00",
"description": "LiveKit Cloud SIP",
"group_id": "10pgb4w5dktp",
"id": "39v8x79rxsz0",
"name": "US West - SIP",
"page_id": "1l77jzt47dnl",
"position": 1,
"showcase": true,
"start_date": "2024-03-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-19T13:49:14.986-07:00"
}
],
"created_at": "2026-10-16T09:04:40.000-07:00",
"id": "03776y0cfgxq",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "39v8x79rxsz0",
"name": "Regional SIP - US West - SIP",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved. Between 14:49 UTC and 15:58 UTC, approximately 14% of outbound calls and 5% of inbound calls in US West failed. We will follow up with more details surrounding the root cause and preventative actions in a postmortem.",
"created_at": "2026-10-16T10:30:21.050-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T10:30:21.050-07:00",
"id": "g10c0vzfsqlj",
"incident_id": "03776y0cfgxq",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T10:30:21.050-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "39v8x79rxsz0",
"name": "Regional SIP - US West - SIP",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We estimate that approximately 14% of outbound calls and 5% of inbound calls in US West failed between 14:49 UTC and 15:58 UTC. We will follow up as soon as possible with more details.",
"created_at": "2026-10-16T09:57:35.418-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-16T09:57:35.000-07:00",
"id": "d1fkr217r8pn",
"incident_id": "03776y0cfgxq",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T10:27:42.103-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "39v8x79rxsz0",
"name": "Regional SIP - US West - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We applied mitigation as of 15:58 UTC and SIP outbound connection success has returned to normal. We are continuing to monitor before marking this resolved.",
"created_at": "2026-10-16T09:35:19.072-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-16T09:35:19.000-07:00",
"id": "qw3b61mbty60",
"incident_id": "03776y0cfgxq",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T09:53:39.467-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "39v8x79rxsz0",
"name": "Regional SIP - US West - SIP",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have identified the source of the issue and are working to route around the impact.",
"created_at": "2026-10-16T09:12:53.017-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-16T09:12:53.017-07:00",
"id": "wjlmz56gxf1l",
"incident_id": "03776y0cfgxq",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T09:12:53.017-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "39v8x79rxsz0",
"name": "Regional SIP - US West - SIP",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating degraded SIP performance in US West starting at 14:49 UTC affecting new outbound calls.",
"created_at": "2026-10-16T09:04:40.195-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T09:04:40.000-07:00",
"id": "xdl3pfdsb6h3",
"incident_id": "03776y0cfgxq",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T09:57:46.784-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-16T09:35:19.000-07:00",
"name": "Degraded SIP Performance in US West",
"page_id": "1l77jzt47dnl",
"postmortem_body": "**Root Cause** \nOn 2026-10-16, a subset of SIP calls in our US West region failed. Two of our SIP servers advertised IPv6 addresses for signaling and media, which many carrier providers don't support. Carriers rejected outbound calls routed through these servers, and inbound calls were answered but dropped almost immediately without audio. We had recently started running SIP in a US West cluster that uses both IPv4 and IPv6 networking and our SIP servers didn't enforce IPv4 addresses.\n\n**Technical Details** \nWhen a SIP server starts, it determines its public address by querying an external address-lookup service. On the two affected servers, our primary lookup service didn't respond in time so they fell back to a backup service that returned their IPv6 address instead. Nothing in the SIP server checked that the address it advertised was IPv4.\n\n**Timeline \\(UTC\\)** \n2026-10-16 14:49 - First affected SIP server started with an IPv6 address; call failures began \n2026-10-16 15:41 - Second affected SIP server started with an IPv6 address \n2026-10-16 ~15:48 - Customers began reporting failures to our support team \n2026-10-16 15:58 - SIP traffic began moving away from the final affected cluster due to automatic cluster draining \n2026-10-16 15:59 - Last affected call; customer impact ended \n2026-10-16 17:08 - Began rolling out the long term fix requiring the usage of IPv4 \n2026-10-16 ~21:28 - Fix rollout completed \n2026-10-16 ~22:30 - SIP traffic restored to the affected cluster\n\n**Scope of Impact** \nBetween 14:49 and 15:59 UTC on 2026-10-16, SIP calls handled by the two affected servers in US West failed. During that window, about 14% of outbound calls in US West were rejected at setup and about 5% of inbound calls were answered and then dropped without audio.\n\n**Detection and Response** \nOur customers noticed this before our monitoring did. Our alerts watch for error-level logs, but carrier rejections are logged as routine events since they happen during normal operation. We also weren't tracking call failure rates on individual servers so our on-call engineers were paged in manually about an hour after the first failure.\n\n**Mitigations and Follow-ups**\n\n* We deployed a fix so SIP servers only use IPv4 addresses when determining their public address.\n* We'll add safeguards and automated tests to ensure a SIP server's call setup messages only contain IPv4 addresses.\n* We are additionally adding improvements in monitoring so that we can proactively detect and prevent these sorts of failures.\n\nWe apologize for the disruption this caused. We appreciate your understanding and are committed to continuously improving our platform's reliability. If you have any questions, please reach out to our support team.",
"postmortem_body_last_updated_at": "2026-10-22T07:26:35.068-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T07:26:35.068-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-16T10:30:21.050-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/dwszh1v29gsq",
"started_at": "2026-10-16T09:04:40.000-07:00",
"status": "resolved",
"updated_at": "2026-10-22T07:26:35.093-07:00"
},
{
"components": [
{
"created_at": "2025-07-23T07:55:29.532-07:00",
"description": "LiveKit Cloud API and media backend",
"group_id": "wnz36vpsr8r6",
"id": "g3swm83zqq5b",
"name": "Europe Central - Real Time Communication",
"page_id": "1l77jzt47dnl",
"position": 21,
"showcase": true,
"start_date": "2025-07-23T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T09:14:59.837-07:00"
}
],
"created_at": "2026-10-16T03:55:03.000-07:00",
"id": "9yj8wctx3393",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "g3swm83zqq5b",
"name": "Regional Real Time Communication - Europe Central - Real Time Communication",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident is resolved. Connections to the Frankfurt region have returned to normal.",
"created_at": "2026-10-16T04:59:12.984-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T04:59:12.984-07:00",
"id": "jjc67f3wpbk4",
"incident_id": "9yj8wctx3393",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T04:59:12.984-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "g3swm83zqq5b",
"name": "Regional Real Time Communication - Europe Central - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have applied a mitigation and new connections should no longer time out. We are monitoring to confirm that connection times have returned to normal.",
"created_at": "2026-10-16T04:15:28.423-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T04:15:28.000-07:00",
"id": "kxn8g92hc4qt",
"incident_id": "9yj8wctx3393",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T05:29:10.197-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "g3swm83zqq5b",
"name": "Regional Real Time Communication - Europe Central - Real Time Communication",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating intermittent timeouts when establishing new connections to our Frankfurt (EU) region. A small percentage of connection attempts may time out. We are working on a mitigation and will post an update as soon as possible.",
"created_at": "2026-10-16T03:55:03.962-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T03:55:03.000-07:00",
"id": "h3yywqmvxryj",
"incident_id": "9yj8wctx3393",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T05:28:42.917-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-16T04:15:28.000-07:00",
"name": "Intermittent connection timeouts in the Frankfurt region",
"page_id": "1l77jzt47dnl",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-16T04:59:12.984-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/dqmd6knx66n1",
"started_at": "2026-10-16T03:55:03.000-07:00",
"status": "resolved",
"updated_at": "2026-10-16T05:29:10.238-07:00"
},
{
"components": [
{
"created_at": "2022-03-14T14:26:39.493-07:00",
"description": "LiveKit Cloud Dashboard (analytics, settings, and billing)",
"group_id": null,
"id": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"page_id": "1l77jzt47dnl",
"position": 2,
"showcase": true,
"start_date": "2022-03-14T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-04T20:49:10.078-08:00"
}
],
"created_at": "2026-10-13T03:01:02.000-07:00",
"id": "x45ytm36m8c3",
"impact": "critical",
"impact_override": "critical",
"incident_updates": [
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved. Between 08:15 and 12:35 UTC, agents were not able to upload agent observability data to LiveKit's servers in all regions. Agent sessions completed normally, but, unfortunately, any Agent Insights data which attempted to upload within that window (traces/transcripts/logs/recordings) is unrecoverable. The ability to download existing Agent Insights data was impaired until 13:15 UTC. Agent Simulations were unavailable due to a upstream dependency and came back online at 13:25 UTC. \n\nSincere apologies for the impact. We will follow up with a postmortem as soon as possible.",
"created_at": "2026-10-13T06:44:11.992-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T06:44:11.000-07:00",
"id": "5mygp4m4n1n9",
"incident_id": "x45ytm36m8c3",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T06:49:03.589-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Agent Insights uploads are now operational and session data has been storing normally since 12:40 UTC. We are still investigating an issue with Simulations. Simulation runs may complete without a transcript, summary, or verdict. We will post an update shortly.",
"created_at": "2026-10-13T06:15:54.554-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T06:15:54.000-07:00",
"id": "2mzlyzvkd9mt",
"incident_id": "x45ytm36m8c3",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T06:28:32.385-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-10-13T04:40:53.101-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T04:40:53.101-07:00",
"id": "8bv0hzqtv0v0",
"incident_id": "x45ytm36m8c3",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T04:40:53.101-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We have identified the cause of the Agent Insights issue and are working on a fix.",
"created_at": "2026-10-13T03:45:01.126-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T03:45:01.000-07:00",
"id": "22gjdtgrqxnf",
"incident_id": "x45ytm36m8c3",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T06:28:12.889-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "mv82wwf02vw4",
"name": "Cloud Dashboard (cloud.livekit.io)",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are investigating an issue with Agent Insights where agents are unable to upload their observability data at the end of their sessions. Some recent agent sessions may be missing from the dashboard's sessions list or shown as “In progress”. Agent sessions themselves are not affected. We will post an update shortly.",
"created_at": "2026-10-13T03:01:02.979-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T03:01:02.000-07:00",
"id": "mg2vxq4ql4mr",
"incident_id": "x45ytm36m8c3",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T06:29:44.677-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Observability upload and Agent Simulations failures",
"page_id": "1l77jzt47dnl",
"postmortem_body": "**Root Cause**\n\nDuring a security rotation of access credentials, we revoked a storage credential that we believed was no longer in use. It was still being used by our observability platform, which stores the data behind the dashboard and the Analytics API, the Agent Insights data that agents upload at the end of each session, and by the service that serves that data back for downloads and Agent Simulations. Once the credential was revoked, uploads failed in all regions.\n\nRestoring service took longer than a credential swap normally would for three reasons:\n\n* New credentials had to be set up, and each affected service had to be updated to use them.\n* The new credentials took longer than expected to become available in all regions \\(about two hours in our US East region\\).\n* Once the observability platform was working again, we found additional services that also needed updating. The incident was resolved once they were updated.\n\n**Timeline \\(UTC\\)**\n\n* 2026-10-13 08:13 - A storage credential is revoked during a security rotation. Observability uploads begin failing in all regions.\n* 08:15 - Agent Insights transcript and audio downloads and Agent Simulations begin failing.\n* 09:39 - Our team detects the failures and opens an incident.\n* 10:45 - The revoked credential is identified as the cause.\n* 11:16 - Affected services are reconfigured to use a new credential.\n* 12:40 - The new credential takes effect in the last region. Observability uploads recover in all regions.\n* 13:13 - Agent Insights downloads recover.\n* 13:26 - Agent Simulations recover.\n* 13:30 - Incident resolved.\n* 2026-10-17 - Recovery of observability data from the impact window completes \\(see below\\).\n\n**Scope of Impact**\n\n* **Observability uploads \\(08:13 to 12:40 UTC, all regions\\):** Observability data written during this window, including the data behind the dashboard and the Analytics API, failed to store. Nearly all agent sessions that ended in this window also failed to upload their Agent Insights data, so in Agent Insights these sessions were missing from the sessions list, or shown as \"In progress\". Agent sessions themselves ran normally, and media and calls were not affected.\n* **Data recovery:** In our resolved update we said all Agent Insights data from this window was unrecoverable. We have since restored most of it. **Session recordings and transcripts from this window are permanently lost.** All other observability data for these sessions has been restored and is visible in the dashboard and through the Analytics API.\n* **Agent Insights downloads \\(08:15 to 13:13 UTC\\):** Downloading transcripts and audio for existing sessions failed during the incident window.\n* **Agent Simulations \\(08:15 to 13:26 UTC\\):** Simulation runs couldn't load conversation transcripts, so they finished without a summary or verdict, or were cancelled. Runs from this window need to be re-run.\n* **Billing:** Agent Insights usage is only recorded after a successful upload, so failed uploads were **not** billed.\n\n**Detection and Response**\n\nWe detected this issue about 85 minutes after it began. We did not have alerting on upload failures to observability storage, so it was caught when an engineer noticed elevated errors rather than by an automated alert. We also initially marked the incident resolved once uploads recovered, before confirming that Agent Simulations and downloads had recovered too.\n\n**Mitigations and Follow-ups**\n\n* We have added alerting that pages our on-call team when observability storage uploads fail, along with a continuous check that stored data can still be read and downloaded and an end-to-end test that verifies our storage credentials still work.\n* We have built tooling to replay and restore observability data from a past window, which we used to recover data from this incident.\n* We are moving these services to dedicated service credentials managed with our infrastructure and auditing other services for the same pattern.\n* We have updated our credential rotation process to verify every consumer of a credential before revoking it, and to account for regional propagation delays when rolling out replacements.\n* We are adding redundancy so a single failed credential can no longer stop observability uploads.\n\nWe sincerely apologize for the disruption. We appreciate your understanding and are committed to continuously improving our platform's reliability. If you have any questions, please reach out to our support team.",
"postmortem_body_last_updated_at": "2026-10-21T16:44:47.610-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-21T16:44:47.610-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-13T06:44:11.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/0tnknwzkh37q",
"started_at": "2026-10-13T03:01:02.000-07:00",
"status": "resolved",
"updated_at": "2026-10-21T16:44:47.633-07:00"
},
{
"components": [
{
"created_at": "2025-08-08T16:51:06.436-07:00",
"description": "LiveKit Cloud Agents",
"group_id": "jnv1myhtwskd",
"id": "rg84mxb77y57",
"name": "US East - Cloud Agents",
"page_id": "1l77jzt47dnl",
"position": 1,
"showcase": true,
"start_date": "2025-08-08T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-05T21:52:15.931-07:00"
}
],
"created_at": "2026-10-05T20:46:14.159-07:00",
"id": "jyck45xf97pd",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "rg84mxb77y57",
"name": "Regional Cloud Agents - US East - Cloud Agents",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix was applied and all builds are back to normal.",
"created_at": "2026-10-05T21:52:16.010-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-05T21:52:16.010-07:00",
"id": "16186zpd2c9v",
"incident_id": "jyck45xf97pd",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-05T21:52:16.010-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "rg84mxb77y57",
"name": "Regional Cloud Agents - US East - Cloud Agents",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating an issue causing some cloud agent builds to be backed up in US East. This issue only impacts new builds for a small number of customers, existing agents are running without any impact.",
"created_at": "2026-10-05T20:46:14.255-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-05T20:46:14.255-07:00",
"id": "7wd65fgk2pjg",
"incident_id": "jyck45xf97pd",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-05T20:46:14.255-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Investigating delayed cloud agent builds in US East",
"page_id": "1l77jzt47dnl",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-05T21:52:16.010-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/mtsdwnnrj9xk",
"started_at": "2026-10-05T20:46:14.003-07:00",
"status": "resolved",
"updated_at": "2026-10-06T09:05:13.381-07:00"
}
],
"page": {
"id": "1l77jzt47dnl",
"name": "LiveKit",
"url": "https://status.livekit.io"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}