Snapshot 11270
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"w7n2r3wjq1w7",
"xdq19lkmjm66",
"dhn9vbms75d8",
"p3pq4zsh46r9",
"6xrf32p25fkd",
"2sjkthtj0w4b",
"ft27jgbbxb82",
"ysnph04qrvdd",
"2sssj0j1hwrh",
"jms6czc33yj2",
"mp7hxmkqm09w",
"l2lh7w90l681",
"ycyjc23km689",
"llsghyxnxs2v",
"8hfy5fr31g27",
"gbgpxh2kfcpk"
],
"created_at": "2019-11-22T13:05:13.444-05:00",
"description": null,
"group_id": null,
"id": "gmwvdm9hg6dr",
"name": "Prod1 (US)",
"page_id": "lhn2sb4njk08",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2019-11-22T13:06:24.268-05:00"
},
{
"components": [
"gwd39q9lp69b",
"x1qh9c5zjq9m",
"c52wr1364949",
"s0v2msb48f9g",
"8dkd2yjf7z5t",
"qg912by4hnfw",
"q3zhkgm7n4c6",
"c6y4d79nm7gj",
"yb4zp1d65cqg",
"95t141ydgn9b",
"9t994vfyf9bs",
"5zz3dqntsjlp",
"1pd5346fbvfw",
"s415bym88sdy",
"vh17f3xfs7c9",
"hg9kv70m4kt6"
],
"created_at": "2019-11-22T13:01:14.823-05:00",
"description": null,
"group_id": null,
"id": "w2qyf5v9gm38",
"name": "Prod2 (EU)",
"page_id": "lhn2sb4njk08",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2019-11-22T13:06:29.676-05:00"
},
{
"components": [
"0dl0l8d1f6pq",
"rkq98sw3w7yk",
"84r15bhg8m6w",
"4mtf1tsdrmdb",
"c7zvz4xvhs6q",
"q8p7kz8cfgsj",
"wbzwfzqdhqht",
"2ll3dds32n2t",
"qxsf5k9q4t88",
"xbf1tf0w2m51",
"gl848w2z3p0t",
"jkj1sw2c7ntf",
"5prfpzrdzjc2",
"by9tzc6g9htz",
"wsbfqtxwkkgw",
"nks9jkpwfv1v"
],
"created_at": "2025-03-14T14:14:02.469-04:00",
"description": null,
"group_id": null,
"id": "vz4171pkd3m7",
"name": "Prod4 (IN)",
"page_id": "lhn2sb4njk08",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-03-14T14:14:10.212-04:00"
},
{
"components": [
"xl3frs6395qd"
],
"created_at": "2024-12-23T15:58:31.514-05:00",
"description": null,
"group_id": null,
"id": "9n21n0kt57yk",
"name": "Kustomer Voice \u0026 Text",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-03-14T14:14:10.222-04:00"
},
{
"components": [
"n9lty2bmvn86",
"7f1fcvk68k2j",
"n42f9nwfz0tn",
"35hgy37dbjy6",
"s5375wvv9pyt",
"kz5v8kb60zfv"
],
"created_at": "2021-03-17T16:40:59.294-04:00",
"description": null,
"group_id": null,
"id": "2whbcrvv0534",
"name": "Third Party",
"page_id": "lhn2sb4njk08",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-03-14T14:14:10.234-04:00"
},
{
"created_at": "2022-06-13T10:59:55.788-04:00",
"description": null,
"group_id": null,
"id": "jxsc98gvplgl",
"name": "Regional Incident",
"page_id": "lhn2sb4njk08",
"position": 6,
"showcase": true,
"start_date": "2022-06-13T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2025-10-20T06:18:25.120-04:00"
}
],
"incidents": [
{
"components": [
{
"created_at": "2024-09-03T17:57:24.905-04:00",
"description": null,
"group_id": "9n21n0kt57yk",
"id": "xl3frs6395qd",
"name": "Kustomer Voice",
"page_id": "lhn2sb4njk08",
"position": 1,
"showcase": true,
"start_date": "2024-09-03T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-09-09T15:52:08.576-04:00"
}
],
"created_at": "2026-09-09T15:25:29.462-04:00",
"id": "nspp539dxsv3",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "xl3frs6395qd",
"name": "Kustomer Voice \u0026 Text - Kustomer Voice",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has resolved an event affecting Kustomer Voice across all PRODs. To resolve this issue, our team has rolled back an update.\n\nAfter careful monitoring, our team has determined that all affected areas are now fully restored. Please reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-09-09T15:52:08.604-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-09T15:52:08.604-04:00",
"id": "794bb52p71qf",
"incident_id": "nspp539dxsv3",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-09T15:52:08.604-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xl3frs6395qd",
"name": "Kustomer Voice \u0026 Text - Kustomer Voice",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has implemented an update to address an event affecting Kustomer Voice across all PRODs.\n\nOur team is currently monitoring this update to ensure the issue is fully resolved. In order to fully resolve the issue on your end please have all agents refresh their Kustomer Instance.",
"created_at": "2026-09-09T15:45:04.921-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-09T15:45:04.921-04:00",
"id": "s092q23tk7h4",
"incident_id": "nspp539dxsv3",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-09T15:45:04.921-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xl3frs6395qd",
"name": "Kustomer Voice \u0026 Text - Kustomer Voice",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer has identified an event affecting Kustomer Voice across all PRODS that may cause issues when attempting to make outbound calls.\n\nOur team is currently working to implement a resolution. Please expect further updates within the next 30 minutes.",
"created_at": "2026-09-09T15:25:29.533-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-09T15:25:29.533-04:00",
"id": "t70bxh0rrvyk",
"incident_id": "nspp539dxsv3",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-09T15:25:29.533-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-09T15:45:04.921-04:00",
"name": "[Kustomer Voice] Unable to Dial Outbound Calls - All PRODS",
"page_id": "lhn2sb4njk08",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-09T15:52:08.604-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/pvs6npy0q0zj",
"started_at": "2026-09-09T15:25:29.453-04:00",
"status": "resolved",
"updated_at": "2026-09-09T15:52:08.618-04:00"
},
{
"components": [
{
"created_at": "2019-05-24T04:37:45.742-04:00",
"description": null,
"group_id": "gmwvdm9hg6dr",
"id": "2sjkthtj0w4b",
"name": "CSAT",
"page_id": "lhn2sb4njk08",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-08T09:39:02.158-04:00"
},
{
"created_at": "2019-11-22T13:19:56.726-05:00",
"description": null,
"group_id": "w2qyf5v9gm38",
"id": "qg912by4hnfw",
"name": "CSAT",
"page_id": "lhn2sb4njk08",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-08T09:39:02.176-04:00"
},
{
"created_at": "2025-03-14T14:15:22.418-04:00",
"description": null,
"group_id": "vz4171pkd3m7",
"id": "q8p7kz8cfgsj",
"name": "CSAT",
"page_id": "lhn2sb4njk08",
"position": 6,
"showcase": true,
"start_date": "2025-03-14T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-09-08T09:39:02.190-04:00"
},
{
"created_at": "2021-04-19T10:54:21.876-04:00",
"description": null,
"group_id": "gmwvdm9hg6dr",
"id": "2sssj0j1hwrh",
"name": "Knowledge base",
"page_id": "lhn2sb4njk08",
"position": 9,
"showcase": true,
"start_date": "2021-04-19T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-09-08T09:39:02.203-04:00"
},
{
"created_at": "2021-04-19T10:54:35.020-04:00",
"description": null,
"group_id": "w2qyf5v9gm38",
"id": "yb4zp1d65cqg",
"name": "Knowledge base",
"page_id": "lhn2sb4njk08",
"position": 9,
"showcase": false,
"start_date": "2021-04-19T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-09-08T09:39:02.217-04:00"
},
{
"created_at": "2025-03-14T14:17:53.681-04:00",
"description": null,
"group_id": "vz4171pkd3m7",
"id": "qxsf5k9q4t88",
"name": "Knowledge base",
"page_id": "lhn2sb4njk08",
"position": 9,
"showcase": true,
"start_date": "2025-03-14T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-09-08T09:39:02.244-04:00"
}
],
"created_at": "2026-09-08T09:23:00.169-04:00",
"id": "x5bf49hdf3s4",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "qg912by4hnfw",
"name": "Prod2 (EU) - CSAT",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "yb4zp1d65cqg",
"name": "Prod2 (EU) - Knowledge base",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "2sjkthtj0w4b",
"name": "Prod1 (US) - CSAT",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "2sssj0j1hwrh",
"name": "Prod1 (US) - Knowledge base",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "q8p7kz8cfgsj",
"name": "Prod4 (IN) - CSAT",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "qxsf5k9q4t88",
"name": "Prod4 (IN) - Knowledge base",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has resolved an event affecting links and content utilising the default kustomer.help domain (not custom client domains) that caused visibility issues.\n\nAfter careful monitoring, our team has determined that all affected areas are now fully restored. Please reach out to Kustomer Support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-09-08T09:39:02.271-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-08T09:39:02.271-04:00",
"id": "zkg67j8p0587",
"incident_id": "x5bf49hdf3s4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-08T09:39:02.271-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "qg912by4hnfw",
"name": "Prod2 (EU) - CSAT",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "yb4zp1d65cqg",
"name": "Prod2 (EU) - Knowledge base",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "2sjkthtj0w4b",
"name": "Prod1 (US) - CSAT",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "2sssj0j1hwrh",
"name": "Prod1 (US) - Knowledge base",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "q8p7kz8cfgsj",
"name": "Prod4 (IN) - CSAT",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "qxsf5k9q4t88",
"name": "Prod4 (IN) - Knowledge base",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-09-08T09:24:04.179-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-08T09:24:04.179-04:00",
"id": "br6ft8txf7vh",
"incident_id": "x5bf49hdf3s4",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-08T09:24:04.179-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "qg912by4hnfw",
"name": "Prod2 (EU) - CSAT",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "yb4zp1d65cqg",
"name": "Prod2 (EU) - Knowledge base",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "2sjkthtj0w4b",
"name": "Prod1 (US) - CSAT",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "2sssj0j1hwrh",
"name": "Prod1 (US) - Knowledge base",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q8p7kz8cfgsj",
"name": "Prod4 (IN) - CSAT",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qxsf5k9q4t88",
"name": "Prod4 (IN) - Knowledge base",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Kustomer is aware of an event affecting links and content utilising the default kustomer.help domain (not custom client domains) — including Help Center URLs and CSAT surveys — that may cause visibility issues.\n\nOur team is currently working to identify the cause of this issue in an effort to implement a resolution. Please expect additional updates within the next 30 minutes, please reach out to Kustomer Support at [email protected] for any further questions or updates.",
"created_at": "2026-09-08T09:23:00.252-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-08T09:23:00.252-04:00",
"id": "7mz18wgzn7rj",
"incident_id": "x5bf49hdf3s4",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-08T09:23:00.252-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "kustomer.help Default Domain – Visibility Issue – ALL PRODs",
"page_id": "lhn2sb4njk08",
"postmortem_body": "# **Summary**\n\nOn September 7–8, 2026, some customers were unable to access Knowledge Base pages and related survey links hosted on the default `*.kustomer.help` domain. Customers using the default Knowledge Base domain in the prod1 environment encountered browser certificate warnings that prevented access to Knowledge Base content. Related links using the same domain, including some survey links, were also affected.\n\nCustomers using custom Knowledge Base domains and shared Knowledge Base domains in other production environments were not affected.\n\n# **Root Cause**\n\nA security certificate used by the shared Knowledge Base domain expired before its renewal validation was completed. This caused browsers to reject connections to links served through that domain, while the underlying Knowledge Base service remained available.\n\n# **Timeline**\n\n**Sep 7, 2026**\n\n**8:00 PM ET**: Kustomer’s default Knowledge Base Domain certificate expired. At this time, customers may have begun encountering certificate warnings when accessing Knowledge Base content.\n\n\n\n**Sep 8, 2026**\n\n**8:37 AM ET** Kustomer receives reports of certificate warnings for default Knowledge Base Domain content and related links\n\n**8:38 AM ET** Kustomer Engineering is notified and begins our On-Call Incident Management process\n\n**9:00 AM ET** Kustomer Engineering identifies the shared-domain certificate issue and begins remediation steps\n\n**9:24 AM ET** Kustomer Engineering completes remediation steps\n\n**9:37 AM ET** Kustomer’s certificate provider acknowledges the remediation steps were complete and provisions a new certificate for the default Knowledge Base domain, restoring service\n\n# **Lessons/Improvements**\n\nKustomer Engineering has a dedicated Infrastructure team that continuously works to maintain and improve Kustomer’s availability, reliability, fault tolerance. As part of our continual investment in these areas, we have identified the following action items related to this issue:\n\n* Move the shared-domain certificate to an automated renewal approach.\n* Add proactive monitoring and alerting for certificate expiration across shared customer-facing domains.\n* Review other certificates using the same renewal approach and prioritize any needed updates.\n* Document recovery procedures and ownership for certificate renewal activities.",
"postmortem_body_last_updated_at": "2026-09-10T17:06:26.570-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-10T17:06:26.570-04:00",
"reminder_intervals": null,
"resolved_at": "2026-09-08T09:39:02.271-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/4jvkzxd0krwh",
"started_at": "2026-09-08T09:23:00.131-04:00",
"status": "resolved",
"updated_at": "2026-09-10T17:06:26.585-04:00"
},
{
"components": [
{
"created_at": "2016-10-27T17:51:50.893-04:00",
"description": "Chat Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "p3pq4zsh46r9",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.821-04:00"
},
{
"created_at": "2016-10-27T17:52:11.810-04:00",
"description": "In-app and email notification services",
"group_id": "gmwvdm9hg6dr",
"id": "jms6czc33yj2",
"name": "Notifications",
"page_id": "lhn2sb4njk08",
"position": 10,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.833-04:00"
},
{
"created_at": "2016-10-27T17:53:07.354-04:00",
"description": "Event Tracking Services",
"group_id": "gmwvdm9hg6dr",
"id": "ycyjc23km689",
"name": "Tracking",
"page_id": "lhn2sb4njk08",
"position": 13,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.845-04:00"
},
{
"created_at": "2021-03-19T11:27:46.503-04:00",
"description": "Realtime Communication Platform and realtime infrastructure-as-a-service.",
"group_id": "2whbcrvv0534",
"id": "n9lty2bmvn86",
"name": "PubNub",
"page_id": "lhn2sb4njk08",
"position": 13,
"showcase": false,
"start_date": "2021-03-19T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-08-25T10:17:26.856-04:00"
},
{
"created_at": "2016-10-27T17:53:19.972-04:00",
"description": "Frontend Web Client Single Page Application",
"group_id": "gmwvdm9hg6dr",
"id": "llsghyxnxs2v",
"name": "Web Client",
"page_id": "lhn2sb4njk08",
"position": 14,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.868-04:00"
}
],
"created_at": "2026-08-25T09:51:38.639-04:00",
"id": "8l2cbp53ppfh",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "jms6czc33yj2",
"name": "Prod1 (US) - Notifications",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "ycyjc23km689",
"name": "Prod1 (US) - Tracking",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "llsghyxnxs2v",
"name": "Prod1 (US) - Web Client",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "n9lty2bmvn86",
"name": "Third Party - PubNub",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "The PubNub-related event previously reported affecting real-time messaging for orgs in Prod1 has been resolved. PubNub has confirmed the issue has been mitigated on their end - no messages were missed and no customer impact occurred during the event.\n\nAll affected areas remain fully operational. Please reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-08-25T10:17:26.892-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T10:17:26.892-04:00",
"id": "2r256pf1cvgc",
"incident_id": "8l2cbp53ppfh",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T10:17:26.892-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "jms6czc33yj2",
"name": "Prod1 (US) - Notifications",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "ycyjc23km689",
"name": "Prod1 (US) - Tracking",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "llsghyxnxs2v",
"name": "Prod1 (US) - Web Client",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "n9lty2bmvn86",
"name": "Third Party - PubNub",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer is aware of an event reported by one of our third party vendors (PubNub) affecting real-time messaging for orgs in Prod1 that may cause chat messages, agent notifications, and live updates to be delayed or not appear in real time within the platform.\n\nOur team is actively monitoring the incident, and working with the vendor where possible to resolve the issue. \n\nYou can also follow PubNub's status here: https://status.pubnub.com/incidents/cz6q37z3s1d5. \n\nPlease expect further updates within the next 3 hours, and reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-08-25T09:51:38.775-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T09:51:38.775-04:00",
"id": "r04d5q7zb4gt",
"incident_id": "8l2cbp53ppfh",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T09:51:38.775-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-08-25T09:51:38.775-04:00",
"name": "Reported Third Party Event - [PubNub]",
"page_id": "lhn2sb4njk08",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-08-25T10:17:26.892-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/1t65h6n2rfvx",
"started_at": "2026-08-25T09:51:38.624-04:00",
"status": "resolved",
"updated_at": "2026-08-25T10:17:26.907-04:00"
},
{
"components": [
{
"created_at": "2016-10-27T17:53:19.972-04:00",
"description": "Frontend Web Client Single Page Application",
"group_id": "gmwvdm9hg6dr",
"id": "llsghyxnxs2v",
"name": "Web Client",
"page_id": "lhn2sb4njk08",
"position": 14,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.868-04:00"
},
{
"created_at": "2019-11-22T13:22:51.809-05:00",
"description": null,
"group_id": "w2qyf5v9gm38",
"id": "s415bym88sdy",
"name": "Web Client",
"page_id": "lhn2sb4njk08",
"position": 14,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-30T14:15:18.079-04:00"
},
{
"created_at": "2025-03-14T14:17:04.521-04:00",
"description": null,
"group_id": "vz4171pkd3m7",
"id": "by9tzc6g9htz",
"name": "Web Client",
"page_id": "lhn2sb4njk08",
"position": 14,
"showcase": true,
"start_date": "2025-03-14T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-07-30T14:15:18.098-04:00"
}
],
"created_at": "2026-07-30T13:43:59.600-04:00",
"id": "xyz5b4bdsxv9",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "s415bym88sdy",
"name": "Prod2 (EU) - Web Client",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "llsghyxnxs2v",
"name": "Prod1 (US) - Web Client",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "by9tzc6g9htz",
"name": "Prod4 (IN) - Web Client",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Kustomer has resolved an event affecting text editor that may cause issues when typing and adding content such as Shortcuts. To resolve this issue, our team has rolled back to a previous version.\n\nAfter careful monitoring, our team has determined that all affected areas are now fully restored. Please reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-07-30T15:15:26.673-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-30T15:15:26.673-04:00",
"id": "0b2cyyg4sg73",
"incident_id": "xyz5b4bdsxv9",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-30T15:15:26.673-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "s415bym88sdy",
"name": "Prod2 (EU) - Web Client",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "llsghyxnxs2v",
"name": "Prod1 (US) - Web Client",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "by9tzc6g9htz",
"name": "Prod4 (IN) - Web Client",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has implemented an update to address an event affecting text editor that may cause issues when typing and adding content such as Shortcuts. Refreshing your browser will fully resolve the issue.\n\nOur team is currently monitoring this update to ensure the issue is fully resolved. Please expect further updates within the next 30 minutes, and reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-07-30T14:15:18.136-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-30T14:15:18.136-04:00",
"id": "8d67dsz2k9tm",
"incident_id": "xyz5b4bdsxv9",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-30T14:15:18.136-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "s415bym88sdy",
"name": "Prod2 (EU) - Web Client",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "llsghyxnxs2v",
"name": "Prod1 (US) - Web Client",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "by9tzc6g9htz",
"name": "Prod4 (IN) - Web Client",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer is aware of an event affecting text editor that may cause issues when typing and adding content such as Shortcuts.\n\nOur team is currently working to identify the cause of this issue in an effort to implement a resolution. Please expect additional updates within the next 30 minutes, please reach out to Kustomer Support via [email protected] for any further questions or updates.",
"created_at": "2026-07-30T13:43:59.711-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-30T13:43:59.711-04:00",
"id": "mnf05w7b5fgw",
"incident_id": "xyz5b4bdsxv9",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-30T13:43:59.711-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-30T14:15:18.136-04:00",
"name": "Issues with Text Editor with typing, pasting, \u0026 adding shortcuts PROD1, PROD2, and PROD4",
"page_id": "lhn2sb4njk08",
"postmortem_body": "# **Summary**\n\nOn July 30, 2026, the draft text editor on Kustomer’s Timeline product experienced degraded functionality in certain user workflows. Impact included sporadic cursor behavior, text erasure, and issues with copying and pasting text and shortcuts.\n\n# **Root Cause**\n\nThis issue was introduced by a change to the editor that was not fully caught before release. Due to misalignment in our QA process our pre-release validation did not adequately cover the real-world editing patterns affected by this change. Furthermore, this misalignment contributed to a delay in Kustomer’s understanding of the incident's resolution status.\n\n# **Timeline**\n\n## **Jul 30, 2026**\n\n**11:12 AM ET:** The editor change was deployed\n\n**12:27 PM ET:** Kustomer Technical Support escalated the incident to Kustomer’s OnCall process, notifying Engineering immediately.\n\n**12:34 PM ET:** Kustomer Engineering identified the issue and completed a rollback\n\n**12:51 PM ET:** Internal testing incorrectly confirms that the issue has been fully resolved, due to some customers reporting the issue no longer presented itself\n\n**1:29 PM ET:** Continued reports of issue are received from customers who did not receive the full resolution rollout\n\n**1:34 PM ET:** Discovery made that the rollback did not fully deploy to resolve the issue. Kustomer Engineering makes additional change to ensure resolution\n\n**2:01 PM ET:** Corrective change deployed, and fix is confirmed\n\n# **Lessons/Improvements**\n\nKustomer Engineering maintains a robust CI/CD process, and a multi-environment release process designed to prevent issues of this nature. As part of our continual investment in these areas, we have identified the following action items:\n\n* Ensure that our internal pre-production environments align with our customer-facing experience, including but not limited to the editor experience, so that issues of this nature are caught earlier in development\n* Complete an audit of our automated CI/CD process and test coverage of major features, removing any gaps that are identified",
"postmortem_body_last_updated_at": "2026-08-08T00:50:28.613-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-08-08T00:50:28.613-04:00",
"reminder_intervals": null,
"resolved_at": "2026-07-30T15:15:26.673-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/95dw616df83d",
"started_at": "2026-07-30T13:43:59.586-04:00",
"status": "resolved",
"updated_at": "2026-08-08T00:50:28.630-04:00"
},
{
"components": [
{
"created_at": "2016-10-25T17:04:46.229-04:00",
"description": "API Gateway Servicing all Public API Requests",
"group_id": "gmwvdm9hg6dr",
"id": "xdq19lkmjm66",
"name": "API",
"page_id": "lhn2sb4njk08",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-25T22:34:15.676-04:00"
},
{
"created_at": "2016-10-27T17:51:50.893-04:00",
"description": "Chat Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "p3pq4zsh46r9",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.821-04:00"
},
{
"created_at": "2016-10-27T17:51:57.085-04:00",
"description": "Email Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "6xrf32p25fkd",
"name": "Channel - Email",
"page_id": "lhn2sb4njk08",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-25T22:34:15.707-04:00"
},
{
"created_at": "2019-06-06T11:31:09.132-04:00",
"description": null,
"group_id": "gmwvdm9hg6dr",
"id": "ft27jgbbxb82",
"name": "Events / Audit Log",
"page_id": "lhn2sb4njk08",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-25T22:34:15.719-04:00"
},
{
"created_at": "2016-10-27T17:53:25.935-04:00",
"description": "Workflow Services",
"group_id": "gmwvdm9hg6dr",
"id": "gbgpxh2kfcpk",
"name": "Workflow",
"page_id": "lhn2sb4njk08",
"position": 16,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-25T22:34:15.732-04:00"
}
],
"created_at": "2026-07-25T21:13:28.513-04:00",
"id": "5bcmgp4trc9y",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "xdq19lkmjm66",
"name": "Prod1 (US) - API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ft27jgbbxb82",
"name": "Prod1 (US) - Events / Audit Log",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "gbgpxh2kfcpk",
"name": "Prod1 (US) - Workflow",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Kustomer has resolved an event affecting platform events in PROD1 orgs that caused delays on event-based data.\n\nAfter careful monitoring, our team has determined that all affected areas are now fully restored. Please reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-07-26T00:34:49.125-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-26T00:34:49.125-04:00",
"id": "p2smbtvsd90d",
"incident_id": "5bcmgp4trc9y",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-26T00:34:49.125-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xdq19lkmjm66",
"name": "Prod1 (US) - API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "ft27jgbbxb82",
"name": "Prod1 (US) - Events / Audit Log",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "gbgpxh2kfcpk",
"name": "Prod1 (US) - Workflow",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has implemented an update to address an event affecting platform events in PROD1 orgs that caused delays on event-based data. \n\nOur team is currently monitoring this update to ensure the issue is fully resolved. \n\nPlease expect further updates within the next 30 minutes, and reach out to Kustomer Support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-07-25T22:34:15.759-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-25T22:34:15.759-04:00",
"id": "6sgtf23hzsk9",
"incident_id": "5bcmgp4trc9y",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-25T22:34:15.759-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xdq19lkmjm66",
"name": "Prod1 (US) - API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "ft27jgbbxb82",
"name": "Prod1 (US) - Events / Audit Log",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "gbgpxh2kfcpk",
"name": "Prod1 (US) - Workflow",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer continues to work on the issue affecting platform events in PROD1 orgs that may cause delays on event-based data. \n\nOur team is actively working to implement a resolution. Please expect additional updates within the next 30 minutes, and reach out to Kustomer Support at [email protected] for any further questions or updates.",
"created_at": "2026-07-25T22:03:18.263-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-25T22:03:18.263-04:00",
"id": "wy7f3gqrtgb7",
"incident_id": "5bcmgp4trc9y",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-25T22:03:18.263-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xdq19lkmjm66",
"name": "Prod1 (US) - API",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "ft27jgbbxb82",
"name": "Prod1 (US) - Events / Audit Log",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "gbgpxh2kfcpk",
"name": "Prod1 (US) - Workflow",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has identified an event affecting platform events in PROD1 orgs that may cause delays on event-based data. \n\nOur team is currently actively working to implement a resolution. Please expect additional updates within the next 30 minutes, and reach out to Kustomer Support at [email protected] for any further questions or updates.",
"created_at": "2026-07-25T21:16:24.975-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-25T21:16:24.000-04:00",
"id": "09b4tctcjnkh",
"incident_id": "5bcmgp4trc9y",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-25T21:17:06.014-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "ft27jgbbxb82",
"name": "Prod1 (US) - Events / Audit Log",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "gbgpxh2kfcpk",
"name": "Prod1 (US) - Workflow",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer has identified an event affecting platform events in PROD1 orgs that may cause delays on event-based data. \n\nOur team is currently working to implement a resolution. Please expect additional updates within the next 30 minutes, and reach out to Kustomer Support at [email protected] for any further questions or updates.",
"created_at": "2026-07-25T21:13:28.664-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-25T21:13:28.664-04:00",
"id": "37gc5jdx91wy",
"incident_id": "5bcmgp4trc9y",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-25T21:13:28.664-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-25T22:34:15.759-04:00",
"name": "Platform Events Service Disruption (PROD1)",
"page_id": "lhn2sb4njk08",
"postmortem_body": "## Summary\n\nOn July 25, 2026, some customers experienced elevated platform latency affecting messaging, conversation updates, routing, voice, and other customer-service workflows. Customers may have seen messages remain in a sending state, delayed inbound or outbound messages, slower page and API responses, delayed routing or assignment, and intermittent voice-call delays.\n\nThe incident was caused by an unusually large burst of background data-processing activity. This created a sudden increase in demand on shared platform infrastructure. Automatic scaling added capacity, but it could not absorb the burst quickly enough to prevent degradation in dependent workflows.\n\nWe restored service by increasing available capacity, reducing the rate of the initiating workload, and carefully processing delayed work while monitoring platform health.\n\n## Impact\n\n* **Customer effect:** Intermittent platform latency; delayed inbound and outbound messaging; slower conversation and API updates; delayed routing or assignment; and intermittent voice-call delays\n* **Duration:** Customer-visible degradation began at approximately 5:22 PM ET. Service initially recovered at approximately 7:31 PM ET, but degradation later recurred. Broad platform performance was restored and the incident was resolved at 10:30 PM ET.\n* **Scope:** The incident affected a subset of customers within one production environment. We did not identify corresponding customer-facing degradation in other production environments.\n\n## Timeline\n\n* **Approximately 5:22 PM ET:** Processing latency and message backlogs began increasing.\n* **6:51 PM ET:** The incident response team began coordinated investigation and recovery.\n* **Approximately 7:00–7:23 PM ET:** We identified affected processing paths and began carefully recovering delayed work at controlled rates.\n* **7:31 PM ET:** Additional capacity had come online, customer-facing performance had materially improved, and the incident was initially resolved while monitoring continued.\n* **8:28 PM ET:** The incident was reopened after renewed reports of intermittent degradation.\n* **Approximately 8:38 PM ET:** Investigation confirmed that messaging, routing, channel, and voice symptoms shared the same underlying platform dependency.\n* **Approximately 9:52 PM ET:** We reduced the rate of the initiating background workload to protect customer-facing traffic.\n* **10:08 PM ET:** Monitoring confirmed that workload pressure had fallen substantially and platform health continued improving.\n* **10:30 PM ET:** After continued monitoring confirmed recovery, the incident was resolved.\n* **After resolution:** Recovery of remaining delayed work continued at controlled rates under monitoring.\n\n## Root cause\n\nA burst of high-volume background data-processing activity generated more downstream work than shared platform infrastructure could safely absorb over a short period.\n\nAutomatic scaling responded and added capacity, but capacity was added incrementally and did not come online quickly enough for the size and speed of the burst. While the platform was catching up, increased request latency and timeouts affected customer-facing workflows that depended on the same infrastructure.\n\n## Resolution\n\nWe restored platform performance by:\n\n* Increasing available capacity and improving the rate at which additional capacity could be added.\n* Reducing the rate of the initiating background workload.\n* Recovering delayed work at controlled rates to avoid creating another traffic spike.\n* Continuing monitoring after customer-facing performance returned to expected levels.\n\n## Preventative actions\n\nWe are taking the following actions to reduce the likelihood and impact of recurrence:\n\n* Add stronger limits and backpressure controls for high-volume background operations.\n* Improve the speed at which shared infrastructure scales during sudden traffic increases.\n* Improve isolation between background processing and latency-sensitive customer workflows.\n* Expand alerting for rapid backlog growth, resource pressure, and unusual workload patterns.\n* Strengthen controlled recovery procedures for delayed work.\n* Expand burst-load and failure-recovery testing for shared platform services.\n\n## Current status\n\nPlatform performance returned to expected levels, and the initiating workload remained controlled. We continued monitoring service health and the recovery of delayed work after resolution.",
"postmortem_body_last_updated_at": "2026-07-30T12:17:28.387-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-07-30T12:17:28.387-04:00",
"reminder_intervals": null,
"resolved_at": "2026-07-26T00:34:49.125-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/m5pl3lfgn01k",
"started_at": "2026-07-25T21:13:28.496-04:00",
"status": "resolved",
"updated_at": "2026-07-30T12:17:28.403-04:00"
},
{
"components": [
{
"created_at": "2016-10-27T17:51:50.893-04:00",
"description": "Chat Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "p3pq4zsh46r9",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.821-04:00"
}
],
"created_at": "2026-06-25T18:24:10.883-04:00",
"id": "qbn5146pry5h",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has resolved the issue affecting PROD 1 that impacted the ability to send outbound messages through MessageBird.\n\nOur team has verified that the fix has been successfully deployed and service has been restored. If you continue to experience issues sending outbound MessageBird messages, please contact Kustomer Support at [email protected]\n\nWe apologize for the disruption and appreciate your patience while we worked to resolve the issue.",
"created_at": "2026-06-25T19:30:56.664-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-25T19:30:56.664-04:00",
"id": "mtl00zhq84md",
"incident_id": "qbn5146pry5h",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-25T19:30:56.664-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has identified and implemented a fix for the issue affecting PROD 1 that impacted the ability to send outbound messages through MessageBird.\n\nOur team is actively monitoring the platform to ensure the fix remains effective and that service has fully recovered. If you continue to experience issues sending outbound MessageBird messages, please contact Kustomer Support at [email protected].\n\nWe will provide another update once monitoring is complete or if there are any significant developments.\n\nThank you for your patience.",
"created_at": "2026-06-25T19:06:22.482-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-25T19:06:22.482-04:00",
"id": "896dtx001271",
"incident_id": "qbn5146pry5h",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-25T19:06:22.482-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has identified and implemented a fix for the issue affecting PROD 1 that impacted the ability to send outbound messages through MessageBird.\n\nOur team is actively monitoring the platform to ensure the fix remains effective and that service has fully recovered. If you continue to experience issues sending outbound MessageBird messages, please contact Kustomer Support at [email protected].\n\nWe will provide another update once monitoring is complete or if there are any significant developments.\n\nThank you for your patience.",
"created_at": "2026-06-25T19:05:49.212-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-25T19:05:49.212-04:00",
"id": "d441h6hft7ck",
"incident_id": "qbn5146pry5h",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-25T19:05:49.212-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has identified the cause affecting PROD 1 that is impacting the ability to send outbound messages through MessageBird.\n\nPlease expect another update within the next 30 minutes, as we reach a resolution. If you have any additional questions or concerns, please reply to this conversation or contact Kustomer Support at [email protected].\n\nThank you for your patience while we work to resolve this issue.",
"created_at": "2026-06-25T18:49:35.229-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-25T18:49:35.229-04:00",
"id": "cymwf43sg650",
"incident_id": "qbn5146pry5h",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-25T18:49:35.229-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer is aware of an event affecting PROD 1 that may affect the ability to send outbound messages for MessageBird.\n\nOur team is currently working to identify the cause of this issue in an effort to implement a resolution. Please expect additional updates within the next 30 minutes, please reach out to Kustomer Support at [email protected] for any further questions or updates.",
"created_at": "2026-06-25T18:24:10.987-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-25T18:24:10.987-04:00",
"id": "mq99w5zh8r4l",
"incident_id": "qbn5146pry5h",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-25T18:24:10.987-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-06-25T19:06:22.482-04:00",
"name": "MessageBird - unable to send outbound replies PROD 1",
"page_id": "lhn2sb4njk08",
"postmortem_body": "## **Summary**\n\nOn June 25, 2026, some customers were unable to send outbound WhatsApp replies from within Kustomer. The issue affected reply sending for a subset of WhatsApp channel configurations, which disrupted agent workflows and prevented some automated outbound messages from being sent through the same path.\n\nThe issue was identified and resolved the same day. Service was fully restored, and the platform is operating normally.\n\n## **Root cause**\n\nA change released earlier that day introduced stricter validation in the outbound WhatsApp reply flow. For a subset of supported channel configurations, valid sender values were incorrectly rejected before messages were sent. This caused reply attempts in those configurations to fail.\n\nThe issue was limited to specific WhatsApp channel setups and did not affect all WhatsApp traffic equally.\n\n## **Timeline**\n\n* **June 25, 2026, approximately 22:12 UTC** — Reports began coming in that outbound WhatsApp replies were failing for some customers.\n* **Shortly after detection** — Investigation confirmed the issue was tied to a recently released validation change in the outbound reply flow.\n* **June 26, 2026, approximately 01:07 UTC** — A fix was deployed and reply sending was restored.\n\n## **Lessons and improvements**\n\n* Validation changes for messaging flows now require broader test coverage across supported channel configuration variants before release.\n* Additional safeguards are being added to reduce the risk of valid outbound requests being rejected.\n* Monitoring and regression checks around outbound messaging paths are being strengthened to detect similar issues more quickly.",
"postmortem_body_last_updated_at": "2026-08-06T10:13:41.832-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-08-06T10:13:41.832-04:00",
"reminder_intervals": null,
"resolved_at": "2026-06-25T19:30:56.664-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/c2t4lzbjbtnk",
"started_at": "2026-06-25T18:24:10.869-04:00",
"status": "resolved",
"updated_at": "2026-08-06T10:13:41.849-04:00"
},
{
"components": [
{
"created_at": "2019-09-13T07:36:22.405-04:00",
"description": null,
"group_id": "2whbcrvv0534",
"id": "kz5v8kb60zfv",
"name": "Channel - WhatsApp",
"page_id": "lhn2sb4njk08",
"position": 18,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-12T17:38:21.375-04:00"
}
],
"created_at": "2026-06-12T10:31:41.000-04:00",
"id": "gslc5mr45yz0",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "kz5v8kb60zfv",
"name": "Third Party - Channel - WhatsApp",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "WhatsApp has confirmed that all services are fully recovered. After careful monitoring, our team has determined that all affected areas are now fully restored. Please reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-12T17:38:21.403-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-12T17:38:21.403-04:00",
"id": "ntcd3v2t2wbz",
"incident_id": "gslc5mr45yz0",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T17:38:21.403-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kz5v8kb60zfv",
"name": "Third Party - Channel - WhatsApp",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer is aware of an event reported by one of our third party vendors affecting ALL PRODS that may cause Issues with WhatsApp messages not being delivered to end users.\n\nOur team is monitoring the incident, and working with the vendor where possible to resolve the issue. We are working on redriving previously unsent messages to ensure all messages are sent successfully. Please expect further updates within the next 3 hours, and reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-12T15:29:46.900-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-12T15:29:46.900-04:00",
"id": "8wdlw7w1dc77",
"incident_id": "gslc5mr45yz0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T15:29:46.900-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kz5v8kb60zfv",
"name": "Third Party - Channel - WhatsApp",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer continues to monitor the ongoing issue on Whatsapp that may cause outbound Whatsapp messages to not be delivered to end users.\n\nClients can also refer to https://metastatus.com/whatsapp-business-api for the latest WhatsApp service status. \n\nPlease expect further updates within the next 30 minutes, and reach out to Kustomer Support at [email protected] if you have additional questions or concerns",
"created_at": "2026-06-12T12:35:30.778-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-12T12:35:30.778-04:00",
"id": "bbxcnglhh04l",
"incident_id": "gslc5mr45yz0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T12:35:30.778-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kz5v8kb60zfv",
"name": "Third Party - Channel - WhatsApp",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer continues to monitor the ongoing issue on Whatsapp that may cause outbound Whatsapp messages to not be delivered to end users.\n\nClients can also refer to https://metastatus.com/whatsapp-business-api for the latest WhatsApp service status. \n\nPlease expect further updates within the next 30 minutes, and reach out to Kustomer Support at [email protected] if you have additional questions or concerns",
"created_at": "2026-06-12T11:17:51.573-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-12T11:17:51.573-04:00",
"id": "40wkl9v5nync",
"incident_id": "gslc5mr45yz0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T11:17:51.573-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kz5v8kb60zfv",
"name": "Third Party - Channel - WhatsApp",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has identified an ongoing issue with WhatsApp that may cause outbound messages to not be delivered to end users.\n\nOur team is closely monitoring the situation and actively working to assess impact. Clients can refer to https://metastatus.com/whatsapp-business-api for the latest WhatsApp service status. \n\nPlease expect further updates within the next 30 minutes, and reach out to Kustomer Support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-12T10:46:26.393-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-12T10:46:26.393-04:00",
"id": "jlt8g5xxpxj2",
"incident_id": "gslc5mr45yz0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T10:46:26.393-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kz5v8kb60zfv",
"name": "Third Party - Channel - WhatsApp",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer is aware of an ongoing WhatsApp issue that may cause outbound messages to not be delivered to end users.\n\nWhile the issue appears to originate with WhatsApp, our team is actively monitoring the situation and working closely to assess the impact on the platform. We will continue to provide updates as more information becomes available.\n\nPlease expect an additional update within the next 30 minutes. If you have any questions, please contact Kustomer Support at [email protected].",
"created_at": "2026-06-12T10:31:41.885-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-12T10:31:41.000-04:00",
"id": "2rzpz4kw7ntw",
"incident_id": "gslc5mr45yz0",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-12T10:32:37.021-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-06-12T15:29:46.900-04:00",
"name": "[Whatsapp] Outbound Message Delivery Disruption [ALL PRODs]",
"page_id": "lhn2sb4njk08",
"postmortem_body": "## Summary\n\nOn June 12, 2026, customers using WhatsApp through Kustomer experienced a service event that prevented some outbound messages from being delivered to end users. During the event, messages could appear as sent in Kustomer before final delivery was confirmed by WhatsApp. Service recovered the same day, and affected messages from the incident window were successfully reprocessed.\n\n## Root cause\n\nThis event was caused by a disruption in WhatsApp Business Platform services operated by Meta. While that disruption was active, Kustomer was unable to complete delivery for some outbound WhatsApp messages and queued affected messages for retry after the upstream service recovered.\n\n## Timeline\n\n* **10:14 AM EDT** — Customer reports led to investigation of outbound messaging behavior.\n* **10:34 AM EDT** — Meta reported high disruptions affecting WhatsApp Business Platform services.\n* **1:45 PM EDT** — Reprocessing of queued messages began as upstream recovery progressed.\n* **2:32 PM EDT** — Meta reported recovery for the remaining affected WhatsApp services.\n* **4:07 PM EDT** — Active message sending was healthy again.\n* **4:34 PM EDT** — Messages from the incident window had been successfully reprocessed.\n\n## Lessons and improvements\n\n* We are improving monitoring and alerting for queued WhatsApp messages so degraded delivery is identified earlier.\n* We are documenting a safer reprocessing procedure for queued WhatsApp traffic to reduce the chance of retry-related rate limiting during recovery.\n* We are reviewing backlog-handling procedures to make recovery more predictable when an upstream provider disruption occurs.\n\nAt this time, system health for WhatsApp message delivery through Kustomer is stable.",
"postmortem_body_last_updated_at": "2026-06-22T10:58:13.279-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-06-22T10:58:13.279-04:00",
"reminder_intervals": null,
"resolved_at": "2026-06-12T17:38:21.403-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/w6wkjkhmzx41",
"started_at": "2026-06-12T10:31:41.000-04:00",
"status": "resolved",
"updated_at": "2026-06-22T10:58:13.296-04:00"
},
{
"components": [
{
"created_at": "2016-10-27T17:51:50.893-04:00",
"description": "Chat Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "p3pq4zsh46r9",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.821-04:00"
},
{
"created_at": "2019-11-22T13:17:15.027-05:00",
"description": null,
"group_id": "w2qyf5v9gm38",
"id": "s0v2msb48f9g",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-09T10:11:51.693-04:00"
},
{
"created_at": "2016-10-27T17:51:57.085-04:00",
"description": "Email Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "6xrf32p25fkd",
"name": "Channel - Email",
"page_id": "lhn2sb4njk08",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-25T22:34:15.707-04:00"
},
{
"created_at": "2019-11-22T13:18:47.439-05:00",
"description": null,
"group_id": "w2qyf5v9gm38",
"id": "8dkd2yjf7z5t",
"name": "Channel - Email",
"page_id": "lhn2sb4njk08",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-09T10:11:51.726-04:00"
}
],
"created_at": "2026-06-09T10:00:01.273-04:00",
"id": "4x4v2c7wwvrh",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "s0v2msb48f9g",
"name": "Prod2 (EU) - Channel - Chat",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "8dkd2yjf7z5t",
"name": "Prod2 (EU) - Channel - Email",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Kustomer has resolved an event affecting Chat and other channels that caused delayed message delivery and slower conversation load times. \n\nAfter careful monitoring, our team has determined that all affected areas are now fully restored. \n\nPlease reach out to Kustomer Support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-09T11:18:07.508-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-09T11:18:07.508-04:00",
"id": "mkf9mcgcbpqp",
"incident_id": "4x4v2c7wwvrh",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-09T11:18:07.508-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "s0v2msb48f9g",
"name": "Prod2 (EU) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "8dkd2yjf7z5t",
"name": "Prod2 (EU) - Channel - Email",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has identified the cause of the event affecting Chat and other channels that resulted in delayed message delivery and slower conversation load times. The situation has stabilized and our team is continuing to monitor to ensure full resolution. Please reach out to Kustomer Support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-09T10:11:51.754-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-09T10:11:51.754-04:00",
"id": "1yt86q3b401q",
"incident_id": "4x4v2c7wwvrh",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-09T10:11:51.754-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "s0v2msb48f9g",
"name": "Prod2 (EU) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "8dkd2yjf7z5t",
"name": "Prod2 (EU) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer is aware of an event affecting Chat and other channels that may cause delayed message delivery and slower conversation load times.\n\nOur team is currently working to identify the cause of this issue in an effort to implement a resolution. Please expect additional updates within the next 30 minutes and please reach out to Kustomer Support at [email protected] for any further questions or updates.",
"created_at": "2026-06-09T10:00:01.385-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-09T10:00:01.385-04:00",
"id": "lxx4hh0xfw8p",
"incident_id": "4x4v2c7wwvrh",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-09T10:00:01.385-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-06-09T10:11:51.754-04:00",
"name": "[Channels] [Platform Latency] [Prods 1 and 2]",
"page_id": "lhn2sb4njk08",
"postmortem_body": "**Summary**\n\nOn June 9, 2026, customers experienced delays in chat message delivery and related real-time updates for a portion of the morning. The issue was caused by a third-party service event affecting the infrastructure used to propagate real-time messaging updates. Service recovered the same morning, and current system health is stable.\n\n**Root cause**\n\nA third-party real-time messaging provider experienced elevated latency and intermittent publish failures in one of its regions. That disruption delayed delivery acknowledgements and other real-time updates in Kustomer. Messages continued to be stored successfully, but some updates were not reflected immediately in the client until the external service recovered or the client refreshed.\n\n**Timeline**\n\n* **9:05 AM EDT** — Customer impact began, including delayed chat message delivery and stale real-time updates.\n* **9:45 AM EDT** — The third-party provider reported active latency affecting its service.\n* **9:48 AM EDT** — The provider reported mitigation and recovery.\n* **Shortly after recovery** — Real-time behavior returned to normal and monitoring confirmed stability.\n\n**Lesson/improvements**\n\n* We added additional monitoring tied to the third-party provider's public service health signals so similar service events can be identified faster.\n* We are improving alerting around real-time messaging errors to reduce time to diagnosis.\n* We are continuing to review operational signals to better distinguish external service issues from internal platform issues.",
"postmortem_body_last_updated_at": "2026-06-11T17:18:58.045-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-06-11T17:18:58.045-04:00",
"reminder_intervals": null,
"resolved_at": "2026-06-09T11:18:07.508-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/94xzw53k91f8",
"started_at": "2026-06-09T10:00:01.259-04:00",
"status": "resolved",
"updated_at": "2026-06-11T17:18:58.063-04:00"
},
{
"components": [
{
"created_at": "2016-10-27T17:51:50.893-04:00",
"description": "Chat Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "p3pq4zsh46r9",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.821-04:00"
},
{
"created_at": "2016-10-27T17:51:57.085-04:00",
"description": "Email Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "6xrf32p25fkd",
"name": "Channel - Email",
"page_id": "lhn2sb4njk08",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-25T22:34:15.707-04:00"
}
],
"created_at": "2026-06-02T10:47:44.866-04:00",
"id": "qfxp5q9gq38s",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has resolved an event affecting Prod 1 that caused chat, email, and routing latency. To resolve this issue, our team reverted an underlying infrastructure change as a precaution, which helped stabilize the service.\n\nAfter careful monitoring, our team has determined that all affected areas are now fully restored. Please reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-02T11:53:13.226-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-02T11:53:13.226-04:00",
"id": "dwt8l2vnklrd",
"incident_id": "qfxp5q9gq38s",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-02T11:53:13.226-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has implemented an update to address an event affecting chat, email, and routing that caused latency in sending and delivering messages.\n\nOur team is currently monitoring this update to ensure the issue is fully resolved. Please expect further updates within the next 30 minutes, and reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-02T11:27:14.215-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-02T11:27:14.215-04:00",
"id": "w2w28fgb3ntm",
"incident_id": "qfxp5q9gq38s",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-02T11:27:14.215-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Kustomer has identified an event affecting chats, emails, and routing that may cause latency with sending, receiving, and routing messages.\n\nOur team is currently working to implement a resolution. Please expect further updates within the next 30 minutes, and reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-06-02T11:19:49.860-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-02T11:19:49.860-04:00",
"id": "1l2g30hlhqlx",
"incident_id": "qfxp5q9gq38s",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-02T11:19:49.860-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "6xrf32p25fkd",
"name": "Prod1 (US) - Channel - Email",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer is aware of an event affecting chats, emails, and routing that may cause latency with sending, receiving, and routing messages.\n\nOur team is currently working to identify the cause of this issue in an effort to implement a resolution. Please expect additional updates within the next 30 minutes, please reach out to Kustomer Support at [email protected] for any further questions or updates.",
"created_at": "2026-06-02T10:47:44.975-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-02T10:47:44.975-04:00",
"id": "39lmms4r06yy",
"incident_id": "qfxp5q9gq38s",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-02T10:47:44.975-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-06-02T11:27:14.215-04:00",
"name": "Chats, Emails, Routing latency PROD 1",
"page_id": "lhn2sb4njk08",
"postmortem_body": "**Summary**\n\nOn June 2, 2026, customers in our prod1 environment experienced elevated latency affecting chat creation, email sending, and conversation routing. The event caused delayed processing for a subset of customer interactions during the incident window. Service performance recovered the same day after mitigation steps were applied, and the environment is currently operating normally.\n\n**Impact**\n\n* Scope: prod1 only\n* Customer effect: delayed chat creation, delayed email sends, and slower conversation routing\n* Duration: customer-visible impact began in the morning ET on June 2 and materially improved after mitigation later that afternoon\n* Other environments: prod2 and prod4 were not impacted by the customer-facing degradation\n\n**Timeline**\n\n* Early June 2: We detected elevated processing latency in services supporting chat, email, and routing in prod1.\n* Midday to afternoon ET: Customer reports confirmed delayed system behavior in prod1.\n* Afternoon ET: We increased service capacity and adjusted resource limits to reduce backlog and restore throughput.\n* Later that day: We completed an infrastructure rollback on affected worker hosts and confirmed stable recovery.\n* June 3: Monitoring confirmed the environment remained healthy.\n\n**Root cause**\n\nThe incident was caused by infrastructure-level resource exhaustion on a set of worker hosts in prod1. That reduced the availability of a metadata-dependent service and created message backlog, which in turn increased latency for customer-facing workflows such as chat creation, email sending, and routing. The issue was isolated to prod1.\n\n**Resolution**\n\nWe restored service by increasing available capacity, raising resource limits for the affected service, and rolling impacted worker infrastructure back to a stable configuration. After those changes were applied, backlog cleared and latency returned to expected levels. Current system health is stable.\n\n**Preventative actions**\n\n* Strengthen host-level capacity and disk safeguards before future infrastructure rollouts\n* Add earlier alerting for infrastructure resource pressure to reduce time to detection\n* Expand validation for production-scale logging and resource usage prior to promotion\n* Continue tuning service capacity thresholds and recovery procedures for metadata-dependent workloads",
"postmortem_body_last_updated_at": "2026-06-11T17:17:41.860-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-06-11T17:17:41.860-04:00",
"reminder_intervals": null,
"resolved_at": "2026-06-02T11:53:13.226-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/h5rhqx6rqnxr",
"started_at": "2026-06-02T10:47:44.853-04:00",
"status": "resolved",
"updated_at": "2026-06-11T17:17:41.875-04:00"
},
{
"components": [
{
"created_at": "2016-10-27T17:51:50.893-04:00",
"description": "Chat Channel Services",
"group_id": "gmwvdm9hg6dr",
"id": "p3pq4zsh46r9",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T10:17:26.821-04:00"
},
{
"created_at": "2019-11-22T13:17:15.027-05:00",
"description": null,
"group_id": "w2qyf5v9gm38",
"id": "s0v2msb48f9g",
"name": "Channel - Chat",
"page_id": "lhn2sb4njk08",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-06-09T10:11:51.693-04:00"
}
],
"created_at": "2026-05-26T18:14:16.472-04:00",
"id": "9rb0yp05kk85",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "s0v2msb48f9g",
"name": "Prod2 (EU) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved. Our teams have confirmed recovery and platform stability following mitigation efforts.\n\nWe will be reaching out independently to affected orgs to provide additional context and follow-up as needed. Thank you for your patience while we worked through this issue.",
"created_at": "2026-05-26T20:21:43.466-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-26T20:21:43.466-04:00",
"id": "h2rh2rpqyt2j",
"incident_id": "9rb0yp05kk85",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-26T20:21:43.466-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "s0v2msb48f9g",
"name": "Prod2 (EU) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "p3pq4zsh46r9",
"name": "Prod1 (US) - Channel - Chat",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Kustomer has implemented an update to address an event affecting WhatsApp that caused messaging failures.\n\nOur team is currently monitoring this update to ensure the issue is fully resolved. Please expect further updates within the next 30 minutes, and reach out to Kustomer support at [email protected] if you have additional questions or concerns.",
"created_at": "2026-05-26T18:14:16.600-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-26T18:14:16.600-04:00",
"id": "fc8my48rl28z",
"incident_id": "9rb0yp05kk85",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-26T18:14:16.600-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-05-26T18:14:16.600-04:00",
"name": "WhatsApp messages failing Prods 1 + 2",
"page_id": "lhn2sb4njk08",
"postmortem_body": "## Summary\n\nOn May 26, 2026, some customers experienced failures when sending WhatsApp messages from existing conversations. The issue affected outbound message delivery for a subset of WhatsApp configurations. Service was restored after the responsible change was rolled back, and the issue is now resolved.\n\n## Impact\n\nDuring the incident window, outbound WhatsApp messages failed for some existing conversations. Newly created conversations were less consistently affected, and impact varied by account configuration.\n\nCustomers may have seen message-send failures or provider errors while attempting to send WhatsApp messages.\n\n## Timeline\n\nAll times EDT.\n\n* **4:05 PM:** The incident window is believed to have begun after a recent WhatsApp-related change.\n* **5:54 PM:** Active investigation began.\n* **5:56 PM:** The team identified a likely connection to recent WhatsApp sender-selection behavior and started a rollback.\n* **6:05 PM:** Rollback completed.\n* **6:17 PM – 7:47 PM:** The team validated recovery across affected examples and narrowed the underlying cause.\n* **8:21 PM:** The incident was declared resolved.\n\n## Root cause\n\nThe incident was caused by an issue in WhatsApp phone-number lookup and sender selection. A recent change expanded support for multiple valid phone-number formats in certain countries. In some cases, that allowed the system to resolve the wrong sender configuration for an existing conversation.\n\nWhen that happened, outbound sends could target an invalid or unavailable sender identity, causing WhatsApp message delivery to fail.\n\n## Resolution\n\nWe mitigated the incident by rolling back the relevant WhatsApp change. After the rollback, we verified recovery against affected examples and confirmed that the incident was resolved.\n\n## Preventative actions\n\n* Move WhatsApp sender resolution toward more stable identifier-based matching rather than relying on ambiguous phone-number formatting.\n* Expand regression coverage for country-specific phone-number formatting edge cases.\n* Improve monitoring and alerting for message delivery failures and queue growth.\n* Improve testing workflows for WhatsApp changes before production rollout.",
"postmortem_body_last_updated_at": "2026-06-11T17:20:08.888-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-06-11T17:20:08.888-04:00",
"reminder_intervals": null,
"resolved_at": "2026-05-26T20:21:43.466-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/xxy669kjmhcq",
"started_at": "2026-05-26T18:14:16.452-04:00",
"status": "resolved",
"updated_at": "2026-06-11T17:20:08.903-04:00"
}
],
"page": {
"id": "lhn2sb4njk08",
"name": "Kustomer",
"url": "https://status.kustomer.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}