Snapshot 214225
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"1hkxj2q2rctq",
"8v81pmpgs8sq",
"frx8ppwtppf7",
"w3dkvn4lx0rl",
"hm568vz5qnvw",
"r9dxbwjjpbxy",
"lxdltc90f0hl",
"tq76t6mrgqwv",
"3x4hyn64yhmh",
"zcbtrz81ztbl"
],
"created_at": "2018-03-01T16:40:42.850Z",
"description": null,
"group_id": null,
"id": "hymc9wr9f9k7",
"name": "Realtime Network",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2018-03-01T16:41:04.778Z"
},
{
"components": [
"tsdmzyxfmkls",
"5jkspb588c37",
"5mcrt3n0xs27",
"w89b16t9cdky"
],
"created_at": "2018-03-01T16:42:43.184Z",
"description": null,
"group_id": null,
"id": "nj61xcnxhrz5",
"name": "Functions",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2018-03-01T16:43:17.273Z"
},
{
"components": [
"gthhcmbf0c50",
"4j3ty03r7c3s",
"9zh0fwsg54wk",
"n01j0h7z2wgy"
],
"created_at": "2018-03-01T16:48:17.494Z",
"description": null,
"group_id": null,
"id": "60bpnf4hmfcr",
"name": "Website and Portals",
"page_id": "j2pr9thwz01t",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2018-03-01T16:50:21.891Z"
},
{
"components": [
"wbrc65lqrp03",
"9j2mvls8yv8p",
"fkmgfx9f3bsk",
"4km3s664qd7l"
],
"created_at": "2018-03-01T16:47:04.444Z",
"description": null,
"group_id": null,
"id": "fmkr256xnbyh",
"name": "Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 15,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2018-03-01T16:48:29.917Z"
}
],
"incidents": [
{
"components": [
{
"created_at": "2018-02-27T16:42:14.196Z",
"description": "All service running across North America status",
"group_id": "fmkr256xnbyh",
"id": "wbrc65lqrp03",
"name": "North America Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.875Z"
},
{
"created_at": "2018-02-27T16:42:48.522Z",
"description": "All services running in Europe status",
"group_id": "fmkr256xnbyh",
"id": "9j2mvls8yv8p",
"name": "European Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:17:10.416Z"
},
{
"created_at": "2018-02-27T16:43:29.592Z",
"description": "All Services running across Asia Pacific",
"group_id": "fmkr256xnbyh",
"id": "fkmgfx9f3bsk",
"name": "Asia Pacific Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.919Z"
},
{
"created_at": "2015-05-01T01:50:05.984Z",
"description": "Join, Leave and Here Now API Calls",
"group_id": "hymc9wr9f9k7",
"id": "w3dkvn4lx0rl",
"name": "Presence Service",
"page_id": "j2pr9thwz01t",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.938Z"
},
{
"created_at": "2018-02-27T16:44:00.007Z",
"description": "All services running in Southern Asia status",
"group_id": "fmkr256xnbyh",
"id": "4km3s664qd7l",
"name": "Southern Asia Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.955Z"
}
],
"created_at": "2026-10-26T21:07:34.082Z",
"id": "qb18vzmmwcgz",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "fkmgfx9f3bsk",
"name": "Points of Presence - Asia Pacific Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4km3s664qd7l",
"name": "Points of Presence - Southern Asia Points of Presence",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved, and we will follow up with a postmortem. If you have questions or concerns, please reach out to support@pubnub.com.",
"created_at": "2026-10-26T21:55:36.124Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-26T21:55:36.124Z",
"id": "7s2l51q5bnvs",
"incident_id": "qb18vzmmwcgz",
"status": "resolved",
"tweet_id": 2114838389598130400,
"twitter_updated_at": "2026-10-26T21:55:38.891Z",
"updated_at": "2026-10-26T21:55:38.894Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "fkmgfx9f3bsk",
"name": "Points of Presence - Asia Pacific Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4km3s664qd7l",
"name": "Points of Presence - Southern Asia Points of Presence",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are continuing to monitor for any further issues.",
"created_at": "2026-10-26T21:17:10.469Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-26T21:17:10.469Z",
"id": "hjf09p4ns8rk",
"incident_id": "qb18vzmmwcgz",
"status": "monitoring",
"tweet_id": 2114828710918648000,
"twitter_updated_at": "2026-10-26T21:17:11.101Z",
"updated_at": "2026-10-26T21:17:11.103Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "fkmgfx9f3bsk",
"name": "Points of Presence - Asia Pacific Points of Presence",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "4km3s664qd7l",
"name": "Points of Presence - Southern Asia Points of Presence",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix for this issue has been implemented and we are monitoring the results. If you feel you have been impacted by this, please reach out to support@pubnub.com.",
"created_at": "2026-10-26T21:12:03.986Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-26T21:12:03.986Z",
"id": "s4tb0m7nvyw3",
"incident_id": "qb18vzmmwcgz",
"status": "monitoring",
"tweet_id": 2114827426958295300,
"twitter_updated_at": "2026-10-26T21:12:04.998Z",
"updated_at": "2026-10-26T21:12:05.001Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "fkmgfx9f3bsk",
"name": "Points of Presence - Asia Pacific Points of Presence",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "4km3s664qd7l",
"name": "Points of Presence - Southern Asia Points of Presence",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Users may be experiencing elevated error and latency rates with the Presence API. We are working on remediation and will update this as we make progress. If you believe you are affected and would like to discuss, please reach out to support@pubnub.com",
"created_at": "2026-10-26T21:07:34.298Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-26T21:07:34.298Z",
"id": "xqp75zly11rr",
"incident_id": "qb18vzmmwcgz",
"status": "investigating",
"tweet_id": 2114826295624560600,
"twitter_updated_at": "2026-10-26T21:07:35.254Z",
"updated_at": "2026-10-26T21:07:35.258Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-10-26T21:12:03.986Z",
"name": "Errors and Latency for Presence",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### Problem Description, Impact, and Resolution \n\nAt approximately 20:40 UTC on October 26, 2026, we observed elevated error rates and latency with the Presence API across multiple regions globally. A software update to the Presence service triggered resource issues that reduced the number of servers available to handle requests. Our team identified the deployment as the source of the errors and initiated a rollback to the previous version. \n\nError rates returned to normal at approximately 21:07 UTC on October 26, 2026. We continued monitoring as latency improved, returning fully to normal at approximately 21:46 UTC. The incident was declared resolved at 21:55 UTC.\n\n### Mitigation Steps and Recommended Future Preventative Measures \n\nTo prevent a similar issue from occurring in the future, we have temporarily paused Presence code deployments while we investigate the update’s resource usage. Before releasing the update again, we will address the identified issues and test resource consumption and service stability, with extra attention to the conditions that we experienced in this scenario.",
"postmortem_body_last_updated_at": "2026-10-29T21:40:43.846Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-10-29T21:40:43.846Z",
"reminder_intervals": null,
"resolved_at": "2026-10-26T21:55:36.124Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/4m4gzc2sqf7g",
"started_at": "2026-10-26T21:07:34.066Z",
"status": "resolved",
"updated_at": "2026-10-29T21:40:45.300Z"
},
{
"components": [
{
"created_at": "2015-05-01T01:50:15.754Z",
"description": "History API Calls",
"group_id": "hymc9wr9f9k7",
"id": "8v81pmpgs8sq",
"name": "Storage and History Service",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-04T23:09:13.026Z"
},
{
"created_at": "2018-02-27T16:42:48.522Z",
"description": "All services running in Europe status",
"group_id": "fmkr256xnbyh",
"id": "9j2mvls8yv8p",
"name": "European Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:17:10.416Z"
}
],
"created_at": "2026-10-26T21:14:18.388Z",
"id": "hfvkv9hbj18h",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "8v81pmpgs8sq",
"name": "Realtime Network - Storage and Playback Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved. If you feel you were impacted, please contact support@pubnub.com.",
"created_at": "2026-10-26T21:46:22.330Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-26T21:46:22.330Z",
"id": "c8hh047446rq",
"incident_id": "hfvkv9hbj18h",
"status": "resolved",
"tweet_id": 2114836058294919400,
"twitter_updated_at": "2026-10-26T21:46:22.878Z",
"updated_at": "2026-10-26T21:46:22.881Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "8v81pmpgs8sq",
"name": "Realtime Network - Storage and Playback Service",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "A fix has been implemented and we are monitoring the results. If you feel you have been impacted by this, please reach out to support@pubnub.com.",
"created_at": "2026-10-26T21:17:51.645Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-26T21:17:51.645Z",
"id": "41sn1tg875tl",
"incident_id": "hfvkv9hbj18h",
"status": "monitoring",
"tweet_id": 2114828883573035300,
"twitter_updated_at": "2026-10-26T21:17:52.278Z",
"updated_at": "2026-10-26T21:17:52.280Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "8v81pmpgs8sq",
"name": "Realtime Network - Storage and Playback Service",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Users may be experiencing elevated errors with History readers in the Frankfurt point-of-presence. We are working on remediation and will update this as we make progress. If you believe you are affected and would like to discuss, please reach out to support@pubnub.com",
"created_at": "2026-10-26T21:14:18.517Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-26T21:14:18.517Z",
"id": "3pvc72rx9wx2",
"incident_id": "hfvkv9hbj18h",
"status": "investigating",
"tweet_id": 2114827990010073300,
"twitter_updated_at": "2026-10-26T21:14:19.237Z",
"updated_at": "2026-10-26T21:14:19.239Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-10-26T21:17:51.645Z",
"name": "Errors with history readers in FRA",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### Problem Description, Impact, and Resolution \n\nOn October 26, 2026, from approximately 20:56 to 21:12 UTC, users accessing History APIs in Frankfurt may have experienced elevated error rates and latency. A sudden surge in history read requests exceeded the available capacity of the infrastructure supporting these APIs, temporarily outpacing the autoscaling configuration for that service.\n\nService returned to normal at approximately 21:12 UTC. Following continued monitoring, the incident was declared resolved at 21:46 UTC.\n\n### Mitigation Steps and Recommended Future Preventative Measures \n\nWe increased capacity to address the immediate issue. To prevent a similar issue from occurring in the future, we are reviewing the autoscaling configuration of the service so it can better respond to similar traffic surges.",
"postmortem_body_last_updated_at": "2026-10-30T17:23:28.215Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-10-30T17:23:28.215Z",
"reminder_intervals": null,
"resolved_at": "2026-10-26T21:46:22.330Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/lymjwc9vlp2n",
"started_at": "2026-10-26T21:14:18.374Z",
"status": "resolved",
"updated_at": "2026-10-30T17:23:31.263Z"
},
{
"components": [
{
"created_at": "2015-05-01T01:50:05.984Z",
"description": "Join, Leave and Here Now API Calls",
"group_id": "hymc9wr9f9k7",
"id": "w3dkvn4lx0rl",
"name": "Presence Service",
"page_id": "j2pr9thwz01t",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.938Z"
}
],
"created_at": "2026-10-22T16:02:02.866Z",
"id": "b8d3z27vvzhz",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved. If you feel you were impacted, please contact support@pubnub.com.",
"created_at": "2026-10-22T16:35:58.354Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T16:35:58.354Z",
"id": "r30ns2f3lbgv",
"incident_id": "b8d3z27vvzhz",
"status": "resolved",
"tweet_id": 2113308393003962600,
"twitter_updated_at": "2026-10-22T16:35:59.080Z",
"updated_at": "2026-10-22T16:35:59.083Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results. If you were impacted by this issue, or feel you are still being impacted, please reach out to support@pubnub.com.",
"created_at": "2026-10-22T16:19:27.689Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T16:19:27.689Z",
"id": "k31r8vjnvx3h",
"incident_id": "b8d3z27vvzhz",
"status": "monitoring",
"tweet_id": 2113304237811908600,
"twitter_updated_at": "2026-10-22T16:19:28.406Z",
"updated_at": "2026-10-22T16:19:28.409Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-22T16:12:48.260Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T16:12:48.260Z",
"id": "0cgzxy2njxsm",
"incident_id": "b8d3z27vvzhz",
"status": "identified",
"tweet_id": 2113302570701865200,
"twitter_updated_at": "2026-10-22T16:12:51.104Z",
"updated_at": "2026-10-22T16:12:51.106Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Users may be experiencing elevated error and latency rates from our Frankfurt and Bombay points-of-presence. We are working on remediation and will update this as we make progress. If you are affected, reach out to support@pubnub.com.",
"created_at": "2026-10-22T16:02:02.975Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T16:02:02.975Z",
"id": "kdqgjzkyvwxp",
"incident_id": "b8d3z27vvzhz",
"status": "investigating",
"tweet_id": 2113299855250698500,
"twitter_updated_at": "2026-10-22T16:02:03.513Z",
"updated_at": "2026-10-22T16:02:03.516Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-10-22T16:19:27.689Z",
"name": "Errors and Latency for Presence in FRA and BOM",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### **Problem Description, Impact, and Resolution**\n\nOn October 22, from approximately 14:45 to 16:13 UTC, users of our Frankfurt \\(FRA\\) and Mumbai \\(BOM\\) points of presence may have seen elevated error rates and latency on Presence API calls. Other PubNub services were not affected.\n\nThe issue began during a software update to the Presence service in Frankfurt. The new version was not compatible with the previous version while both were running at the same time. That caused Presence servers in the region to restart repeatedly, and requests that could not reach a healthy server returned errors.\n\nOur team identified the update as the cause and rolled it back. Service returned to normal at approximately 16:13 UTC.\n\n### **Mitigation Steps and Recommended Future Preventative Measures**\n\nWe are fixing the incompatibility before this update is released again. We are also adding tests that run old and new versions side by side before every Presence release, rolling out updates one availability zone at a time, and improving our rollback procedure so it does not run incompatible versions together.",
"postmortem_body_last_updated_at": "2026-10-26T21:36:18.428Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-10-26T21:36:18.428Z",
"reminder_intervals": null,
"resolved_at": "2026-10-22T16:35:58.354Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/fzz1dpg61ssn",
"started_at": "2026-10-22T16:02:02.856Z",
"status": "resolved",
"updated_at": "2026-10-26T21:36:18.910Z"
},
{
"components": [
{
"created_at": "2013-10-15T02:19:10.032Z",
"description": "Global Data Stream Network Publish and Subscribe",
"group_id": "hymc9wr9f9k7",
"id": "1hkxj2q2rctq",
"name": "Publish/Subscribe Service",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T14:10:07.866Z"
},
{
"created_at": "2017-01-03T23:17:43.878Z",
"description": "Serverless compute for building and scaling realtime applications.",
"group_id": "nj61xcnxhrz5",
"id": "tsdmzyxfmkls",
"name": "Functions Service",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-12-11T22:10:16.234Z"
},
{
"created_at": "2015-05-01T01:50:15.754Z",
"description": "History API Calls",
"group_id": "hymc9wr9f9k7",
"id": "8v81pmpgs8sq",
"name": "Storage and History Service",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-04T23:09:13.026Z"
},
{
"created_at": "2018-03-01T16:42:57.194Z",
"description": "Secure secret and key store",
"group_id": "nj61xcnxhrz5",
"id": "5jkspb588c37",
"name": "Vault",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-12-11T22:10:16.292Z"
},
{
"created_at": "2015-05-01T01:50:42.910Z",
"description": "Channel Groups",
"group_id": "hymc9wr9f9k7",
"id": "frx8ppwtppf7",
"name": "Stream Controller Service",
"page_id": "j2pr9thwz01t",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T08:45:45.225Z"
},
{
"created_at": "2018-03-01T16:44:10.725Z",
"description": "Service to store key value pairs",
"group_id": "nj61xcnxhrz5",
"id": "5mcrt3n0xs27",
"name": "Key Value store",
"page_id": "j2pr9thwz01t",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-12-11T22:10:16.332Z"
},
{
"created_at": "2015-05-01T01:50:05.984Z",
"description": "Join, Leave and Here Now API Calls",
"group_id": "hymc9wr9f9k7",
"id": "w3dkvn4lx0rl",
"name": "Presence Service",
"page_id": "j2pr9thwz01t",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.938Z"
},
{
"created_at": "2018-03-01T16:44:42.866Z",
"description": "Scheduler service to stop and start functions",
"group_id": "nj61xcnxhrz5",
"id": "w89b16t9cdky",
"name": "Scheduler Service",
"page_id": "j2pr9thwz01t",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-12-11T22:10:16.372Z"
},
{
"created_at": "2015-05-01T01:52:31.158Z",
"description": "Security Access Layer Grant/Revoke",
"group_id": "hymc9wr9f9k7",
"id": "hm568vz5qnvw",
"name": "Access Manager Service",
"page_id": "j2pr9thwz01t",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T15:13:33.262Z"
},
{
"created_at": "2016-10-28T17:44:19.025Z",
"description": "Dual DNS resolution service to find closest/lowest latency PoP",
"group_id": "hymc9wr9f9k7",
"id": "r9dxbwjjpbxy",
"name": "DNS Service",
"page_id": "j2pr9thwz01t",
"position": 7,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-22T02:12:20.525Z"
},
{
"created_at": "2015-05-01T01:51:42.397Z",
"description": "APNs, GCM and WPNs",
"group_id": "hymc9wr9f9k7",
"id": "lxdltc90f0hl",
"name": "Mobile Push Gateway",
"page_id": "j2pr9thwz01t",
"position": 8,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T08:45:45.296Z"
},
{
"created_at": "2021-04-20T12:25:36.667Z",
"description": "https://www.pubnub.com/products/app-context/",
"group_id": "hymc9wr9f9k7",
"id": "tq76t6mrgqwv",
"name": "App Context Service",
"page_id": "j2pr9thwz01t",
"position": 10,
"showcase": false,
"start_date": "2021-04-20T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-10-20T08:45:45.313Z"
}
],
"created_at": "2026-10-22T01:11:21.545Z",
"id": "4j03scdh7h0j",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "8v81pmpgs8sq",
"name": "Realtime Network - Storage and Playback Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "frx8ppwtppf7",
"name": "Realtime Network - Stream Controller Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hm568vz5qnvw",
"name": "Realtime Network - Access Manager Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r9dxbwjjpbxy",
"name": "Realtime Network - DNS Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "lxdltc90f0hl",
"name": "Realtime Network - Mobile Push Gateway",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tq76t6mrgqwv",
"name": "Realtime Network - App Context Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tsdmzyxfmkls",
"name": "Functions - Functions Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5jkspb588c37",
"name": "Functions - Vault",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5mcrt3n0xs27",
"name": "Functions - Key Value store",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w89b16t9cdky",
"name": "Functions - Scheduler Service",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We have had no additional reports related to this issue. If you feel you were impacted, please contact support@pubnub.com. A post-mortem will follow.",
"created_at": "2026-10-22T02:47:37.538Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T02:47:37.538Z",
"id": "g6t3jdjyyqky",
"incident_id": "4j03scdh7h0j",
"status": "resolved",
"tweet_id": 2113099932194509000,
"twitter_updated_at": "2026-10-22T02:47:38.149Z",
"updated_at": "2026-10-22T02:47:38.151Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "8v81pmpgs8sq",
"name": "Realtime Network - Storage and Playback Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "frx8ppwtppf7",
"name": "Realtime Network - Stream Controller Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hm568vz5qnvw",
"name": "Realtime Network - Access Manager Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r9dxbwjjpbxy",
"name": "Realtime Network - DNS Service",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "lxdltc90f0hl",
"name": "Realtime Network - Mobile Push Gateway",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tq76t6mrgqwv",
"name": "Realtime Network - App Context Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tsdmzyxfmkls",
"name": "Functions - Functions Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5jkspb588c37",
"name": "Functions - Vault",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5mcrt3n0xs27",
"name": "Functions - Key Value store",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w89b16t9cdky",
"name": "Functions - Scheduler Service",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We have completed remediation for this issue. Impact was minor: only one end user reported an issue, but we are monitoring for additional impact. Please reach out to support@pubnub.com if you have a related issue to report.",
"created_at": "2026-10-22T02:12:20.583Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T02:12:20.583Z",
"id": "gf02rns6k8v8",
"incident_id": "4j03scdh7h0j",
"status": "monitoring",
"tweet_id": 2113091054299980000,
"twitter_updated_at": "2026-10-22T02:12:21.493Z",
"updated_at": "2026-10-22T02:12:21.496Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "8v81pmpgs8sq",
"name": "Realtime Network - Storage and Playback Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "frx8ppwtppf7",
"name": "Realtime Network - Stream Controller Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hm568vz5qnvw",
"name": "Realtime Network - Access Manager Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r9dxbwjjpbxy",
"name": "Realtime Network - DNS Service",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "lxdltc90f0hl",
"name": "Realtime Network - Mobile Push Gateway",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tq76t6mrgqwv",
"name": "Realtime Network - App Context Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tsdmzyxfmkls",
"name": "Functions - Functions Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5jkspb588c37",
"name": "Functions - Vault",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5mcrt3n0xs27",
"name": "Functions - Key Value store",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w89b16t9cdky",
"name": "Functions - Scheduler Service",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We have identified the issue and are well into remediation. Please reach out to support@pubnub.com if you are experiencing issues.",
"created_at": "2026-10-22T01:55:29.472Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T01:55:29.472Z",
"id": "7wdkc2v9xjgg",
"incident_id": "4j03scdh7h0j",
"status": "identified",
"tweet_id": 2113086812323500000,
"twitter_updated_at": "2026-10-22T01:55:30.126Z",
"updated_at": "2026-10-22T01:55:30.129Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "8v81pmpgs8sq",
"name": "Realtime Network - Storage and Playback Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "frx8ppwtppf7",
"name": "Realtime Network - Stream Controller Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w3dkvn4lx0rl",
"name": "Realtime Network - Presence Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hm568vz5qnvw",
"name": "Realtime Network - Access Manager Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r9dxbwjjpbxy",
"name": "Realtime Network - DNS Service",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "lxdltc90f0hl",
"name": "Realtime Network - Mobile Push Gateway",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tq76t6mrgqwv",
"name": "Realtime Network - App Context Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tsdmzyxfmkls",
"name": "Functions - Functions Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5jkspb588c37",
"name": "Functions - Vault",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5mcrt3n0xs27",
"name": "Functions - Key Value store",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w89b16t9cdky",
"name": "Functions - Scheduler Service",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-22T01:17:33.431Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T01:17:33.431Z",
"id": "nt8q38gbttc0",
"incident_id": "4j03scdh7h0j",
"status": "investigating",
"tweet_id": 2113077266855108900,
"twitter_updated_at": "2026-10-22T01:17:34.282Z",
"updated_at": "2026-10-22T01:17:34.286Z",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "An error in updating security certificates may be causing some clients to experience connection issue. We are investigating and will update as we find out more.",
"created_at": "2026-10-22T01:11:21.595Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-22T01:11:21.595Z",
"id": "9v8k24qm3zmm",
"incident_id": "4j03scdh7h0j",
"status": "investigating",
"tweet_id": 2113075706620264700,
"twitter_updated_at": "2026-10-22T01:11:22.318Z",
"updated_at": "2026-10-22T01:11:22.320Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-10-22T02:12:20.583Z",
"name": "Certificate Issue Causing Potential Connection Issues",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### **Problem Description, Impact, and Resolution**\n\nFrom 19:26 UTC on October 21 until approximately 02:04 UTC on October 22, some PubNub endpoints served TLS certificates that our certificate authority had revoked. Clients that check certificate revocation could have failed to connect to the affected endpoints during that window.\n\nOur certificate authority found that about 400,000 certificates it had issued contained policy information that did not match its published certificate policy. The certificates were not compromised, but industry rules require such certificates to be revoked and replaced. PubNub's certificates for \\*.pubnub.com, \\*.pubnub.net, \\*.pndsn.com and \\*.pubnubapi.com were among them and were revoked on October 21. Our replacement certificates were not deployed to all of our endpoints until after the revocation took effect.\n\nClients that do not check certificate revocation were not affected, nor were connections to the default SDK origin [ps.pndsn.com](http://ps.pndsn.com). We were alerted by an end user of one of our customers. We deployed the replacement certificates to all regions, completing at approximately 02:04 UTC on October 22. Hostnames and the certificate expiry date \\(January 7, 2027\\) are unchanged.\n\n### **Mitigation Steps and Recommended Future Preventative Measures**\n\nWe are adding monitoring that alerts our operations team if any certificate we serve is revoked. We are also changing our certificate replacement process so that replacements are deployed to production promptly once issued, and so that certificate updates reach all regions faster.",
"postmortem_body_last_updated_at": "2026-10-26T21:31:12.718Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-10-26T21:31:12.718Z",
"reminder_intervals": null,
"resolved_at": "2026-10-22T02:47:37.538Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/q5x9mhyy641f",
"started_at": "2026-10-22T01:11:21.539Z",
"status": "resolved",
"updated_at": "2026-10-26T21:31:14.190Z"
},
{
"components": [],
"created_at": "2026-10-09T17:30:00.000Z",
"id": "3rj83byxmvx4",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "Starting at 17:32 UTC on October 9, 2026 we observed elevated 5XX error rates and latency on the Subscribe endpoint globally. The issue was resolved by 18:18 UTC and services have returned to normal levels. A limited number of customers may have been affected.\n\nA root cause analysis will be published in the coming days.\n\nWe sincerely apologize for any impact this may have caused. If you feel you were affected and wish to discuss, please email support@pubnub.com.",
"created_at": "2026-10-09T18:45:49.950Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-09T17:30:00.000Z",
"id": "04b3m087w1nk",
"incident_id": "3rj83byxmvx4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T18:45:49.950Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Increased Subscribe error rate and latency observed globally",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### **Problem Description, Impact, and Resolution** \n\nOn October 9, 2026, between 17:30 and 18:20 UTC, we observed elevated HTTP 5xx errors affecting the Subscribe endpoint for a subset of customers. Some subscribe requests may have failed, and a subset of messages may have been delayed or not delivered in real time. No messages were lost, and customers with Message Persistence enabled could retrieve affected messages from History.\n\nThe incident was caused by an unexpected increase in traffic from a high-volume load test against a development keyset that did not have the expected limits set. The resulting load temporarily exhausted memory in some shared infrastructure, which caused the issue.\n\nWe stabilized the affected infrastructure by applying a rate limit to the keyset generating the excess load. Service returned to normal by 18:20 UTC.\n\n\n\n### **Mitigation Steps and Recommended Future Preventative Measures** \n\nWe applied a temporary publish rate limit to contain the unexpected traffic and restore service while preserving the affected keyset’s availability.\n\nTo help prevent similar incidents, PubNub is adding additional checks to ensure that safeguards are correctly in place to detect and automatically limit unusually heavy load before it can affect other traffic sharing the same infrastructure. We are also updating operational procedures for subscriber memory and autoscaling changes to ensure that capacity adjustments remain effective during incident mitigation.",
"postmortem_body_last_updated_at": "2026-10-14T17:32:06.213Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-10-14T17:32:06.213Z",
"reminder_intervals": null,
"resolved_at": "2026-10-09T17:30:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/q2fcfsp1ddbd",
"started_at": "2026-10-09T18:45:49.886Z",
"status": "resolved",
"updated_at": "2026-10-14T17:32:06.608Z"
},
{
"components": [
{
"created_at": "2013-10-15T02:19:10.032Z",
"description": "Global Data Stream Network Publish and Subscribe",
"group_id": "hymc9wr9f9k7",
"id": "1hkxj2q2rctq",
"name": "Publish/Subscribe Service",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T14:10:07.866Z"
},
{
"created_at": "2018-02-27T16:42:14.196Z",
"description": "All service running across North America status",
"group_id": "fmkr256xnbyh",
"id": "wbrc65lqrp03",
"name": "North America Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.875Z"
}
],
"created_at": "2026-08-25T13:22:48.467Z",
"id": "cz6q37z3s1d5",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "With no further issues observed for the past 30 minutes, the incident has been resolved. We will follow up soon with a root cause analysis. If you believe you experienced an impact related to this incident, please report it to PubNub Support at support@pubnub.com.",
"created_at": "2026-08-25T14:51:38.932Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T14:51:38.932Z",
"id": "62mjk2hb3v9k",
"incident_id": "cz6q37z3s1d5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T14:51:38.932Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "As of 13:55 UTC, the issue affecting Publish and Subscribe services in the US East PoP has been mitigated. Synthetic monitoring has returned to normal, and no missed messages have been detected.\n\nThe incident is currently believed to have been related to a monitoring failure. Further investigation is ongoing, and the service continues to be monitored closely.",
"created_at": "2026-08-25T14:10:07.908Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T14:10:07.908Z",
"id": "l2255skckvw7",
"incident_id": "cz6q37z3s1d5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T14:10:07.908Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "As of 12:15 UTC, Users in our US East PoP are currently experiencing an issue with Publish and Subscribe service that is causing some messages to not appear in calls to that service. No errors are being returned from the service, despite messages not being returned correctly. We are investigating the issue.",
"created_at": "2026-08-25T13:22:48.610Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T13:22:48.610Z",
"id": "qqbrpkmg8f2v",
"incident_id": "cz6q37z3s1d5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T13:22:48.610Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-08-25T14:10:07.908Z",
"name": "Potential for some missed messages for subscribers in US East PoP",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### **Problem Description, Impact, and Resolution** \n\nOn Tuesday, August 25th, 2026, at 12:10 UTC, our internal monitoring alerted us to an issue where a very small subset of messages within one availability zone of one region \\(our US East point of presence\\) may not have been immediately delivered to subscribers in that same region. All messages were persisted normally within PubNub Persistence service, **thus no message data was lost.** Traffic to or from any other region or AZ was not affected, and no other PubNub services were affected. No customers reported impact.\n\n### **Root Cause**\n\nDuring planned internal load testing, a group of internal routing servers was removed from service. Due to a configuration flaw, their network addresses were not fully retired and remained cached by our publishing layer.\n\nThis presented no problem for the most part. However, one address was subsequently reassigned to an unrelated internal component that responded normally on the same protocol and port. Because that response appeared successful, the publishing layer received no error, and messages sent to that address were routed incorrectly.\n\nA rolling restart of the affected servers cleared the outdated addresses at 13:25 UTC.\n\n### **Mitigation Steps and Recommended Future Preventative Measures** \n\n* Once the affected servers were identified, a rolling restart was performed to clear the outdated routing addresses.\n* Following an investigation that confirmed the root cause, a fix ensuring outdated internal routing addresses are correctly retired was deployed to all publishing servers worldwide on August 25th, 2026 at 22:00 UTC.",
"postmortem_body_last_updated_at": "2026-08-28T17:46:59.778Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-08-28T17:46:59.778Z",
"reminder_intervals": null,
"resolved_at": "2026-08-25T14:51:38.932Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/qctzhs1v265m",
"started_at": "2026-08-25T13:22:48.451Z",
"status": "resolved",
"updated_at": "2026-08-28T17:46:59.796Z"
},
{
"components": [
{
"created_at": "2015-05-01T04:33:16.368Z",
"description": "Customer Account Dashboard and Billing Portal",
"group_id": "60bpnf4hmfcr",
"id": "4j3ty03r7c3s",
"name": "Administration Portal",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-15T08:00:07.681Z"
}
],
"created_at": "2026-08-13T17:00:33.326Z",
"id": "w5ct0vj26qx0",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "4j3ty03r7c3s",
"name": "Website and Portals - Administration Portal",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-08-15T08:00:07.905Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-15T08:00:07.905Z",
"id": "3j5jcw0kjbnl",
"incident_id": "w5ct0vj26qx0",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-15T08:00:07.905Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "4j3ty03r7c3s",
"name": "Website and Portals - Administration Portal",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-08-15T05:00:07.835Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-15T05:00:07.835Z",
"id": "59x0xbrv0xng",
"incident_id": "w5ct0vj26qx0",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-15T05:00:07.835Z",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "Maintenance will begin as scheduled in 60 minutes.",
"created_at": "2026-08-15T04:00:14.493Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-15T04:00:14.493Z",
"id": "zpm33zkz6fcs",
"incident_id": "w5ct0vj26qx0",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-15T04:00:14.493Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "4j3ty03r7c3s",
"name": "Website and Portals - Administration Portal",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "We will be undergoing scheduled maintenance during this time.",
"created_at": "2026-08-13T17:04:30.612Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-13T17:04:30.612Z",
"id": "1r5y1jl57bsn",
"incident_id": "w5ct0vj26qx0",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-13T17:04:30.612Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "4j3ty03r7c3s",
"name": "Website and Portals - Administration Portal",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "We will be undergoing scheduled maintenance during this time.",
"created_at": "2026-08-13T17:03:00.769Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-13T17:03:00.769Z",
"id": "052s90150kp8",
"incident_id": "w5ct0vj26qx0",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-13T17:03:00.769Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "4j3ty03r7c3s",
"name": "Website and Portals - Administration Portal",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Please be advised of scheduled maintenance for the PubNub Admin Portal on Saturday, August 15, 2026.\n\nImpact:\n- There will be NO interruption to the Messaging Platform and NO impact on end users.\n- Admin Portal \u0026 Billing: Intermittent availability as we upgrade databases in short 5-10min bursts.\n\nMaintenance Window:\n- UTC: 5:00 AM - 8:00 AM\n- EDT(EST): 01:00 AM - 4:00 AM\n- PDT(PST): 10:00 PM - 01:00 AM (Friday night, August 14)\n\n\nSecurity \u0026 Status:\nFollow updates about this maintenance window at status.pubnub.com.\n\nThank you for your understanding.",
"created_at": "2026-08-13T17:00:33.395Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-13T17:00:33.395Z",
"id": "f9f17gqlsgll",
"incident_id": "w5ct0vj26qx0",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-13T17:00:33.395Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled Maintenance: PubNub Admin Portal",
"page_id": "j2pr9thwz01t",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-08-15T08:00:07.905Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-08-15T05:00:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": "2026-08-15T04:00:14.791Z",
"scheduled_until": "2026-08-15T08:00:00.000Z",
"shortlink": "https://stspg.io/cx3lyzhf4922",
"started_at": "2026-08-13T17:00:33.316Z",
"status": "completed",
"updated_at": "2026-08-15T08:00:08.003Z"
},
{
"components": [
{
"created_at": "2013-10-15T02:19:10.032Z",
"description": "Global Data Stream Network Publish and Subscribe",
"group_id": "hymc9wr9f9k7",
"id": "1hkxj2q2rctq",
"name": "Publish/Subscribe Service",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T14:10:07.866Z"
},
{
"created_at": "2017-01-03T23:17:43.878Z",
"description": "Serverless compute for building and scaling realtime applications.",
"group_id": "nj61xcnxhrz5",
"id": "tsdmzyxfmkls",
"name": "Functions Service",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-12-11T22:10:16.234Z"
},
{
"created_at": "2018-02-27T16:42:14.196Z",
"description": "All service running across North America status",
"group_id": "fmkr256xnbyh",
"id": "wbrc65lqrp03",
"name": "North America Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.875Z"
},
{
"created_at": "2018-02-27T16:42:48.522Z",
"description": "All services running in Europe status",
"group_id": "fmkr256xnbyh",
"id": "9j2mvls8yv8p",
"name": "European Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:17:10.416Z"
},
{
"created_at": "2018-02-27T16:43:29.592Z",
"description": "All Services running across Asia Pacific",
"group_id": "fmkr256xnbyh",
"id": "fkmgfx9f3bsk",
"name": "Asia Pacific Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.919Z"
},
{
"created_at": "2018-02-27T16:44:00.007Z",
"description": "All services running in Southern Asia status",
"group_id": "fmkr256xnbyh",
"id": "4km3s664qd7l",
"name": "Southern Asia Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.955Z"
}
],
"created_at": "2026-08-14T14:29:49.892Z",
"id": "kgvrqkjhy98x",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tsdmzyxfmkls",
"name": "Functions - Functions Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9j2mvls8yv8p",
"name": "Points of Presence - European Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "fkmgfx9f3bsk",
"name": "Points of Presence - Asia Pacific Points of Presence",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4km3s664qd7l",
"name": "Points of Presence - Southern Asia Points of Presence",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "From 13:55 to 14:15 UTC, users globally may have experienced errors with failure on Publish, and published messages may not have been received by subscribers. We also experienced errors and failures with PubNub Functions, and delays with Events \u0026 Actions during the period. Services have returned to normal and we are actively monitoring.\n\nA root cause analysis will be published in the coming days. If you believe you were impacted and would like to speak with us, please report impact to support@pubnub.com.",
"created_at": "2026-08-14T14:29:49.948Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-14T14:29:49.948Z",
"id": "133nfwcn9sk0",
"incident_id": "kgvrqkjhy98x",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-14T14:29:49.948Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": null,
"name": "Global Errors and Failures with Publish and Functions, Delays with Events \u0026 Actions",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### **Problem Description, Impact, and Resolution** \n\nAt approximately **13:55 UTC on Aug 14, 2026**, we observed elevated publish errors and message replication failures in our publish/subscribe service, which also caused latency in other PubNub services globally. Customers may have experienced increased publish error rates, delayed or missed message delivery, delayed message persistence, and increased latency for Functions and Events \u0026 Actions workflows.\n\nThe root cause of the incident was an unusually large concentration of global publish traffic that was not limited by our throttling layers. That traffic created resource pressure in the publish and replication layers, increased load on storage systems, and caused downstream processing delays in dependent services. We mitigated the issue by adjusting targeted traffic controls, increasing capacity for affected publish and replication components, and isolating the high-volume traffic pattern to reduce broader platform impact. The issue was resolved at approximately **14:15 UTC on Aug 14, 2026**.\n\nThis issue occurred because of an issue with our automated controls specific to this exceptional traffic pattern. As a result, the increased load affected multiple services before a mitigation could be fully applied. \n\n### **Mitigation Steps and Recommended Future Preventative Measures**\n\nTo prevent a similar issue from occurring in the future, we have isolated the identified high-volume traffic pattern onto dedicated infrastructure and also increased baseline capacity for the affected components across our PoPs.\n\nWe are also further strengthening our traffic detection and management processes for this exceptional load pattern so it will be identified and contained earlier without cascading impact across dependent services.",
"postmortem_body_last_updated_at": "2026-08-20T16:58:05.407Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-08-20T16:58:05.407Z",
"reminder_intervals": null,
"resolved_at": "2026-08-14T14:29:49.948Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/wd3pg1rlq378",
"started_at": "2026-08-14T14:29:49.875Z",
"status": "resolved",
"updated_at": "2026-08-20T16:58:05.421Z"
},
{
"components": [
{
"created_at": "2013-10-15T02:19:10.032Z",
"description": "Global Data Stream Network Publish and Subscribe",
"group_id": "hymc9wr9f9k7",
"id": "1hkxj2q2rctq",
"name": "Publish/Subscribe Service",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-25T14:10:07.866Z"
},
{
"created_at": "2018-02-27T16:42:14.196Z",
"description": "All service running across North America status",
"group_id": "fmkr256xnbyh",
"id": "wbrc65lqrp03",
"name": "North America Points of Presence",
"page_id": "j2pr9thwz01t",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-26T21:12:03.875Z"
}
],
"created_at": "2026-06-10T20:14:36.048Z",
"id": "rxwm0z7krbhf",
"impact": "none",
"impact_override": "none",
"incident_updates": [
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Services have returned to normal. A root cause analysis will be published in the coming days. If you believe you were impacted and would like to speak with us, please report impact to support@pubnub.com.",
"created_at": "2026-06-10T20:45:41.309Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-10T20:45:41.309Z",
"id": "mrfj17tbnt9c",
"incident_id": "rxwm0z7krbhf",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-10T20:45:41.309Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "The PubNub Technical Staff identified that failures are limited to publishes from US-East and has identified, have applied a fix, and are now monitoring the results. \n\nIf you are experiencing issues that you believe to be related to this incident, please report the details to PubNub Support (support@pubnub.com).",
"created_at": "2026-06-10T20:27:51.979Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-10T20:27:51.979Z",
"id": "7pgd3psdvv3z",
"incident_id": "rxwm0z7krbhf",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-10T20:27:51.979Z",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "1hkxj2q2rctq",
"name": "Realtime Network - Publish/Subscribe Service",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "wbrc65lqrp03",
"name": "Points of Presence - North America Points of Presence",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Starting at 17:00 UTC on June 10, a subset of publishes originating from the North America POP failed to replicate to subscribers globally. PubNub Technical Staff is investigating, and more information will be posted as it becomes available.",
"created_at": "2026-06-10T20:14:36.147Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-10T20:14:36.147Z",
"id": "lqn36jk8vmj9",
"incident_id": "rxwm0z7krbhf",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-10T20:14:36.147Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-06-10T20:27:51.979Z",
"name": "Replication failures",
"page_id": "j2pr9thwz01t",
"postmortem_body": "## Problem Description, Impact, and Resolution \n\nAt 19:50 UTC on June 10, 2026, we observed a small fraction of publishes originating from US-EAST-1 failing to replicate to subscribers globally. We removed the degraded publisher pod from service and the issue was resolved at 21:21 UTC on June 10, 2026. The root cause of the incident was triggered by a single process that fell into a degraded state where it continued receiving inbound traffic and passing health checks, but traffic sent outbound from the process was failing at an abnormally high rate. Our automated health check/recovery system did not auto-detect and replace the degraded process because its health check API reported itself as healthy. \n\n## Mitigation Steps and Recommended Future Preventative Measures \n\nTo prevent a similar issue from occurring in the future, we are improving the data within our health check APIs to return more complete performance metrics over a rolling time window. We are enhancing the issue detection logic to detect more patterns that infer process failure, even if the process itself is reporting as healthy.",
"postmortem_body_last_updated_at": "2026-06-16T00:00:33.023Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-06-16T00:00:33.023Z",
"reminder_intervals": null,
"resolved_at": "2026-06-10T20:45:41.309Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/vj0zdnglj3bn",
"started_at": "2026-06-10T20:14:36.036Z",
"status": "resolved",
"updated_at": "2026-06-16T00:00:33.039Z"
},
{
"components": [],
"created_at": "2026-06-09T14:00:54.000Z",
"id": "2yp4r6sdms7l",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "From 13:20 to 13:48 UTC, users globally may have experienced errors with failure on Publish, and published messages may not have been received by subscribers. We also experienced errors and failures with PubNub Functions, and delays with Events \u0026 Actions during the period. Services have returned to normal and we are actively monitoring.\n\nA root cause analysis will be published in the coming days. If you believe you were impacted and would like to speak with us, please report impact to support@pubnub.com.",
"created_at": "2026-06-09T14:00:54.249Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-09T14:00:54.000Z",
"id": "76x255tlnqgg",
"incident_id": "2yp4r6sdms7l",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-09T14:02:07.548Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Global Errors and Failures with Publish and Functions, Delays with Events \u0026 Actions",
"page_id": "j2pr9thwz01t",
"postmortem_body": "### **Problem Description, Impact, and Resolution** \n\nAt approximately **13:20 UTC on June 9, 2026**, we observed elevated publish errors and message replication failures in our publish/subscribe service, which also caused latency in other PubNub services globally. Customers may have experienced increased publish error rates, delayed or missed message delivery, delayed message persistence, and increased latency for Functions and Events \u0026 Actions workflows.\n\n\n\nThe root cause of the incident was an unusually large concentration of global publish traffic that was not limited by our throttling layers. That traffic created resource pressure in the publish and replication layers, increased load on storage systems, and caused downstream processing delays in dependent services. We mitigated the issue by adjusting targeted traffic controls, increasing capacity for affected publish and replication components, and isolating the high-volume traffic pattern to reduce broader platform impact. The issue was resolved at approximately **13:48 UTC on June 9, 2026**.\n\n\n\nThis issue occurred because we did not have sufficient automated controls and isolation processes in place to protect shared infrastructure from this type of exceptional traffic pattern. As a result, the increased load affected multiple services before mitigation could be fully applied. \n\n\n\n### **Mitigation Steps and Recommended Future Preventative Measures** \n\nTo prevent a similar issue from occurring in the future, we have isolated the identified high-volume traffic pattern onto dedicated infrastructure and also increased baseline capacity for the affected components across our PoPs.\n\n\n\nWe are also further strengthening our traffic detection and management processes for exceptional load patterns so they can be identified and contained earlier without cascading impact across dependent services.",
"postmortem_body_last_updated_at": "2026-06-12T18:33:26.762Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2026-06-12T18:33:26.762Z",
"reminder_intervals": null,
"resolved_at": "2026-06-09T14:00:54.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/pb1d9hy71tpp",
"started_at": "2026-06-09T14:00:54.000Z",
"status": "resolved",
"updated_at": "2026-06-12T18:33:26.777Z"
}
],
"page": {
"id": "j2pr9thwz01t",
"name": "PubNub",
"url": "https://status.pubnub.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}