Snapshot 207695
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2013-08-26T04:27:06.711Z",
"description": null,
"group_id": null,
"id": "r86fqtcmnn65",
"name": "Web",
"page_id": "ltljpr68dygn",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-10T05:07:21.220Z"
},
{
"created_at": "2015-05-11T01:35:06.981Z",
"description": null,
"group_id": null,
"id": "1mpkbbd1kh0h",
"name": "Agent API",
"page_id": "ltljpr68dygn",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-03T20:02:23.943Z"
},
{
"components": [
"r21z46hqn47c",
"fx3sc4nr2w8v"
],
"created_at": "2026-05-11T02:54:29.661Z",
"description": null,
"group_id": null,
"id": "0zsswmznk66n",
"name": "REST API",
"page_id": "ltljpr68dygn",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-11T02:54:44.557Z"
},
{
"created_at": "2015-02-28T01:22:55.169Z",
"description": null,
"group_id": null,
"id": "3n7fzwsysx86",
"name": "Job Queue",
"page_id": "ltljpr68dygn",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T05:46:23.085Z"
},
{
"components": [
"m3fr6snn2496",
"rj879wvx0lyd",
"r0jrhpsydz4w",
"mk54vxw70994"
],
"created_at": "2015-05-11T01:32:50.895Z",
"description": null,
"group_id": null,
"id": "tplm771kdwfs",
"name": "Notifications",
"page_id": "ltljpr68dygn",
"position": 6,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-11T02:54:44.596Z"
},
{
"components": [
"9pwnnl0dp0j5",
"87hpgxv274l0",
"sd4q7mk8n853",
"fb1llxplp5xq",
"h8x0zl3m0qj7"
],
"created_at": "2025-02-06T06:29:12.958Z",
"description": null,
"group_id": null,
"id": "56ph4s45xl9s",
"name": "Hosted Agents",
"page_id": "ltljpr68dygn",
"position": 8,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-11T02:54:44.623Z"
},
{
"components": [
"75z9h6yq0vrj",
"12zzrz9bykcj",
"jdvdr3k7kdbq"
],
"created_at": "2025-10-30T02:26:14.118Z",
"description": null,
"group_id": null,
"id": "dr5xs6jv5m8v",
"name": "Package Registries",
"page_id": "ltljpr68dygn",
"position": 9,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-11T02:54:44.638Z"
},
{
"components": [
"sw9wpknm8p8g",
"dmqj9zfy07kf",
"msyttmm8jpns"
],
"created_at": "2022-09-06T05:23:51.907Z",
"description": null,
"group_id": null,
"id": "lnkc32srq0df",
"name": "Test Engine",
"page_id": "ltljpr68dygn",
"position": 10,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-11T02:54:44.653Z"
},
{
"created_at": "2019-05-27T11:22:24.081Z",
"description": null,
"group_id": null,
"id": "pcwww1g6plvh",
"name": "SCM Integrations",
"page_id": "ltljpr68dygn",
"position": 11,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-11T02:54:44.666Z"
},
{
"components": [
"z9xdz6bnf069",
"7hzjm77m5lhd",
"r0qvv92dtt1x",
"92bvq28m13sk",
"nmmdp7mwb1nw",
"nf6qypynz4gc"
],
"created_at": "2015-05-11T01:19:03.082Z",
"description": "Third party SCM providers which may affect your builds",
"group_id": null,
"id": "wj7nh2k1l54h",
"name": "SCM Providers",
"page_id": "ltljpr68dygn",
"position": 12,
"showcase": false,
"start_date": null,
"status": "degraded_performance",
"updated_at": "2026-05-11T02:54:44.700Z"
},
{
"components": [
"m20p1dn7lr5h",
"5m66h4qss3g9",
"sq0ls35dpmvy",
"3bwdr960jnym"
],
"created_at": "2015-05-11T01:17:54.959Z",
"description": "Third party services we depend upon",
"group_id": null,
"id": "hd242hws8942",
"name": "Third Party Services",
"page_id": "ltljpr68dygn",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-11T02:54:44.728Z"
},
{
"created_at": "2023-05-30T05:57:03.881Z",
"description": "https://buildkite.com/docs",
"group_id": null,
"id": "8wzdjbrhcyjk",
"name": "Docs",
"page_id": "ltljpr68dygn",
"position": 14,
"showcase": false,
"start_date": "2023-05-30T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-05-11T02:54:44.739Z"
}
],
"incidents": [
{
"components": [
{
"created_at": "2015-02-28T01:22:55.169Z",
"description": null,
"group_id": null,
"id": "3n7fzwsysx86",
"name": "Job Queue",
"page_id": "ltljpr68dygn",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T05:46:23.085Z"
}
],
"created_at": "2026-11-03T18:14:39.079Z",
"id": "1vywxdrq7r2f",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "3n7fzwsysx86",
"name": "Job Queue",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We have seen recovery and latency has returned to normal.",
"created_at": "2026-11-03T20:22:55.034Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T20:22:55.034Z",
"id": "l5zy85cvc8m2",
"incident_id": "1vywxdrq7r2f",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T20:22:55.034Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We’ve scaled out the resources that process incoming webhooks and are seeing recovery. We’re now working through the backlog of queued jobs and continuing to monitor",
"created_at": "2026-11-03T19:43:44.356Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T19:43:44.356Z",
"id": "76p5b6qwqyjh",
"incident_id": "1vywxdrq7r2f",
"status": "monitoring",
"tweet_id": 2117704300168454400,
"twitter_updated_at": "2026-11-03T19:43:45.032Z",
"updated_at": "2026-11-03T19:43:45.036Z",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "We are continuing to investigate. At this time, the issue is limited to a small subset of customers.",
"created_at": "2026-11-03T18:54:58.119Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T18:54:58.000Z",
"id": "2zbzd6zz0nkm",
"incident_id": "1vywxdrq7r2f",
"status": "investigating",
"tweet_id": 2117692025751638500,
"twitter_updated_at": "2026-11-03T18:54:58.574Z",
"updated_at": "2026-11-03T19:02:30.848Z",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "We're seeing increased latency and error rates for a subset of our customers. We're currently investigating and will provide status updates as they become available.",
"created_at": "2026-11-03T18:14:39.136Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T18:14:39.136Z",
"id": "22h2sd870f4t",
"incident_id": "1vywxdrq7r2f",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T18:14:39.136Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-03T19:43:44.356Z",
"name": "Increased latency and error rates",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-03T20:22:55.034Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/y06rrgzzwqz4",
"started_at": "2026-11-03T18:14:39.072Z",
"status": "resolved",
"updated_at": "2026-11-03T20:22:55.065Z"
},
{
"components": [
{
"created_at": "2013-08-26T04:27:06.655Z",
"description": null,
"group_id": "0zsswmznk66n",
"id": "r21z46hqn47c",
"name": "REST API",
"page_id": "ltljpr68dygn",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-03T20:02:23.913Z"
},
{
"created_at": "2015-05-11T01:35:06.981Z",
"description": null,
"group_id": null,
"id": "1mpkbbd1kh0h",
"name": "Agent API",
"page_id": "ltljpr68dygn",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-03T20:02:23.943Z"
}
],
"created_at": "2026-11-03T19:40:03.371Z",
"id": "5lqdn0tqmdjc",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "1mpkbbd1kh0h",
"name": "Agent API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r21z46hqn47c",
"name": "REST API - REST API",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "API errors have subsided and we're seeing recovery. From 19:19-19:39 UTC we saw elevated errors on the Agent API and REST API.",
"created_at": "2026-11-03T20:12:38.812Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T20:12:38.000Z",
"id": "4lkzrwbjzzmb",
"incident_id": "5lqdn0tqmdjc",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T20:21:28.402Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "1mpkbbd1kh0h",
"name": "Agent API",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "r21z46hqn47c",
"name": "REST API - REST API",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "We're seeing recovery with previously elevated API error rates.",
"created_at": "2026-11-03T20:02:23.991Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T19:59:23.000Z",
"id": "jr52n8jtpkdt",
"incident_id": "5lqdn0tqmdjc",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T20:03:49.378Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "1mpkbbd1kh0h",
"name": "Agent API",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "r21z46hqn47c",
"name": "REST API - REST API",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-11-03T19:59:05.961Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T19:42:05.000Z",
"id": "4d1c2qtgrftg",
"incident_id": "5lqdn0tqmdjc",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T19:59:47.904Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "1mpkbbd1kh0h",
"name": "Agent API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r21z46hqn47c",
"name": "REST API - REST API",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We're investigating elevated error rates with our API.",
"created_at": "2026-11-03T19:40:03.559Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T19:40:03.559Z",
"id": "c82qmpmtl2qv",
"incident_id": "5lqdn0tqmdjc",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T19:40:03.559Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Elevated API error rates",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-03T20:12:38.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/hr8p0qlvkchh",
"started_at": "2026-11-03T19:42:05.000Z",
"status": "resolved",
"updated_at": "2026-11-03T20:21:28.427Z"
},
{
"components": [
{
"created_at": "2015-02-28T01:22:55.169Z",
"description": null,
"group_id": null,
"id": "3n7fzwsysx86",
"name": "Job Queue",
"page_id": "ltljpr68dygn",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T05:46:23.085Z"
}
],
"created_at": "2026-10-29T19:57:28.411Z",
"id": "5mh604lts4g4",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "3n7fzwsysx86",
"name": "Job Queue",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are seeing the job dispatch times are back to normal.",
"created_at": "2026-10-29T21:58:13.445Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-29T21:58:13.445Z",
"id": "ysjrndp8s4p1",
"incident_id": "5mh604lts4g4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-29T21:58:13.445Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3n7fzwsysx86",
"name": "Job Queue",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We’ve applied mitigations and are seeing recovery. We’re continuing to monitor the issue",
"created_at": "2026-10-29T21:41:03.293Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-29T21:41:03.293Z",
"id": "b1bny6vgdf1f",
"incident_id": "5mh604lts4g4",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-29T21:41:03.293Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3n7fzwsysx86",
"name": "Job Queue",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We've spotted that something has gone wrong. We've applied the mitigation and currently monitoring the issue, and will provide an update soon.",
"created_at": "2026-10-29T21:15:57.989Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-29T21:15:57.989Z",
"id": "89hbzfsqctz6",
"incident_id": "5mh604lts4g4",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-29T21:15:57.989Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3n7fzwsysx86",
"name": "Job Queue",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We've identified the cause and are currently applying mitigations",
"created_at": "2026-10-29T20:33:36.210Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-29T20:33:36.210Z",
"id": "901rxp4swlkd",
"incident_id": "5mh604lts4g4",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-29T20:33:36.210Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3n7fzwsysx86",
"name": "Job Queue",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are investigating elevated job dispatch rates that is impacting a subset of customers.",
"created_at": "2026-10-29T19:57:28.479Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-29T19:57:28.479Z",
"id": "472162zjhc3c",
"incident_id": "5mh604lts4g4",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-29T19:57:28.479Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-29T21:15:57.989Z",
"name": "Elevated job dispatch times",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-29T21:58:13.445Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/dygl1qjp1nt5",
"started_at": "2026-10-29T19:57:28.404Z",
"status": "resolved",
"updated_at": "2026-10-29T21:58:13.478Z"
},
{
"components": [
{
"created_at": "2013-08-26T04:27:06.655Z",
"description": null,
"group_id": "0zsswmznk66n",
"id": "r21z46hqn47c",
"name": "REST API",
"page_id": "ltljpr68dygn",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-03T20:02:23.913Z"
},
{
"created_at": "2015-05-11T01:35:06.981Z",
"description": null,
"group_id": null,
"id": "1mpkbbd1kh0h",
"name": "Agent API",
"page_id": "ltljpr68dygn",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-03T20:02:23.943Z"
}
],
"created_at": "2026-10-28T23:17:52.919Z",
"id": "z1x5k3cyrctv",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "1mpkbbd1kh0h",
"name": "Agent API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r21z46hqn47c",
"name": "REST API - REST API",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We identified that the majority of the impact was for a single customer and was caused by a database failover. We are investigating the cause and we will work with the impacted customer.",
"created_at": "2026-10-28T23:49:29.970Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T23:49:29.970Z",
"id": "5tl2fct26ss4",
"incident_id": "z1x5k3cyrctv",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T23:49:29.970Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "r21z46hqn47c",
"name": "REST API - REST API",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We saw increased latency and error rates in REST and GraphQL for a subset of our customers starting around 22:30 UTC until 22:45. The platform has been stable since. We're investigating the root cause",
"created_at": "2026-10-28T23:17:53.008Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T23:17:53.008Z",
"id": "9rzszhp108s9",
"incident_id": "z1x5k3cyrctv",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T23:17:53.008Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-28T23:17:53.008Z",
"name": "Increased latency and error rates",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-28T23:49:29.970Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/n1sw1vccqzg7",
"started_at": "2026-10-28T23:17:52.910Z",
"status": "resolved",
"updated_at": "2026-10-28T23:49:29.998Z"
},
{
"components": [],
"created_at": "2026-10-26T17:07:00.000Z",
"id": "qhkvkzd0sv8g",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "Between 17:07-17:19 UTC, our API endpoints experienced elevated latencies and errors due to issues with a shared service. Our monitoring systems detected these issues, which triggered automated system remediations and resulted in the service self-healing. Although our APIs are once again stable, we are continuing to investigate the cause of this very brief service degradation.",
"created_at": "2026-10-26T18:31:12.556Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-26T17:07:00.000Z",
"id": "5vyfxxxnxblr",
"incident_id": "qhkvkzd0sv8g",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-26T18:31:12.556Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Elevated API latency and error rates",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-26T17:07:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bwscvjwzf7vq",
"started_at": "2026-10-26T18:31:12.503Z",
"status": "resolved",
"updated_at": "2026-10-26T18:31:12.564Z"
},
{
"components": [
{
"created_at": "2013-08-26T04:27:06.655Z",
"description": null,
"group_id": "0zsswmznk66n",
"id": "r21z46hqn47c",
"name": "REST API",
"page_id": "ltljpr68dygn",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-03T20:02:23.913Z"
},
{
"created_at": "2013-08-26T04:27:06.711Z",
"description": null,
"group_id": null,
"id": "r86fqtcmnn65",
"name": "Web",
"page_id": "ltljpr68dygn",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-10T05:07:21.220Z"
},
{
"created_at": "2015-05-11T01:35:06.981Z",
"description": null,
"group_id": null,
"id": "1mpkbbd1kh0h",
"name": "Agent API",
"page_id": "ltljpr68dygn",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-03T20:02:23.943Z"
},
{
"created_at": "2026-05-11T02:54:29.694Z",
"description": null,
"group_id": "0zsswmznk66n",
"id": "fx3sc4nr2w8v",
"name": "Remote MCP Server",
"page_id": "ltljpr68dygn",
"position": 2,
"showcase": true,
"start_date": "2026-03-11T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-06-12T00:16:07.916Z"
},
{
"created_at": "2015-02-28T01:22:55.169Z",
"description": null,
"group_id": null,
"id": "3n7fzwsysx86",
"name": "Job Queue",
"page_id": "ltljpr68dygn",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T05:46:23.085Z"
}
],
"created_at": "2026-10-24T23:40:38.930Z",
"id": "mvh7mjg5ldp7",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "r86fqtcmnn65",
"name": "Web",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "3n7fzwsysx86",
"name": "Job Queue",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "1mpkbbd1kh0h",
"name": "Agent API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "r21z46hqn47c",
"name": "REST API - REST API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "fx3sc4nr2w8v",
"name": "REST API - Remote MCP Server",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Between 23:16 and 23:27 we saw elevated error rates across our platform due to a scheduled database upgrade. That upgrade was not intended to cause any impact, but due to an unforeseen dependency on this database customers experienced error rates averages 2.5% during this period.",
"created_at": "2026-10-24T23:40:38.993Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-24T23:40:38.993Z",
"id": "gyy824z0djgm",
"incident_id": "mvh7mjg5ldp7",
"status": "resolved",
"tweet_id": 2114140041111032000,
"twitter_updated_at": "2026-10-24T23:40:39.431Z",
"updated_at": "2026-10-24T23:40:39.433Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": null,
"name": "Elevated error rates",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-24T23:40:38.993Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/z5ck98pdlnf4",
"started_at": "2026-10-24T23:40:38.920Z",
"status": "resolved",
"updated_at": "2026-10-24T23:40:39.438Z"
},
{
"components": [
{
"created_at": "2024-10-23T08:15:58.991Z",
"description": "Buildkite's hosted compute in the Pipelines product\r\n\r\nhttps://buildkite.com/docs/pipelines/hosted-agents/overview",
"group_id": "56ph4s45xl9s",
"id": "9pwnnl0dp0j5",
"name": "Hosted Agents",
"page_id": "ltljpr68dygn",
"position": 1,
"showcase": true,
"start_date": "2024-10-23T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-22T17:37:17.823Z"
}
],
"created_at": "2026-10-22T16:51:19.292Z",
"id": "chvlk4w3v79d",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "9pwnnl0dp0j5",
"name": "Hosted Agents - Hosted Agents",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Processing of Hosted Agents jobs are no longer delayed.",
"created_at": "2026-10-22T17:37:17.857Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T17:37:17.857Z",
"id": "qv5wqkh530cg",
"incident_id": "chvlk4w3v79d",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T17:37:17.857Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "9pwnnl0dp0j5",
"name": "Hosted Agents - Hosted Agents",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We continue to see improvements and are monitoring processing of the Hosted Agents jobs backlog.",
"created_at": "2026-10-22T17:22:10.441Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T17:22:10.441Z",
"id": "hlhtk34gc62c",
"incident_id": "chvlk4w3v79d",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T17:22:10.441Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "9pwnnl0dp0j5",
"name": "Hosted Agents - Hosted Agents",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We've identified the issue and have applied remediations. We are already beginning to see recovery in Hosted Agents job processing times for affected customers.",
"created_at": "2026-10-22T17:09:34.434Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T17:09:34.434Z",
"id": "6qj173htw0y8",
"incident_id": "chvlk4w3v79d",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T17:09:34.434Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "9pwnnl0dp0j5",
"name": "Hosted Agents - Hosted Agents",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We've received reports that Hosted Agents jobs are delayed in starting and are investigating.",
"created_at": "2026-10-22T16:51:19.397Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-22T16:51:19.397Z",
"id": "s89340884d1y",
"incident_id": "chvlk4w3v79d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T16:51:19.397Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-22T17:22:10.441Z",
"name": "Delays with Hosted Agents jobs",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-22T17:37:17.857Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/ky4wqb4n02gf",
"started_at": "2026-10-22T16:51:19.281Z",
"status": "resolved",
"updated_at": "2026-10-22T17:37:17.880Z"
},
{
"components": [],
"created_at": "2026-10-20T22:13:50.331Z",
"id": "hnnlf9y36vt5",
"impact": "none",
"impact_override": "none",
"incident_updates": [
{
"affected_components": null,
"body": "All services that power Buildkite have been updated to distrust the affected certificates, and updated releases are available for the components we distribute that run in customer infrastructure:\n\nBuildkite agent: v4.7.0 and v3.138.2\nBuildkite CLI: v3.64.2\nbktec: v3.3.1\nelastic-ci-stack-for-aws: v7.8.0\nagent-stack-k8s: v0.52.0\n\nWe recommend updating to these versions to prevent exposure to the affected certificate providers.",
"created_at": "2026-10-21T22:38:39.952Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T22:38:39.952Z",
"id": "w05g97jjd3nr",
"incident_id": "hnnlf9y36vt5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T22:38:39.952Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We’ve finished updating the services that power Buildkite so they no longer trust the affected eMudhra certificates. No action is needed from customers for Buildkite hosted services.\n\nFor components that run in customer infrastructure, we still recommend upgrading to the releases listed in our previous update (Buildkite agent v4.7.0 / v3.138.2, Buildkite CLI v3.64.2, and bktec v3.3.1). We’re continuing to release updates for the other binaries we distribute and will post them here as they ship.",
"created_at": "2026-10-21T05:07:25.171Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T05:07:25.171Z",
"id": "jk4mbpyfvwqz",
"incident_id": "hnnlf9y36vt5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T05:07:25.171Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We're continuing to monitor the situation, and are making releases to many of our components that run in customer infrastructure. At time of writing, we've released versions that distrust the affected eMudhra certificates for:\n- Buildkite agent: https://github.com/buildkite/agent:\n - v4.7.0: https://github.com/buildkite/agent/releases/tag/v4.7.0\n - v3.138.2: https://github.com/buildkite/agent/releases/tag/v3.138.2\n- Buildkite CLI: https://github.com/buildkite/cli/releases/tag/v3.64.2\n- bktec: https://github.com/buildkite/test-engine-client/releases/tag/v3.3.1\n\nWe recommend updating to these versions to prevent exposure to the affected certificate providers. We will continue to issue updates for the other binaries we distribute, and are updating any internal services that are affected.",
"created_at": "2026-10-20T23:25:11.425Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T23:25:11.000Z",
"id": "krfb7dwqvg4q",
"incident_id": "hnnlf9y36vt5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T23:30:06.567Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The private keys for eight certificates issued by eMudhra, an external certificate authority, have been compromised. Software that still trusts these certificates may accept fraudulent HTTPS certificates.\n\nWe’re updating the services that power Buildkite and preparing new CLI and Agent releases, which we expect to ship shortly. We’re also tracking fixes to the libraries and trust stores we depend on and will apply them as they become available.\n\nWe’ll update this page with the new versions and any action customers need to take. Mozilla has published background on the compromise.",
"created_at": "2026-10-20T22:13:50.382Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-20T22:13:50.382Z",
"id": "923z4dkbywcf",
"incident_id": "hnnlf9y36vt5",
"status": "investigating",
"tweet_id": 2112668645096849700,
"twitter_updated_at": "2026-10-20T22:13:51.294Z",
"updated_at": "2026-10-20T22:13:51.297Z",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-10-20T23:25:11.000Z",
"name": "Buildkite response to the eMudhra certificate compromise",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-21T22:38:39.952Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/yn3cmf9rh4rd",
"started_at": "2026-10-20T22:13:50.325Z",
"status": "resolved",
"updated_at": "2026-10-21T22:38:39.986Z"
},
{
"components": [
{
"created_at": "2015-05-11T01:32:50.904Z",
"description": null,
"group_id": "tplm771kdwfs",
"id": "m3fr6snn2496",
"name": "GitHub Commit Status Notifications",
"page_id": "ltljpr68dygn",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-20T21:05:24.219Z"
},
{
"created_at": "2015-05-11T01:33:31.806Z",
"description": null,
"group_id": "tplm771kdwfs",
"id": "rj879wvx0lyd",
"name": "Email Notifications",
"page_id": "ltljpr68dygn",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-20T21:05:24.250Z"
},
{
"created_at": "2015-05-11T01:33:51.204Z",
"description": null,
"group_id": "tplm771kdwfs",
"id": "r0jrhpsydz4w",
"name": "Slack Notifications",
"page_id": "ltljpr68dygn",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-20T21:05:24.279Z"
},
{
"created_at": "2018-05-30T21:53:19.571Z",
"description": null,
"group_id": "tplm771kdwfs",
"id": "mk54vxw70994",
"name": "Webhook Notifications",
"page_id": "ltljpr68dygn",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-20T21:05:24.308Z"
}
],
"created_at": "2026-10-20T19:57:06.939Z",
"id": "665n4gh9msq5",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "m3fr6snn2496",
"name": "Notifications - GitHub Commit Status Notifications",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "rj879wvx0lyd",
"name": "Notifications - Email Notifications",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "r0jrhpsydz4w",
"name": "Notifications - Slack Notifications",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "mk54vxw70994",
"name": "Notifications - Webhook Notifications",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Adding additional capacity has resolved the notifications latency.",
"created_at": "2026-10-20T21:05:24.349Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T21:05:24.349Z",
"id": "mhlp3wrwtzvp",
"incident_id": "665n4gh9msq5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T21:05:24.349Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "m3fr6snn2496",
"name": "Notifications - GitHub Commit Status Notifications",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "rj879wvx0lyd",
"name": "Notifications - Email Notifications",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "r0jrhpsydz4w",
"name": "Notifications - Slack Notifications",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "mk54vxw70994",
"name": "Notifications - Webhook Notifications",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We've identified the issue, and added some additional capacity. Notification latency has returned to normal levels. We are monitoring for stability.",
"created_at": "2026-10-20T20:15:08.311Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T20:15:08.311Z",
"id": "l0yw8ld9674y",
"incident_id": "665n4gh9msq5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T20:15:08.311Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "m3fr6snn2496",
"name": "Notifications - GitHub Commit Status Notifications",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "rj879wvx0lyd",
"name": "Notifications - Email Notifications",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "r0jrhpsydz4w",
"name": "Notifications - Slack Notifications",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "mk54vxw70994",
"name": "Notifications - Webhook Notifications",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating delays to build and job notifications for a subset of customers.",
"created_at": "2026-10-20T19:57:07.181Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T19:57:07.181Z",
"id": "tqxzcr47zqtm",
"incident_id": "665n4gh9msq5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T19:57:07.181Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-20T20:15:08.311Z",
"name": "Delayed notifications",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-20T21:05:24.349Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/wt38syb0r744",
"started_at": "2026-10-20T19:57:06.924Z",
"status": "resolved",
"updated_at": "2026-10-20T21:05:24.483Z"
},
{
"components": [],
"created_at": "2026-10-19T15:45:00.000Z",
"id": "rmr24xv96msz",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": null,
"body": "We received reports from a few customers that Hosted Agent jobs were delayed in starting. We investigated and found this related to a GitHub incident with webhooks that caused throttling in Hosted Agent job creation once webhooks were once again being ingested. Any affected Hosted Agents customers would have seen delays in instance creation between 15:45-17:12UTC.",
"created_at": "2026-10-19T17:39:44.928Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-19T15:45:00.000Z",
"id": "79k75qhj8h52",
"incident_id": "rmr24xv96msz",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T17:44:15.017Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Delayed Hosted Agents instance creation",
"page_id": "ltljpr68dygn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-19T15:45:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/h3zb421lwvm8",
"started_at": "2026-10-19T15:45:00.000Z",
"status": "resolved",
"updated_at": "2026-10-19T17:44:15.022Z"
}
],
"page": {
"id": "ltljpr68dygn",
"name": "Buildkite",
"url": "https://www.buildkitestatus.com"
},
"status": {
"description": "Partially Degraded Service",
"indicator": "minor"
}
}