Snapshot 15051
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"pages": [
{
"end_time": "2026-09-30T23:59:59Z",
"months": [
{
"incidents": [
{
"code": "t1h2d1f4y6qx",
"impact": "major",
"message": "Services are fully recovered. Webhook ingestion was affected between 19:09 UTC - 19:32 UTC.",
"name": "Issues with webhook ingestion, GraphQL queries",
"timestamp": "Sep \u003cvar data-var='date'\u003e22\u003c/var\u003e, \u003cvar data-var='time'\u003e20:07\u003c/var\u003e - \u003cvar data-var='time'\u003e20:50\u003c/var\u003e UTC"
},
{
"code": "vr7dsq0p8pq0",
"impact": "none",
"message": "We did not experience any customer impact during this AWS us-east-1 outage.",
"name": "AWS us-east-1 outage",
"timestamp": "Sep \u003cvar data-var='date'\u003e21\u003c/var\u003e, \u003cvar data-var='time'\u003e23:47\u003c/var\u003e - Sep \u003cvar data-var='date'\u003e22\u003c/var\u003e, \u003cvar data-var='time'\u003e00:32\u003c/var\u003e UTC"
},
{
"code": "r2w4j1wyj0ss",
"impact": "minor",
"message": "This incident has been resolved.",
"name": "Increased latency and error rates",
"timestamp": "Sep \u003cvar data-var='date'\u003e21\u003c/var\u003e, \u003cvar data-var='time'\u003e22:35\u003c/var\u003e - \u003cvar data-var='time'\u003e23:46\u003c/var\u003e UTC"
},
{
"code": "7nc7xd7zkxjx",
"impact": "major",
"message": "We have seen full recovery for customers since 20:28 UTC. \nWe experienced an autoscaling feedback loop which increased the number of connections to our redis cluster above its ability to respond. This had widespread impact for all of our customers with Web UI, Agent API, REST API and job queue impact between 18:43-19:21 UTC, and again between 20:02-20:28 UTC. A full post incident review will be available later this week.",
"name": "Buildkite service disruption",
"timestamp": "Sep \u003cvar data-var='date'\u003e9\u003c/var\u003e, \u003cvar data-var='time'\u003e19:07\u003c/var\u003e - Sep \u003cvar data-var='date'\u003e10\u003c/var\u003e, \u003cvar data-var='time'\u003e01:54\u003c/var\u003e UTC"
},
{
"code": "0qmndj65cf0r",
"impact": "minor",
"message": "Jobs running on Hosted Agents are now being dispatched promptly.",
"name": "Hosted Agents - Increased latency and error rates",
"timestamp": "Sep \u003cvar data-var='date'\u003e4\u003c/var\u003e, \u003cvar data-var='time'\u003e01:42\u003c/var\u003e - \u003cvar data-var='time'\u003e02:39\u003c/var\u003e UTC"
},
{
"code": "v59rky62f2h4",
"impact": "minor",
"message": "Jobs running on Hosted Agents are now being dispatched promptly.",
"name": "Scheduling delay for hosted agents",
"timestamp": "Sep \u003cvar data-var='date'\u003e3\u003c/var\u003e, \u003cvar data-var='time'\u003e06:00\u003c/var\u003e - \u003cvar data-var='time'\u003e07:06\u003c/var\u003e UTC"
},
{
"code": "zf4v5577x0gf",
"impact": "major",
"message": "Job dispatch for Hosted Agents is no longer experiencing delays.",
"name": "Delays in Hosted Agents job dispatch",
"timestamp": "Sep \u003cvar data-var='date'\u003e2\u003c/var\u003e, \u003cvar data-var='time'\u003e14:45\u003c/var\u003e - \u003cvar data-var='time'\u003e15:51\u003c/var\u003e UTC"
},
{
"code": "9hd5d7qxhtq4",
"impact": "minor",
"message": "Our outbound email queue has finished processing all previously failed deliveries. Email notifications are processing successfully.",
"name": "Delivery issues with email notifications",
"timestamp": "Sep \u003cvar data-var='date'\u003e1\u003c/var\u003e, \u003cvar data-var='time'\u003e15:32\u003c/var\u003e - \u003cvar data-var='time'\u003e16:52\u003c/var\u003e UTC"
}
],
"name": "September",
"year": 2026
},
{
"incidents": [
{
"code": "rg1tyqgzbg3w",
"impact": "minor",
"message": "Processing of uploaded test execution data has caught up and is back to normal.",
"name": "Test Engine ingestion delayed, catching up",
"timestamp": "Aug \u003cvar data-var='date'\u003e27\u003c/var\u003e, \u003cvar data-var='time'\u003e10:56\u003c/var\u003e - \u003cvar data-var='time'\u003e11:47\u003c/var\u003e UTC"
},
{
"code": "hclyxydrwn9p",
"impact": "minor",
"message": "Mitigation strategies worked as expected and now agent registration is back to normal. All systems are stable now.",
"name": "Issues identified with buildkite-agent registration",
"timestamp": "Aug \u003cvar data-var='date'\u003e26\u003c/var\u003e, \u003cvar data-var='time'\u003e01:42\u003c/var\u003e - \u003cvar data-var='time'\u003e03:13\u003c/var\u003e UTC"
},
{
"code": "tm6746k61p73",
"impact": "major",
"message": "We're considering this incident resolved.",
"name": "Buildkite service disruption",
"timestamp": "Aug \u003cvar data-var='date'\u003e25\u003c/var\u003e, \u003cvar data-var='time'\u003e22:55\u003c/var\u003e - Aug \u003cvar data-var='date'\u003e26\u003c/var\u003e, \u003cvar data-var='time'\u003e00:18\u003c/var\u003e UTC"
},
{
"code": "l48g42jpv2sm",
"impact": "minor",
"message": "This incident has been resolved.",
"name": "Unexpected agent disconnection for some customers",
"timestamp": "Aug \u003cvar data-var='date'\u003e25\u003c/var\u003e, \u003cvar data-var='time'\u003e06:41\u003c/var\u003e - \u003cvar data-var='time'\u003e08:16\u003c/var\u003e UTC"
},
{
"code": "09lgjj74c1b7",
"impact": "minor",
"message": "This incident has been resolved.",
"name": "Buildkite service disruption",
"timestamp": "Aug \u003cvar data-var='date'\u003e17\u003c/var\u003e, \u003cvar data-var='time'\u003e00:46\u003c/var\u003e - \u003cvar data-var='time'\u003e02:43\u003c/var\u003e UTC"
},
{
"code": "7ccvl0yw7wrg",
"impact": "minor",
"message": "We think the impact from the issue is over.",
"name": "Slow web UI",
"timestamp": "Aug \u003cvar data-var='date'\u003e13\u003c/var\u003e, \u003cvar data-var='time'\u003e15:41\u003c/var\u003e - \u003cvar data-var='time'\u003e16:40\u003c/var\u003e UTC"
},
{
"code": "rcqyh8p883db",
"impact": "minor",
"message": "For the last week we have been internally testing our agent traffic running on our new EKS infrastructure successfully. \nAt 23:01 11 August 2026 UTC we started routing \u0026lt;1% of agent traffic for a small subset of our customer to this EKS infrastructure. Within that subset of customers, any that were using an agent version \u0026lt; 3.97.0 saw an error \"failed to decode JSON response: invalid character '\u0026lt;' looking for beginning of value\". These older agents crashed once they received that error, which for some customers triggered repeated agent cycling. \nWe reverted the routing to EKS infrastructure at 03:25 UTC after being alerted to the issue.",
"name": "Buildkite service disruption",
"timestamp": "Aug \u003cvar data-var='date'\u003e12\u003c/var\u003e, \u003cvar data-var='time'\u003e02:44\u003c/var\u003e - \u003cvar data-var='time'\u003e04:08\u003c/var\u003e UTC"
},
{
"code": "bhnjck3s6vs9",
"impact": "minor",
"message": "This incident has been resolved.",
"name": "Slow web UI",
"timestamp": "Aug \u003cvar data-var='date'\u003e10\u003c/var\u003e, \u003cvar data-var='time'\u003e23:00\u003c/var\u003e - Aug \u003cvar data-var='date'\u003e11\u003c/var\u003e, \u003cvar data-var='time'\u003e02:04\u003c/var\u003e UTC"
},
{
"code": "zpy7rm7mdhj9",
"impact": "minor",
"message": "Between 22:03 - 22:16 UTC customers experienced delayed job dispatch, build processing and notifications. Additionally there were errors receiving webhooks during this period.",
"name": "Pipelines latency and webhook errors",
"timestamp": "Aug \u003cvar data-var='date'\u003e5\u003c/var\u003e, \u003cvar data-var='time'\u003e22:00\u003c/var\u003e - \u003cvar data-var='time'\u003e22:00\u003c/var\u003e UTC"
},
{
"code": "vtz6bhz26gj2",
"impact": "major",
"message": "We've confirmed all impact from this incident has been resolved.",
"name": "Latency on Pipelines",
"timestamp": "Aug \u003cvar data-var='date'\u003e3\u003c/var\u003e, \u003cvar data-var='time'\u003e05:11\u003c/var\u003e - \u003cvar data-var='time'\u003e05:36\u003c/var\u003e UTC"
}
],
"name": "August",
"year": 2026
},
{
"incidents": [
{
"code": "p94fzx3dbvln",
"impact": "minor",
"message": "Notification latency for the affected shard is now under 5s.",
"name": "Delayed notifications",
"timestamp": "Jul \u003cvar data-var='date'\u003e28\u003c/var\u003e, \u003cvar data-var='time'\u003e20:38\u003c/var\u003e - \u003cvar data-var='time'\u003e20:57\u003c/var\u003e UTC"
}
],
"name": "July",
"year": 2026
}
],
"start_time": "2026-07-01T00:00:00Z",
"time_zone": "UTC"
},
{
"end_time": "2026-06-30T23:59:59Z",
"months": [
{
"incidents": [
{
"code": "wdkm31c8ym4h",
"impact": "major",
"message": "We have seen a full recovery of services.",
"name": "Increased latency and error rates",
"timestamp": "Jun \u003cvar data-var='date'\u003e18\u003c/var\u003e, \u003cvar data-var='time'\u003e08:05\u003c/var\u003e - \u003cvar data-var='time'\u003e09:43\u003c/var\u003e UTC"
},
{
"code": "6tr7whjn0vl8",
"impact": "major",
"message": "The mitigation applied before the last update had the intended effect, and we have seen recovery in REST API latency.",
"name": "Increased latency on REST and GraphQL APIs",
"timestamp": "Jun \u003cvar data-var='date'\u003e11\u003c/var\u003e, \u003cvar data-var='time'\u003e23:18\u003c/var\u003e - Jun \u003cvar data-var='date'\u003e12\u003c/var\u003e, \u003cvar data-var='time'\u003e00:16\u003c/var\u003e UTC"
},
{
"code": "pfr7xmgj25l7",
"impact": "minor",
"message": "Between 00:05 - 00:34 UTC, a subset of customers experienced increased latency and timeout errors on the Agent API. This impacts job assignment. At peak impact, we saw an error rate of 1.3% of requests and job acceptance latency up to 53s.",
"name": "Increased latency and error rates for Agent API",
"timestamp": "Jun \u003cvar data-var='date'\u003e11\u003c/var\u003e, \u003cvar data-var='time'\u003e00:32\u003c/var\u003e - \u003cvar data-var='time'\u003e00:51\u003c/var\u003e UTC"
}
],
"name": "June",
"year": 2026
},
{
"incidents": [
{
"code": "497l2tl5987g",
"impact": "none",
"message": "We have received reports email deliveries have not been working, affecting signup and invite emails as well as build notification emails.\nThis issue has now been resolved.",
"name": "Email deliveries are delayed",
"timestamp": "May \u003cvar data-var='date'\u003e30\u003c/var\u003e, \u003cvar data-var='time'\u003e00:30\u003c/var\u003e - \u003cvar data-var='time'\u003e00:30\u003c/var\u003e UTC"
},
{
"code": "8zy2b81jz4r9",
"impact": "major",
"message": "This incident has been resolved.",
"name": "Delayed notifications",
"timestamp": "May \u003cvar data-var='date'\u003e28\u003c/var\u003e, \u003cvar data-var='time'\u003e20:20\u003c/var\u003e - \u003cvar data-var='time'\u003e21:18\u003c/var\u003e UTC"
},
{
"code": "cwsbddzngdlq",
"impact": "none",
"message": "We think the impact from the issue is over.",
"name": "Increased latency and error rates",
"timestamp": "May \u003cvar data-var='date'\u003e26\u003c/var\u003e, \u003cvar data-var='time'\u003e09:56\u003c/var\u003e - \u003cvar data-var='time'\u003e10:38\u003c/var\u003e UTC"
},
{
"code": "3vngjcnbkycl",
"impact": "major",
"message": "The incident is resolved",
"name": "Delayed notifications",
"timestamp": "May \u003cvar data-var='date'\u003e20\u003c/var\u003e, \u003cvar data-var='time'\u003e16:40\u003c/var\u003e - \u003cvar data-var='time'\u003e17:39\u003c/var\u003e UTC"
},
{
"code": "gq6z38gd2tzc",
"impact": "minor",
"message": "Processing of the backlog is complete.",
"name": "Delayed Test Engine ingestion processing",
"timestamp": "May \u003cvar data-var='date'\u003e15\u003c/var\u003e, \u003cvar data-var='time'\u003e06:51\u003c/var\u003e - \u003cvar data-var='time'\u003e07:35\u003c/var\u003e UTC"
},
{
"code": "xrsf324tzwgh",
"impact": "minor",
"message": "Additional capacity was added to our redis caches. This triggered a failover between UTC 15:10 - 15:14 and there was a spike of errors on the REST and GraphQL APIs. Customers would have seen some errors in the Buildkite UI during this period as well. \n\nWe have been monitoring the situation since then and things have returned to baseline.",
"name": "Error rates increasing",
"timestamp": "May \u003cvar data-var='date'\u003e13\u003c/var\u003e, \u003cvar data-var='time'\u003e15:14\u003c/var\u003e - \u003cvar data-var='time'\u003e15:34\u003c/var\u003e UTC"
},
{
"code": "xmss20cnm094",
"impact": "minor",
"message": "The fix was successful and the backlog has now been cleared.",
"name": "Delayed Test Engine ingestion processing",
"timestamp": "May \u003cvar data-var='date'\u003e12\u003c/var\u003e, \u003cvar data-var='time'\u003e12:59\u003c/var\u003e - \u003cvar data-var='time'\u003e16:09\u003c/var\u003e UTC"
},
{
"code": "19bspyky0px4",
"impact": "minor",
"message": "The upstream AWS incident in us-east-1 has been resolved by AWS, and all Buildkite services are operating normally. No further customer impact is expected. We appreciate your patience during this incident.",
"name": "AWS us-east-1 single availability zone outage",
"timestamp": "May \u003cvar data-var='date'\u003e8\u003c/var\u003e, \u003cvar data-var='time'\u003e01:12\u003c/var\u003e - May \u003cvar data-var='date'\u003e9\u003c/var\u003e, \u003cvar data-var='time'\u003e04:34\u003c/var\u003e UTC"
},
{
"code": "pxdy9k3x8l7k",
"impact": "minor",
"message": "The delayed backlog is now cleared and Test Engine ingestion is operating normally.",
"name": "Delayed Test Engine ingestion processing",
"timestamp": "May \u003cvar data-var='date'\u003e8\u003c/var\u003e, \u003cvar data-var='time'\u003e21:10\u003c/var\u003e - \u003cvar data-var='time'\u003e23:42\u003c/var\u003e UTC"
},
{
"code": "0kwwp7j1vqjf",
"impact": "minor",
"message": "The Test Engine backlog is now cleared and operating normally.",
"name": "Delayed Test Engine ingestion processing",
"timestamp": "May \u003cvar data-var='date'\u003e8\u003c/var\u003e, \u003cvar data-var='time'\u003e12:20\u003c/var\u003e - \u003cvar data-var='time'\u003e13:06\u003c/var\u003e UTC"
},
{
"code": "98sd15lxsxr3",
"impact": "major",
"message": "All customer workloads have now recovered.",
"name": "Delays in job dispatch, webhook processing, and outbound webhooks",
"timestamp": "May \u003cvar data-var='date'\u003e7\u003c/var\u003e, \u003cvar data-var='time'\u003e22:45\u003c/var\u003e - May \u003cvar data-var='date'\u003e8\u003c/var\u003e, \u003cvar data-var='time'\u003e00:13\u003c/var\u003e UTC"
},
{
"code": "j8qlchvxr102",
"impact": "major",
"message": "We've reverted a change that caused stale environment variables provided to acquire job used in Hosted Agents, agent-stack-k8s and other agent implementations using acquire job.",
"name": "Jobs not starting on hosted agents and agent-stack-k8s",
"timestamp": "May \u003cvar data-var='date'\u003e7\u003c/var\u003e, \u003cvar data-var='time'\u003e08:48\u003c/var\u003e - \u003cvar data-var='time'\u003e09:37\u003c/var\u003e UTC"
},
{
"code": "c52rftbq9qtm",
"impact": "minor",
"message": "Processing of test execution ingestion data has successfully caught up.",
"name": "Test Engine: Delayed processing of test result ingestion",
"timestamp": "May \u003cvar data-var='date'\u003e6\u003c/var\u003e, \u003cvar data-var='time'\u003e03:57\u003c/var\u003e - \u003cvar data-var='time'\u003e05:26\u003c/var\u003e UTC"
},
{
"code": "kzk9jkd5z7py",
"impact": "none",
"message": "Notifications continue to be delivered without delay for the previously affected subset of customers. This incident is resolved.",
"name": "Delayed notifications",
"timestamp": "May \u003cvar data-var='date'\u003e4\u003c/var\u003e, \u003cvar data-var='time'\u003e17:02\u003c/var\u003e - \u003cvar data-var='time'\u003e19:36\u003c/var\u003e UTC"
},
{
"code": "ygzyngc0zq98",
"impact": "minor",
"message": "An increase in requests has lead to the API service being temporarily saturated. We have updated rate limits to ensure this doesn't re-occur and will add further resources if necessary",
"name": "Increased latency and error rates",
"timestamp": "May \u003cvar data-var='date'\u003e4\u003c/var\u003e, \u003cvar data-var='time'\u003e06:02\u003c/var\u003e - \u003cvar data-var='time'\u003e06:30\u003c/var\u003e UTC"
},
{
"code": "8nd3cm8kqlnn",
"impact": "major",
"message": "This incident has been resolved.",
"name": "Buildkite service disruption",
"timestamp": "May \u003cvar data-var='date'\u003e1\u003c/var\u003e, \u003cvar data-var='time'\u003e02:41\u003c/var\u003e - \u003cvar data-var='time'\u003e03:30\u003c/var\u003e UTC"
}
],
"name": "May",
"year": 2026
},
{
"incidents": [
{
"code": "wl8m1fgbvn6f",
"impact": "minor",
"message": "We have confirmed that latency and error rates have returned to normal for impacted customers.",
"name": "Increased latency and error rates",
"timestamp": "Apr \u003cvar data-var='date'\u003e29\u003c/var\u003e, \u003cvar data-var='time'\u003e17:43\u003c/var\u003e - \u003cvar data-var='time'\u003e18:48\u003c/var\u003e UTC"
},
{
"code": "klqc4f7yfc2y",
"impact": "minor",
"message": "Previously elevated loads with Hosted Agents dispatch have fully recovered.",
"name": "Increased dispatch latency and error rates",
"timestamp": "Apr \u003cvar data-var='date'\u003e28\u003c/var\u003e, \u003cvar data-var='time'\u003e18:00\u003c/var\u003e - \u003cvar data-var='time'\u003e19:16\u003c/var\u003e UTC"
},
{
"code": "n8dph8ll0ddb",
"impact": "minor",
"message": "The issue is resolved.",
"name": "Auth failures with remote MCP server",
"timestamp": "Apr \u003cvar data-var='date'\u003e22\u003c/var\u003e, \u003cvar data-var='time'\u003e21:19\u003c/var\u003e - \u003cvar data-var='time'\u003e22:59\u003c/var\u003e UTC"
},
{
"code": "6jv363dscqdc",
"impact": "minor",
"message": "The backlog has been cleared and all systems are fully operational. Thank you for your patience.",
"name": "Delayed processing of test execution",
"timestamp": "Apr \u003cvar data-var='date'\u003e22\u003c/var\u003e, \u003cvar data-var='time'\u003e02:32\u003c/var\u003e - \u003cvar data-var='time'\u003e05:07\u003c/var\u003e UTC"
},
{
"code": "j7ngpbpgnw66",
"impact": "major",
"message": "We experienced an issue that caused a brief increase in errors for the Agent API. This also impacted latency for notifications. All notifications were stored in a queue and processed. Latency is now back to normal.",
"name": "Degraded performance and increased error rates",
"timestamp": "Apr \u003cvar data-var='date'\u003e8\u003c/var\u003e, \u003cvar data-var='time'\u003e22:26\u003c/var\u003e - \u003cvar data-var='time'\u003e23:12\u003c/var\u003e UTC"
}
],
"name": "April",
"year": 2026
}
],
"start_time": "2026-04-01T00:00:00Z",
"time_zone": "UTC"
},
{
"end_time": "2026-03-31T23:59:59Z",
"months": [
{
"incidents": [
{
"code": "mbftp72pn0hq",
"impact": "minor",
"message": "We have deployed the fix and we have confirmed customer builds are working. If you encounter any further issues please contact support.",
"name": "Hosted Agents jobs immediately cancelled",
"timestamp": "Mar \u003cvar data-var='date'\u003e31\u003c/var\u003e, \u003cvar data-var='time'\u003e07:51\u003c/var\u003e - \u003cvar data-var='time'\u003e08:34\u003c/var\u003e UTC"
},
{
"code": "41m4sncc2nz1",
"impact": "minor",
"message": "The incident is now resolved. We are no longer seeing errors when viewing pipelines.",
"name": "504 errors viewing builds",
"timestamp": "Mar \u003cvar data-var='date'\u003e27\u003c/var\u003e, \u003cvar data-var='time'\u003e07:02\u003c/var\u003e - \u003cvar data-var='time'\u003e08:53\u003c/var\u003e UTC"
},
{
"code": "g6wg153ksf8v",
"impact": "minor",
"message": "This incident is now resolved. We are no longer seeing further networking issues with Hosted Agents, which affected delays in creating them for Jobs, resolving external traffic and interactions with Cache - affecting Git Mirror Cloning.",
"name": "Increased Delays with Hosted Agents",
"timestamp": "Mar \u003cvar data-var='date'\u003e25\u003c/var\u003e, \u003cvar data-var='time'\u003e14:26\u003c/var\u003e - \u003cvar data-var='time'\u003e15:50\u003c/var\u003e UTC"
},
{
"code": "xq4wwj2h5f4r",
"impact": "major",
"message": "This incident has been resolved.",
"name": "Increased queue times on hosted agents",
"timestamp": "Mar \u003cvar data-var='date'\u003e11\u003c/var\u003e, \u003cvar data-var='time'\u003e19:50\u003c/var\u003e - \u003cvar data-var='time'\u003e21:14\u003c/var\u003e UTC"
},
{
"code": "gl6sy293jb6d",
"impact": "major",
"message": "Our mitigations have resolved the elevated latency and likelihood of suboptimal fallback test plans. We have also identified and fixed a blind-spot in our automated alerting, which was previously unable to detect this scenario as an issue. Work continues this week to resolve the underlying performance issue by restructuring how the relevant data is ingested and accessed.",
"name": "Increased error rates from Test Plan API",
"timestamp": "Mar \u003cvar data-var='date'\u003e10\u003c/var\u003e, \u003cvar data-var='time'\u003e01:21\u003c/var\u003e - \u003cvar data-var='time'\u003e09:34\u003c/var\u003e UTC"
},
{
"code": "jtq1m5z7yj7j",
"impact": "minor",
"message": "Processing of test execution ingestion data has successfully caught up.",
"name": "Elevated ingestion latency for Test Engine",
"timestamp": "Mar \u003cvar data-var='date'\u003e7\u003c/var\u003e, \u003cvar data-var='time'\u003e00:21\u003c/var\u003e - \u003cvar data-var='time'\u003e01:05\u003c/var\u003e UTC"
},
{
"code": "3ghj6s56kybc",
"impact": "minor",
"message": "With artifact upload latency continuing to be stable, we are resolving this incident.",
"name": "Slow artifact uploads",
"timestamp": "Mar \u003cvar data-var='date'\u003e5\u003c/var\u003e, \u003cvar data-var='time'\u003e22:14\u003c/var\u003e - Mar \u003cvar data-var='date'\u003e6\u003c/var\u003e, \u003cvar data-var='time'\u003e10:23\u003c/var\u003e UTC"
},
{
"code": "z5vhpsjg402w",
"impact": "minor",
"message": "Buildkite Hosted Agents experienced degraded start-time performance due to a network partition issue in the Hosted Agents control plane. A small subset of customers may have seen delayed job starts during 04:40-04:50 UTC and 05:06-05:16 UTC. The issue has been resolved and we are monitoring to confirm stability.",
"name": "Hosted Agents: Job start latency for a small subset of customers",
"timestamp": "Mar \u003cvar data-var='date'\u003e6\u003c/var\u003e, \u003cvar data-var='time'\u003e04:30\u003c/var\u003e - \u003cvar data-var='time'\u003e04:30\u003c/var\u003e UTC"
},
{
"code": "3jk7hr3qj0cy",
"impact": "minor",
"message": "Processing of test execution ingestion data has successfully caught up.",
"name": "Elevated ingestion latency for Test Engine",
"timestamp": "Mar \u003cvar data-var='date'\u003e4\u003c/var\u003e, \u003cvar data-var='time'\u003e03:59\u003c/var\u003e - \u003cvar data-var='time'\u003e12:35\u003c/var\u003e UTC"
},
{
"code": "pc0sgm2n3d4d",
"impact": "major",
"message": "We have completed the provisioning of additional capacity mentioned in our last update, and error rates and response times have returned to normal.\n\nThis incident is now resolved.",
"name": "Latency issues",
"timestamp": "Mar \u003cvar data-var='date'\u003e3\u003c/var\u003e, \u003cvar data-var='time'\u003e21:51\u003c/var\u003e - Mar \u003cvar data-var='date'\u003e4\u003c/var\u003e, \u003cvar data-var='time'\u003e05:24\u003c/var\u003e UTC"
}
],
"name": "March",
"year": 2026
},
{
"incidents": [
{
"code": "xmc8mcv12qvj",
"impact": "major",
"message": "We have seen a full recovery of service, and have a good understanding of the underlying cause. We will publish a post-incident review next week.",
"name": "Increased dispatch latency",
"timestamp": "Feb \u003cvar data-var='date'\u003e26\u003c/var\u003e, \u003cvar data-var='time'\u003e19:10\u003c/var\u003e - Feb \u003cvar data-var='date'\u003e27\u003c/var\u003e, \u003cvar data-var='time'\u003e02:30\u003c/var\u003e UTC"
},
{
"code": "482frnr5h7n1",
"impact": "minor",
"message": "Response times have returned to normal. This incident is now resolved.",
"name": "Increased latency for secrets endpoints for some customers",
"timestamp": "Feb \u003cvar data-var='date'\u003e26\u003c/var\u003e, \u003cvar data-var='time'\u003e00:43\u003c/var\u003e - \u003cvar data-var='time'\u003e02:44\u003c/var\u003e UTC"
},
{
"code": "qt2qnk2y25z9",
"impact": "minor",
"message": "The cause of pipeline upload failures and dispatch latency that was impacting some of our customers has been resolved.",
"name": "Increased dispatch latency",
"timestamp": "Feb \u003cvar data-var='date'\u003e17\u003c/var\u003e, \u003cvar data-var='time'\u003e20:09\u003c/var\u003e - \u003cvar data-var='time'\u003e21:05\u003c/var\u003e UTC"
},
{
"code": "mtzljbp4r9w5",
"impact": "minor",
"message": "Between 2026-02-12 01:33 and 04:49 UTC, an elevated error rate in job dispatch was observed. This affected a subset of jobs. Affected jobs were unable to be dispatched during the disruption window, but were eventually dispatched due to automatic retries. A subset of pipeline step uploads were also affected; the agent would check for successful step upload over a 5-minute period before timing out.\n\nThis occurred due to a bug in a configuration change that resulted in some Sidekiq instances not receiving all required database connection configuration. The configuration change was subsequently reverted, with further investigation underway.",
"name": "Some jobs not dispatching",
"timestamp": "Feb \u003cvar data-var='date'\u003e12\u003c/var\u003e, \u003cvar data-var='time'\u003e01:33\u003c/var\u003e - \u003cvar data-var='time'\u003e01:33\u003c/var\u003e UTC"
},
{
"code": "9c6kqdz0yh18",
"impact": "minor",
"message": "This incident has been resolved.",
"name": "Delays with GitHub Webhooks",
"timestamp": "Feb \u003cvar data-var='date'\u003e3\u003c/var\u003e, \u003cvar data-var='time'\u003e15:36\u003c/var\u003e - \u003cvar data-var='time'\u003e18:44\u003c/var\u003e UTC"
}
],
"name": "February",
"year": 2026
},
{
"incidents": [
{
"code": "bjbgtmq0lr7h",
"impact": "major",
"message": "We believe this issue is now resolved and all Test Engine systems are functioning. Ingestion of test data was delayed during this incident but we believe there has been no data loss.\n\nCustomers using Buildkite Test Engine Client (bktec) to run tests will have experienced slower builds during the incident due to bktec using a sub-optimal test planning strategy while Test Engine APIs were unavailable.",
"name": "Test Engine services unavailable",
"timestamp": "Jan \u003cvar data-var='date'\u003e31\u003c/var\u003e, \u003cvar data-var='time'\u003e10:06\u003c/var\u003e - \u003cvar data-var='time'\u003e11:35\u003c/var\u003e UTC"
},
{
"code": "m69w09ycxsmx",
"impact": "major",
"message": "A maintenance process incorrectly marked some connected agents as lost between January 23–25 UTC. Agents on Elastic CI Stack recovered automatically; others may need a service restart. We've deployed a fix to prevent this from recurring.",
"name": "Agent unexpectedly disconnected",
"timestamp": "Jan \u003cvar data-var='date'\u003e25\u003c/var\u003e, \u003cvar data-var='time'\u003e08:00\u003c/var\u003e - \u003cvar data-var='time'\u003e08:00\u003c/var\u003e UTC"
},
{
"code": "6qp681ppdyqj",
"impact": "minor",
"message": "This incident is now resolved.",
"name": "Increased dispatch latency",
"timestamp": "Jan \u003cvar data-var='date'\u003e16\u003c/var\u003e, \u003cvar data-var='time'\u003e03:42\u003c/var\u003e - \u003cvar data-var='time'\u003e03:46\u003c/var\u003e UTC"
},
{
"code": "zxw15z5smgmd",
"impact": "maintenance",
"message": "The scheduled maintenance has been completed.",
"name": "[Scheduled] Database maintenance for subset of customers",
"timestamp": "Jan \u003cvar data-var='date'\u003e11\u003c/var\u003e, \u003cvar data-var='time'\u003e06:00\u003c/var\u003e - \u003cvar data-var='time'\u003e10:00\u003c/var\u003e UTC"
},
{
"code": "v1wfrvp0krln",
"impact": "minor",
"message": "This incident has been resolved.",
"name": "Slow Loading of Dashboard Pages",
"timestamp": "Jan \u003cvar data-var='date'\u003e6\u003c/var\u003e, \u003cvar data-var='time'\u003e01:07\u003c/var\u003e - \u003cvar data-var='time'\u003e02:39\u003c/var\u003e UTC"
}
],
"name": "January",
"year": 2026
}
],
"start_time": "2026-01-01T00:00:00Z",
"time_zone": "UTC"
},
{
"end_time": "2025-12-31T23:59:59Z",
"months": [
{
"incidents": [],
"name": "December",
"year": 2025
},
{
"incidents": [
{
"code": "9gsnx1f0fygy",
"impact": "minor",
"message": "We have confirmed that error rates for log retrieval have returned to nominal levels.",
"name": "Elevated error rate for viewing logs",
"timestamp": "Nov \u003cvar data-var='date'\u003e21\u003c/var\u003e, \u003cvar data-var='time'\u003e04:11\u003c/var\u003e - \u003cvar data-var='time'\u003e05:03\u003c/var\u003e UTC"
},
{
"code": "zbs54hxmg525",
"impact": "maintenance",
"message": "Maintenance completed 02:20 UTC. We have confirmed builds to be in their correct states.",
"name": "[Scheduled] Database maintenance",
"timestamp": "Nov \u003cvar data-var='date'\u003e16\u003c/var\u003e, \u003cvar data-var='time'\u003e02:00\u003c/var\u003e - \u003cvar data-var='time'\u003e03:14\u003c/var\u003e UTC"
},
{
"code": "9982nj7c5yhh",
"impact": "minor",
"message": "We believe the impact from the issue is over, but we will continue monitoring today.",
"name": "Increased latency and error rates",
"timestamp": "Nov \u003cvar data-var='date'\u003e12\u003c/var\u003e, \u003cvar data-var='time'\u003e20:05\u003c/var\u003e - \u003cvar data-var='time'\u003e22:28\u003c/var\u003e UTC"
},
{
"code": "sv6phcn6xwwg",
"impact": "major",
"message": "This incident has been resolved. All services are operating normally and remaining jobs have been processed.\n\nWe apologise for the disruption and thank you for your patience.",
"name": "Increased error rate affecting Web UI and APIs",
"timestamp": "Nov \u003cvar data-var='date'\u003e10\u003c/var\u003e, \u003cvar data-var='time'\u003e05:22\u003c/var\u003e - \u003cvar data-var='time'\u003e12:00\u003c/var\u003e UTC"
}
],
"name": "November",
"year": 2025
},
{
"incidents": [
{
"code": "r9n9lqb3xrks",
"impact": "major",
"message": "Increase in 403 (Forbidden) errors affecting some Packagecloud repositories. Users may experience access denied errors when attempting to read packages.",
"name": "Increased 403 Errors on Package Registries package access",
"timestamp": "Oct \u003cvar data-var='date'\u003e24\u003c/var\u003e, \u003cvar data-var='time'\u003e03:30\u003c/var\u003e - \u003cvar data-var='time'\u003e03:30\u003c/var\u003e UTC"
},
{
"code": "3bjtdp9tll09",
"impact": "major",
"message": "Our services have been fully recovered for the last hour, so we are marking this as resolved. \n\nOur engineers will continue to monitor AWS and will keep services scaled up to prevent impact from any additional failures.",
"name": "Increased latency and error rates",
"timestamp": "Oct \u003cvar data-var='date'\u003e20\u003c/var\u003e, \u003cvar data-var='time'\u003e14:18\u003c/var\u003e - \u003cvar data-var='time'\u003e22:02\u003c/var\u003e UTC"
},
{
"code": "h3ksq8zzyzc2",
"impact": "none",
"message": "With the recovery of service within AWS, we are treating this incident as resolved. Customers running their agents in AWS may experience lingering issues as service within the region stabilizes.",
"name": "Ongoing AWS incident",
"timestamp": "Oct \u003cvar data-var='date'\u003e20\u003c/var\u003e, \u003cvar data-var='time'\u003e08:38\u003c/var\u003e - \u003cvar data-var='time'\u003e11:52\u003c/var\u003e UTC"
},
{
"code": "04yntwxj8t3b",
"impact": "none",
"message": "The rollback has been completed.",
"name": "Increased latency",
"timestamp": "Oct \u003cvar data-var='date'\u003e15\u003c/var\u003e, \u003cvar data-var='time'\u003e03:34\u003c/var\u003e - \u003cvar data-var='time'\u003e03:50\u003c/var\u003e UTC"
},
{
"code": "ynfllh6s5gfy",
"impact": "minor",
"message": "We think the impact from the issue is over.",
"name": "Delayed processing of execution uploads",
"timestamp": "Sep \u003cvar data-var='date'\u003e30\u003c/var\u003e, \u003cvar data-var='time'\u003e19:44\u003c/var\u003e - Oct \u003cvar data-var='date'\u003e1\u003c/var\u003e, \u003cvar data-var='time'\u003e06:28\u003c/var\u003e UTC"
}
],
"name": "October",
"year": 2025
}
],
"start_time": "2025-10-01T00:00:00Z",
"time_zone": "UTC"
}
]
}