Snapshot 316512
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2017-03-17T19:43:18.878Z",
"description": "CircleCI jobs using the docker executor.",
"group_id": null,
"id": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-20T23:48:21.962Z"
},
{
"created_at": "2019-05-02T20:42:22.366Z",
"description": "CircleCI jobs using the machine executor, and jobs using the setup_remote_docker step.",
"group_id": null,
"id": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T19:22:09.856Z"
},
{
"created_at": "2017-11-09T08:22:00.885Z",
"description": "CircleCI jobs using the macos executor.",
"group_id": null,
"id": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T19:22:09.876Z"
},
{
"created_at": "2020-10-23T23:37:10.052Z",
"description": "Jobs using the Windows executor.",
"group_id": null,
"id": "svt88btymymk",
"name": "Windows Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 4,
"showcase": true,
"start_date": "2020-10-23T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-11-19T19:22:09.896Z"
},
{
"created_at": "2017-07-10T21:13:25.145Z",
"description": null,
"group_id": null,
"id": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"page_id": "6w4r0ttlx5ft",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-14T00:26:03.788Z"
},
{
"created_at": "2025-05-02T13:47:11.283Z",
"description": "Public facing APIs",
"group_id": null,
"id": "t489m38xq83m",
"name": "CircleCI API",
"page_id": "6w4r0ttlx5ft",
"position": 6,
"showcase": true,
"start_date": "2025-05-02T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-14T00:22:39.024Z"
},
{
"created_at": "2017-08-11T03:29:32.538Z",
"description": null,
"group_id": null,
"id": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"page_id": "6w4r0ttlx5ft",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-14T00:23:12.766Z"
},
{
"created_at": "2016-02-19T18:21:42.888Z",
"description": "Build artifacts, workspaces, and test results.",
"group_id": null,
"id": "bckmjhmfhjnb",
"name": "Artifacts",
"page_id": "6w4r0ttlx5ft",
"position": 8,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T12:06:12.864Z"
},
{
"created_at": "2020-11-25T21:13:25.466Z",
"description": "Subsystem responsible for managing and executing runners and their jobs.",
"group_id": null,
"id": "g157tj6jrzwy",
"name": "Runner",
"page_id": "6w4r0ttlx5ft",
"position": 9,
"showcase": true,
"start_date": "2020-11-17T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-06-04T02:10:22.794Z"
},
{
"created_at": "2021-08-16T17:40:51.027Z",
"description": "Webhooks sent from CircleCI",
"group_id": null,
"id": "60j1gp5j396y",
"name": "CircleCI Webhooks",
"page_id": "6w4r0ttlx5ft",
"position": 10,
"showcase": true,
"start_date": "2021-07-26T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-05-02T13:48:42.226Z"
},
{
"created_at": "2021-09-08T16:47:55.711Z",
"description": "The CircleCI Insights dashboard provides an overview of the health and usage of your repository build processes.",
"group_id": null,
"id": "gs1qyd7g3l17",
"name": "CircleCI Insights",
"page_id": "6w4r0ttlx5ft",
"position": 11,
"showcase": true,
"start_date": "2021-09-08T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-08-17T21:36:07.627Z"
},
{
"created_at": "2024-02-19T17:40:30.067Z",
"description": "Subsystem responsible for tracking and managing releases.",
"group_id": null,
"id": "k7z3xkf61sff",
"name": "CircleCI Releases",
"page_id": "6w4r0ttlx5ft",
"position": 12,
"showcase": true,
"start_date": "2024-02-19T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-05-02T13:48:42.246Z"
},
{
"created_at": "2019-01-21T21:45:43.473Z",
"description": "GitHub Checks, Statuses, and status badges.",
"group_id": null,
"id": "rms1xkdxxqx1",
"name": "Notifications \u0026 Status Updates",
"page_id": "6w4r0ttlx5ft",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-07T04:05:16.613Z"
},
{
"created_at": "2020-05-22T18:12:56.408Z",
"description": "Payment processing and billing account changes.",
"group_id": null,
"id": "gx397f9wvq4w",
"name": "Billing \u0026 Account",
"page_id": "6w4r0ttlx5ft",
"position": 14,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-06T15:24:59.832Z"
},
{
"components": [
"31vvcfzgyyzk",
"sj405f2dg5ny",
"6djp8tfd8jwz",
"kyythj8ww584",
"w7wdrfrg5vxc",
"gk4m0c5zbbwp",
"08wtmlrf87bv",
"rrppq2vq36l9",
"87pyfnhlkqyk",
"gmbh6htwlsxr"
],
"created_at": "2020-10-23T23:02:58.992Z",
"description": "Third party services that CircleCI infrastructure depends on.",
"group_id": null,
"id": "5051gf6x40v1",
"name": "CircleCI Dependencies",
"page_id": "6w4r0ttlx5ft",
"position": 15,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-05-02T13:48:42.302Z"
},
{
"components": [
"8gmrsrb87lxr",
"9vxrymzc4lh5",
"gx07jwf1z30r",
"xv4vppsntybr",
"x113zcvdh42r",
"9284lhdn3v38",
"3rs8bptvb4bj",
"ys748wcgjv2s",
"q3bnj2nd1cfv",
"mqknb26g2crg",
"ccsb4l1ld2ph",
"gv5l5f77r93l",
"84h32fvwzs11",
"3f11clbhz1mx",
"n8jcyrrghcr3"
],
"created_at": "2020-10-23T23:23:34.529Z",
"description": "Third party services that may impact jobs.",
"group_id": null,
"id": "s21fdjds0k15",
"name": "Upstream Services",
"page_id": "6w4r0ttlx5ft",
"position": 16,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-05-02T13:48:42.342Z"
}
],
"incidents": [
{
"components": [
{
"created_at": "2017-03-17T19:43:18.878Z",
"description": "CircleCI jobs using the docker executor.",
"group_id": null,
"id": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-20T23:48:21.962Z"
}
],
"created_at": "2026-11-20T21:22:21.768Z",
"id": "vlz24tmd3tpd",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We have seen recovery for IP range jobs for gen2 docker jobs. We appreciate your patience while we worked through the incident.",
"created_at": "2026-11-20T23:48:21.999Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T23:48:21.999Z",
"id": "g30vmcmglg9l",
"incident_id": "vlz24tmd3tpd",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T23:48:21.999Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to monitor the rollout for recovery.",
"created_at": "2026-11-20T23:11:26.018Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T23:11:26.018Z",
"id": "r9vb870sx5h5",
"incident_id": "vlz24tmd3tpd",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T23:11:26.018Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been fully deployed, we are now monitoring for recovery.",
"created_at": "2026-11-20T22:27:18.318Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T22:27:18.318Z",
"id": "jq7sp9yp42lf",
"incident_id": "vlz24tmd3tpd",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T22:27:18.318Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The fix for IP range docker jobs are rolling out across the fleet. We will monitor for recovery in infra fails.",
"created_at": "2026-11-20T22:19:10.375Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T22:19:10.375Z",
"id": "txmmqrn2hs7g",
"incident_id": "vlz24tmd3tpd",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T22:19:10.375Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to rollout a fix for IP range docker jobs. Customers may continue to retry jobs that use IP ranges that experience infra fails.\n\nWe will provide an update within 20 minutes on the fix.",
"created_at": "2026-11-20T21:58:55.924Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T21:58:55.924Z",
"id": "9899j8ff36g5",
"incident_id": "vlz24tmd3tpd",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T21:58:55.924Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Customers running Docker executor jobs with the IP ranges feature enabled are experiencing a higher rate of infrastructure failures. A fix is being rolled out. Customers who see an infrastructure failure on one of these jobs can rerun the failed job. We will provide another update as we have more information to share.",
"created_at": "2026-11-20T21:22:21.859Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T21:22:21.859Z",
"id": "nkgz4v14xs2z",
"incident_id": "vlz24tmd3tpd",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T21:22:21.859Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-20T22:27:18.318Z",
"name": "Jobs using IP ranges failing with infrastructure errors",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-20T23:48:21.999Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/m8bpg1brfjcw",
"started_at": "2026-11-20T21:22:21.762Z",
"status": "resolved",
"updated_at": "2026-11-20T23:48:22.020Z"
},
{
"components": [
{
"created_at": "2017-03-17T19:43:18.878Z",
"description": "CircleCI jobs using the docker executor.",
"group_id": null,
"id": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-20T23:48:21.962Z"
},
{
"created_at": "2019-05-02T20:42:22.366Z",
"description": "CircleCI jobs using the machine executor, and jobs using the setup_remote_docker step.",
"group_id": null,
"id": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T19:22:09.856Z"
},
{
"created_at": "2017-11-09T08:22:00.885Z",
"description": "CircleCI jobs using the macos executor.",
"group_id": null,
"id": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T19:22:09.876Z"
},
{
"created_at": "2020-10-23T23:37:10.052Z",
"description": "Jobs using the Windows executor.",
"group_id": null,
"id": "svt88btymymk",
"name": "Windows Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 4,
"showcase": true,
"start_date": "2020-10-23T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-11-19T19:22:09.896Z"
}
],
"created_at": "2026-11-19T16:34:11.594Z",
"id": "s501b363k4hs",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "svt88btymymk",
"name": "Windows Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Smarter Testing functionality is restored.\n\nWe appreciate your patience as we worked through this.",
"created_at": "2026-11-19T19:22:09.927Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T19:22:09.927Z",
"id": "p21hy8pb9xqc",
"incident_id": "s501b363k4hs",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T19:22:09.927Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "svt88btymymk",
"name": "Windows Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The maintenance was successful. Smarter Testing functionality has been restored.\n\nWe are monitoring to ensure performance remains stable.",
"created_at": "2026-11-19T19:08:10.618Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T19:08:10.618Z",
"id": "cs7q3bdy132l",
"incident_id": "s501b363k4hs",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T19:08:10.618Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "svt88btymymk",
"name": "Windows Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "At 18:45 UTC, we will perform emergency maintenance, which is expected to take a few minutes. During this window, test jobs for customers using Smarter Testing may be delayed or fail.\n\nThank you for your patience while our engineers work to resolve this.\nWe will provide another update by 19:05 UTC.",
"created_at": "2026-11-19T18:40:41.768Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T18:40:41.768Z",
"id": "vqt2sl0j682l",
"incident_id": "s501b363k4hs",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T18:40:41.768Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "svt88btymymk",
"name": "Windows Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Since 17:55 UTC, test jobs for customers using Smarter Testing have been running again.\n\nDynamic Test Splitting is still not running optimally.\n\nDynamic Parallelism is also not running optimally, so jobs that use it should be expected to fall back to using all nodes.\n\nThank you for your patience while our engineers work to resolve this.\n\nWe will provide another update by 18:50 UTC",
"created_at": "2026-11-19T18:30:34.840Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T18:30:34.000Z",
"id": "c9vghpmr7qst",
"incident_id": "s501b363k4hs",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T18:31:41.177Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "svt88btymymk",
"name": "Windows Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Since 17:55 UTC, test jobs for customers using Smarter Testing have been running again.\n\nDynamic Test Splitting is still not running optimally.\n\nDynamic Parallelism is also not running optimally, so jobs that use it should be expected to fall back to using all nodes.\n\nThank you for your patience while our engineers work to resolve this.\n\nWe will provide another update by 18:30 UTC.",
"created_at": "2026-11-19T18:13:12.688Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T18:13:12.688Z",
"id": "4rs9t3bv3m5b",
"incident_id": "s501b363k4hs",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T18:13:12.688Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Since 17:38 UTC, some test jobs for customers using Smarter Testing have been failing.\n\nThank you for your patience while our engineers work to resolve this.",
"created_at": "2026-11-19T17:45:38.381Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-19T17:45:38.381Z",
"id": "gbx8gvvx0q9w",
"incident_id": "s501b363k4hs",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T17:45:38.381Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Since 17:09 UTC, we have been seeing delays in updating test impact analysis data.\n\nTest selection may be less precise, which may cause more tests than necessary to run. Thank you for your patience while our engineers work to resolve this.",
"created_at": "2026-11-19T17:30:29.464Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-19T17:30:29.464Z",
"id": "4v5vq9xfph5q",
"incident_id": "s501b363k4hs",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T17:30:29.464Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Starting at 16:38 UTC, test jobs for customers using Smarter Testing should be back running. Dynamic Test Splitting may still not be working optimally for some customers.",
"created_at": "2026-11-19T17:04:41.243Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-19T17:04:41.243Z",
"id": "kx3wrx9qwgcl",
"incident_id": "s501b363k4hs",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T17:04:41.243Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Some Smarter Testing customers may not be able to run tests.\n\nWe are also seeing delays updating the impact analysis data map, which may cause more tests to be selected by Smarter Testing.",
"created_at": "2026-11-19T16:34:11.653Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-19T16:34:11.653Z",
"id": "gq4xf3yl31tr",
"incident_id": "s501b363k4hs",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T16:34:11.653Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-19T19:08:10.618Z",
"name": "Delays and intermittent failures running Smarter Testing",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": "## Summary\n\n_\\(All times UTC\\)_\n\nFrom 13:20 to 19:19 on November 19, customers using Smarter Testing saw degraded test selection. Between 15:50 and 18:00, some test jobs using Smarter Testing failed intermittently. In that window, 13% of jobs using Smarter Testing encountered an error, and the rest ran without errors. Most failures happened during two outages, from 16:20 to 16:38 and from 17:37 to 17:55. Customers not using Smarter Testing were not affected.\n\nThe incident was caused by unusually large impact data that our system did not handle well. It retried the updates repeatedly, which kept CPU on the database behind Smarter Testing at 96-98% for over two hours. The database ran on an instance type that relies on burst capacity. When that capacity ran out at 16:20, the database stopped responding.\n\nIn response, our engineering team stopped processing the oversized data and deleted the queued updates for it. At 18:50, we moved the database to a larger instance type that does not rely on burst capacity. We resolved the incident at 19:19.\n\n**Customers do not need to run a full analysis**. Impact data updates were delayed, and we deleted some queued updates during the incident, so some projects may select more tests than necessary until their next analysis. Running regular analysis reconverges on the correct impact data.\n\nThe original status page can be found here: [Delays and intermittent failures running Smarter Testing](https://status.circleci.com/incidents/s501b363k4hs).\n\n## Background\n\nSmarter Testing uses data stored for each project to decide which tests to run \\(test impact analysis\\), how to split them across parallel nodes, and how many nodes a job needs. To decide which tests to run, Smarter Testing uses impact data, a map of which files each test touches, and this was the part of the system most affected by this incident. When a job runs, Smarter Testing records updates to this data, and our system applies them to a database in the background.\n\nDuring a test job, Smarter Testing contacts our backend service, which reads from that database. If that service is unavailable, test jobs using Smarter Testing cannot run their tests.\n\nThe size of a project’s impact data depends on how many tests it has and how many files each test touches. Unusually large impact data can place heavy load on the database, and our safeguards did not stop this.\n\n## What Happened\n\n_\\(All times UTC\\)_\n\nAt 12:12 on November 19, a Smarter Testing analysis began producing impact data that grew steadily until it was unusually large. Between 13:15 and 14:30, our system scaled the workers that apply these updates. The additional workers added load to the database, and between 13:15 and 14:00, database CPU rose to 96-98%. At 13:20, the workers began timing out on these updates and retrying them repeatedly. From then on, impact data updates were delayed, so test selection used older data and some jobs could run more tests than necessary.\n\nAt 14:12, our engineering team was alerted to delays in processing impact data, and at 14:17 we began investigating. At 14:33, we opened an incident. At first, the impact appeared limited to delayed updates.\n\nAt 15:07, we found that the impact data being applied was far larger than normal and was driving the load on the database. By 15:17, we projected that the database would use up its disk throughput burst balance in about an hour and that free storage would run out in about two.\n\nWe took three steps to remove the source of the load. At 15:27, we blocked the analysis that was producing this data, and at 15:28, we cancelled the job running it. From 15:50, some test jobs using Smarter Testing began to fail intermittently. At 15:51, we dropped the queued updates for this data so the workers would stop retrying them.\n\nWe also considered moving the database to a larger instance type. That change requires a restart and a short outage and at that point, the impact appeared limited to delays in impact data, so we opted to remove the load first. At 16:05, we increased storage because free space was falling quickly. That change put the database into a modifying state, which blocked the instance type change until 18:18.\n\nAt 16:20, the database used up its disk throughput burst balance and stopped responding. Jobs using Smarter Testing failed. We had no way to switch Smarter Testing off for customers, so we focused on cutting load. Jobs recovered at 16:38. We scaled the workers to zero \\(16:41\\) and we deleted the remaining queued updates for this data \\(16:42\\), and a single restarted worker had cleared the backlog by 16:58.\n\nAt 17:00, database CPU rose again, to 65-82%, while the burst balance was still exhausted. To reduce load, we reduced batch sizes and ran a single worker. By 17:34, we had concluded that only an instance type without burst limits would fix this, but the storage change still had the database in a modifying state. At 17:37, the database stopped responding again, and test jobs using Smarter Testing failed. At 17:53, it failed over to its standby instance, which restored its burst balance, and test jobs were running again by 17:55.\n\nFrom 16:20 to 18:00, some requests for selection, timing and parallelism data timed out. During that time, dynamic test splitting may have used older timing data, and jobs using dynamic parallelism ran on their maximum number of nodes.\n\nWhen the storage change completed at 18:18, we announced emergency maintenance, and at 18:50 we moved the database to a larger instance type without burst limits, with about 45 seconds of downtime. We scaled the workers back up to clear the backlog, and we resolved the incident at 19:19.\n\n## Future Prevention and Process Improvement\n\n### Changes already in place:\n\n**A larger database.** We moved the Smarter Testing database to an instance type that does not rely on burst capacity, which was the limit that caused both outages. We also increased its storage and tuned its background cleanup. Database CPU fell afterward and we are monitoring it closely.\n\n**We made test jobs resilient to Smarter Testing outages.** Since the incident, we added a fallback that our engineering team can turn on if Smarter Testing becomes unavailable. With the fallback on, jobs continue to run their tests without Smarter Testing features, rather than failing.\n\n### Further improvements\n\n**Stronger safeguards against unusually large data.** We are adding protections so that unusually large data is limited and we are improving how our system handles failed updates.\n\n**We are improving database performance.** We are optimizing the queries our system runs against the Smarter Testing database, so it can handle more load.\n\n**Warning us sooner.** We are adding earlier alerts on database capacity.\n\n## Conclusion\n\nSince the incident, the Smarter Testing database has been running on infrastructure with substantially more resources and no burst limits, and it has been stable since the change. Customer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. We thank our customers for their patience while our team worked to resolve this incident.\n\nPlease reach out to our support team with any questions or concerns.",
"postmortem_body_last_updated_at": "2026-11-20T23:40:29.661Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-20T23:40:29.661Z",
"reminder_intervals": null,
"resolved_at": "2026-11-19T19:22:09.927Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/g20zht6r8c4w",
"started_at": "2026-11-19T18:30:34.000Z",
"status": "resolved",
"updated_at": "2026-11-20T23:40:29.684Z"
},
{
"components": [
{
"created_at": "2019-05-02T20:42:22.366Z",
"description": "CircleCI jobs using the machine executor, and jobs using the setup_remote_docker step.",
"group_id": null,
"id": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T19:22:09.856Z"
}
],
"created_at": "2026-11-03T14:14:39.000Z",
"id": "qqh50c0z6mn3",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Wait times have returned to normal for Machine and Remote Docker. We appreciate your patience as we worked through this.",
"created_at": "2026-11-03T15:08:13.535Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T15:08:13.535Z",
"id": "9vbl33p2jhtn",
"incident_id": "qqh50c0z6mn3",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T15:08:13.535Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Our team has applied a fix and we're starting to see wait times recover. We're keeping an eye on this as it continues to rollout and we'll update again shortly. We thank you again for your patience.",
"created_at": "2026-11-03T14:45:31.270Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T14:45:31.270Z",
"id": "dk0whgt6tyxf",
"incident_id": "qqh50c0z6mn3",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T14:45:31.270Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "0l5xn1l7qs8f",
"name": "Machine Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Some users might see increased wait times for Machine and Remote Docker jobs due to increased usage. Wait times right now are just under 7 mins. Our team is investigating and we'll post another update shortly. We appreciate your patience.",
"created_at": "2026-11-03T14:14:39.425Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-03T14:14:39.000Z",
"id": "dprk97m9jwgr",
"incident_id": "qqh50c0z6mn3",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T14:41:29.928Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-03T14:45:31.270Z",
"name": "Increased wait time for Machine and Remote Docker jobs",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": "## Summary\n\nFrom 13:55 to 15:07 UTC on November 3, customers running Linux machine jobs and Remote Docker saw increased wait times for their jobs to start, and the delays were longest from 14:16 to 14:41. Some Docker jobs also saw increased wait times.\n\nThe incident was caused by our usage reaching the vCPU limit on our cloud provider account in our main cloud region. Machine and Docker jobs share this limit, and the additional Docker capacity we have run since the [October 21 incident](https://discuss.circleci.com/t/post-incident-report-oct-21-2026-increased-wait-times-for-gen-2-docker-jobs/54940) brought our usage to the limit as demand increased. Machine jobs did not move to our backup regions because our system did not treat reaching this limit the same way as a shortage of cloud instances.\n\nIn response, our engineering team reconfigured machine jobs to balance across all of our regions instead of relying on backup regions, and reduced Docker capacity to make room under the limit. By 14:46, wait times had improved, and by 15:07, machine wait times had fully recovered.\n\nAfter we resolved the incident, our cloud provider doubled the vCPU limit in our main region. We have also restored the Docker capacity we reduced during the incident.\n\nThe original status page can be found here: [Increased wait time for Machine and Remote Docker jobs](https://status.circleci.com/incidents/qqh50c0z6mn3).\n\n## Background\n\nMachine jobs need a dedicated virtual machine, so when CircleCI runs a machine job, we create a fresh virtual machine for that job and delete it when the job ends. Remote Docker runs on these same virtual machines. Docker jobs run on shared cloud instances, and each instance can run multiple isolated jobs.\n\nOur cloud provider sets a vCPU limit for each region, which caps the total computing capacity our account can run there. Machine and Docker jobs run in the same account, so they count against the same vCPU limit. When our usage reaches that limit, the cloud provider rejects requests for new cloud instances until existing ones shut down.\n\nMachine jobs normally run in our main region, with other cloud regions available as backups. If the cloud provider cannot supply the instance types we need in our main region, our system sends new work to a backup region.\n\nFollowing the October 21 incident, we increased the minimum fleet size for our Docker clusters during weekday peak hours, so that capacity is in place before demand arrives. That capacity counts against the same vCPU limit as machine jobs.\n\n## What Happened\n\n_\\(All times UTC\\)_\n\nStarting at 13:55 on November 3, Linux machine jobs saw increased wait times because our usage in our main region had reached our vCPU limit, and our cloud provider was rejecting requests for new virtual machines. We had not noticed that our alert for this limit was no longer receiving data, so it did not warn us as our usage approached the limit.\n\nAt 13:55, our engineering team was alerted to errors launching new virtual machines and began investigating. By 14:06, 40% of Linux machine jobs waited more than 3 minutes to start. By 14:08, we had confirmed the cause and requested a higher limit from our cloud provider.\n\nOur backup regions had capacity available under their own limits, but machine jobs did not move there because our system did not treat reaching our vCPU limit the same way as a shortage of cloud instances. From 14:16 to 14:41, most Linux machine jobs waited 5-9 minutes to start, and some waited more than 20 minutes.\n\nAt 14:34, our engineering team reconfigured machine jobs to balance new work across all of our regions, rather than only using backup regions when our main region runs short. The change took effect at 14:40, and at the same time we reduced the minimum fleet size for our Docker clusters to make room under the limit.\n\nBy 14:46, wait times had improved, and nearly all Linux machine jobs started within 3 minutes. Delays of 1-3 minutes for some jobs continued until 15:07. We resolved the incident at 15:08.\n\nDocker jobs also saw increased wait times during the incident. Up to 7% of Docker jobs waited longer than usual to start, and that share rose to 19% between 14:39 and 14:43. Docker wait times returned to normal by 15:13.\n\n## Future Prevention and Process Improvement\n\n**We raised our vCPU limit.** Our cloud provider doubled the vCPU limit in our main region, and we have requested the same increase in our other US regions. We have also requested higher limits for related storage and networking resources. Capacity we reserve in advance counts against the limit whether or not jobs are using it, so we are moving some of our reserved capacity out of our main region.\n\n**We are changing how machine jobs handle the vCPU limit.** We are updating our system to treat reaching a vCPU limit the same way as a shortage of cloud instances, so new work will move to a backup region.\n\n**We are separating machine and Docker capacity.** We are moving machine jobs into their own cloud provider account, so Docker capacity will no longer use up the vCPU limit that machine jobs depend on.\n\n**We are fixing our alerts for cloud provider limits.** We are restoring the data our vCPU limit alert depends on and adding alerts for our other limits, so our engineering team will be warned before usage reaches them. If any of these alerts stops receiving data, our engineering team will be alerted, so a gap like this one does not go unnoticed again.\n\n## Conclusion\n\nCustomer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. We thank our customers for their patience while our team worked to resolve this incident.\n\nPlease reach out to our support team with any questions or concerns.",
"postmortem_body_last_updated_at": "2026-11-04T22:14:36.203Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-04T22:14:36.203Z",
"reminder_intervals": null,
"resolved_at": "2026-11-03T15:08:13.535Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/pf820dqbcwhc",
"started_at": "2026-11-03T14:14:39.000Z",
"status": "resolved",
"updated_at": "2026-11-04T22:14:36.224Z"
},
{
"components": [
{
"created_at": "2017-03-17T19:43:18.878Z",
"description": "CircleCI jobs using the docker executor.",
"group_id": null,
"id": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-20T23:48:21.962Z"
}
],
"created_at": "2026-10-28T18:15:52.753Z",
"id": "8q9j33r2rssy",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Wait times for gen2 dedicated IP range jobs have fully returned to normal. We appreciate your patience while worked through this issue.",
"created_at": "2026-10-28T19:19:03.311Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T19:19:03.311Z",
"id": "g6zyncfbvp1g",
"incident_id": "8q9j33r2rssy",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T19:19:03.311Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are starting to see signs of recovery now, wait times are starting to return to normal. We're keeping an eye on this and we'll update again soon.",
"created_at": "2026-10-28T19:06:41.630Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T19:06:41.630Z",
"id": "w1kq4tfpl13g",
"incident_id": "8q9j33r2rssy",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T19:06:41.630Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We’ve identified the cause as limited compute availability from our cloud provider, and we are working to free up additional capacity for these jobs to restore normal start times. We'll provide another update again shortly.",
"created_at": "2026-10-28T18:51:35.778Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T18:51:35.778Z",
"id": "21hrh7mzv7ky",
"incident_id": "8q9j33r2rssy",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T18:51:35.778Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating delays in starting docker gen2 jobs that use dedicated IP ranges. Jobs are currently delayed by about 10 minutes.\n\nNo other resource classes are impacted at this time. We will continue to provide updates as more information is available.",
"created_at": "2026-10-28T18:15:52.870Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-28T18:15:52.870Z",
"id": "x6tpvbmqw9j1",
"incident_id": "8q9j33r2rssy",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T18:15:52.870Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-28T19:06:41.630Z",
"name": "Delays for gen2 dedicated IP range jobs",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": "## **Summary**\n\nFrom 14:20 to 19:08 UTC on October 28, customers running Gen 2 Docker jobs using IP Ranges saw increased wait times for their jobs to start. From 14:20 until 17:02, the delays were intermittent, and from 17:02 to 18:50, the delays were continuous. No other job types were affected.\n\nThe incident was caused by limited availability of the cloud instances we had configured for the IP Ranges cluster. In response, our engineering team added additional fallback instance types to the IP Ranges cluster and transferred capacity from one of our other Gen 2 clusters to the IP Ranges cluster. By 18:50, wait times had improved, and by 19:08 they had fully recovered.\n\nAfter we resolved the incident, we increased the number of baseline cloud instances we maintain for IP Ranges jobs. We have also configured an IP Ranges cluster in an additional cloud region.\n\nThe original status page can be found here: [Delays for gen2 dedicated IP range jobs](https://status.circleci.com/incidents/8q9j33r2rssy).\n\n## **Background**\n\nIP Ranges is a CircleCI feature that runs jobs from a published static list of IP addresses, so customers can configure their firewalls to allow those addresses. Gen 2 Docker jobs using IP Ranges need to run on their own dedicated cluster of cloud instances, separate from our other Gen 2 Docker jobs. Customers rely on the published IP address list, so we give 30 days’ notice before adding new IP addresses. Because of that notice period, we cannot move IP Ranges jobs to a new cloud region on short notice.\n\nWhen CircleCI runs a Docker job, we schedule it on one of several clusters of cloud instances, and each instance can run multiple isolated jobs. We provision cloud instances using a ranked list of instance types. If the cloud provider cannot supply our first choice, we fall back to the next type on the list. Each request tries our preferred types first, so when several of them are unavailable, each request takes longer and the cluster grows more slowly.\n\nWe also maintain a minimum fleet size: a minimum number of instances that stay running, so that capacity is ready before demand arrives. Above that minimum, we add instances as demand rises. When there is insufficient capacity to run a job, the job waits in a queue until capacity becomes available.\n\n## **What Happened**\n\n_\\(All times UTC\\)_\n\nStarting at 14:20 on October 28, Gen 2 Docker jobs using IP Ranges saw intermittent delays because the instance types we had configured for the IP Ranges cluster were not available in the quantities we needed. Wait times rose to 5 minutes and then returned to normal several times until 17:02. From 17:02, the delays were continuous.\n\nOur wait time alerts did not specifically isolate IP Ranges jobs, so these delays did not immediately alert our engineering team. At 17:22, our engineering team began investigating and confirmed that the IP Ranges cluster could not provision sufficient instances. An unrelated incident at our observability provider from 16:10 to 18:39 also slowed our investigation by affecting dashboards and metric queries.\n\nFollowing the [October 21 incident](https://discuss.circleci.com/t/post-incident-report-oct-21-2026-increased-wait-times-for-gen-2-docker-jobs/54940), we added fallback instance types and additional cloud regions for our main Gen 2 clusters, but the IP Ranges cluster had not yet received those changes. Our engineering team had scheduled work for October 30 to configure the IP Ranges cluster with additional fallback instance types, but that work had not begun at the time of this incident.\n\nBecause of the 30-day notice, we could not immediately move IP Ranges jobs to another region, so our engineering team reallocated capacity from other CircleCI clusters in the same region. By 18:50, the IP Ranges cluster had provisioned sufficient instances, and wait times began returning to normal. Delays of 1-2 minutes continued until 19:08. We resolved the incident at 19:19.\n\n## **Future Prevention and Process Improvement**\n\n**We raised the minimum fleet size for the IP Ranges cluster.** We now keep more cloud instances running for IP Ranges jobs during weekday peak hours. This means capacity is in place before demand arrives, and we do not depend on the cloud provider supplying instances during an increase in demand.\n\n**We have added an additional region for IP Ranges jobs.** We have set up dedicated IP Ranges clusters for Gen 1 and Gen 2 in a second cloud region, so they will be able to run jobs as soon as the 30 day notice period ends. During that notice period, the higher minimum fleet size and the expanded list of fallback instance types protect IP Ranges jobs in our current region.\n\n**We are expanding our alerting.** We are adding more targeted wait time alerts specifically for IP Ranges jobs, so our team can respond immediately to delays.\n\n## **Conclusion**\n\nCustomer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. We thank our customers for their patience while our team worked to resolve this incident.\n\nPlease reach out to our support team with any questions or concerns.",
"postmortem_body_last_updated_at": "2026-10-31T00:22:05.766Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-31T00:22:05.766Z",
"reminder_intervals": null,
"resolved_at": "2026-10-28T19:19:03.311Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/t4pc57wmjnql",
"started_at": "2026-10-28T18:15:52.747Z",
"status": "resolved",
"updated_at": "2026-10-31T00:22:05.790Z"
},
{
"components": [
{
"created_at": "2017-03-17T19:43:18.878Z",
"description": "CircleCI jobs using the docker executor.",
"group_id": null,
"id": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-20T23:48:21.962Z"
}
],
"created_at": "2026-10-21T19:31:05.335Z",
"id": "5jb50gh55gtj",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Wait times for gen2 Docker jobs have recovered and jobs are running within normal wait times.",
"created_at": "2026-10-21T20:54:46.671Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T20:54:46.671Z",
"id": "syj6pmr2751w",
"incident_id": "5jb50gh55gtj",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T20:54:46.671Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have seen a full recovery of wait times for docker jobs.",
"created_at": "2026-10-21T20:40:44.495Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T20:40:44.495Z",
"id": "4wz4sfm9d91v",
"incident_id": "5jb50gh55gtj",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T20:40:44.495Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Wait times for 2xlarge docker jobs are now under 60 seconds.\n\nWait times for other resource sizes were not impacted by this incident and remain under 10 seconds.",
"created_at": "2026-10-21T20:35:42.888Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T20:35:42.000Z",
"id": "7vfgg8shy11c",
"incident_id": "5jb50gh55gtj",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T20:37:51.135Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Wait times for 2xlarge docker jobs are recovering. We will continue to update as capacity comes online.",
"created_at": "2026-10-21T20:22:53.499Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T20:22:53.499Z",
"id": "cv891p8g78ml",
"incident_id": "5jb50gh55gtj",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T20:22:53.499Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are adding additional capacity to help mitigate the issue. Note: We are still seeing delays up to 20 minutes on 2xlarge.",
"created_at": "2026-10-21T20:03:35.498Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T20:03:35.498Z",
"id": "27mq666303jj",
"incident_id": "5jb50gh55gtj",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T20:03:35.498Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "x1v3t5qrnlhk",
"name": "Docker Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Some users might see increased wait times for gen2 Docker jobs due to capacity issues.",
"created_at": "2026-10-21T19:31:05.416Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T19:31:05.416Z",
"id": "x0ryrjdchnkp",
"incident_id": "5jb50gh55gtj",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T19:31:05.416Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-21T20:40:44.495Z",
"name": "Increased Docker gen2 wait times",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": "## Summary\n\nFrom 15:44 to 20:55 UTC on October 21, customers running Gen 2 Docker jobs saw increased wait times for their jobs to start. Delays were longest on the 2xlarge resource class, where jobs waited up to 20 minutes. The incident was caused by limited availability of the cloud instances we had configured to meet demand, particularly for 2xlarge jobs.\n\nIn response, our engineering team configured additional instance types, configured additional Gen 2 Docker clusters. By 20:55 UTC, wait times had returned to normal and the incident was resolved.\n\nAfter the incident was resolved, we expanded our Gen 2 Docker fleet into additional cloud regions, and we continue actively standing up additional cloud regions to prevent similar incidents in the future.\n\nThe original status page can be found here: [October 21 - Increased Docker gen2 wait times](https://status.circleci.com/incidents/5jb50gh55gtj).\n\n## Background\n\nWhen CircleCI executes a Docker job, we schedule it on one of several clusters of cloud instances, and each instance is capable of executing multiple isolated jobs. Scheduling a job on a cloud instance is a classic [“bin packing problem”](https://en.wikipedia.org/wiki/Bin_packing_problem): finding available capacity that fits the requested resource class’ CPU, memory, and storage. Larger resource classes like 2xlarge are harder to fit than smaller ones, which pack together more tightly.\n\nWe provision cloud instances using a ranked list of instance types. If the cloud provider cannot supply our first choice, we fall back to the next type on the list. We further maintain a minimum fleet size: a minimum number of instances stay running, so that we do not give up capacity when demand is low. Above that minimum fleet size, we add instances as demand rises.\n\nWhen there is insufficient capacity to execute a workload, we queue that work until capacity becomes available.\n\n## What Happened\n\n_\\(All times UTC\\)_\n\nStarting at 15:44 on October 21, demand for the Gen 2 2xlarge resource class rose sharply, and we did not provision additional capacity quickly enough to keep up. Our engineering team began investigating. By 19:25, wait times for the Gen 2 2xlarge resource class had reached 20 minutes, and wait times for other resource classes had increased as well.\n\nBy 19:56, our engineering team had configured additional instance types to the Gen 2 fleet in order to get more capacity online, and by 19:59, we were provisioning those instance types. At 20:11, we added several additional Gen 2 clusters, and increased the minimum fleet size on those clusters.\n\nBy 20:35, the number of queued jobs returned to normal levels, and by 20:41, 95% of 2xlarge jobs were starting within 4 seconds. We resolved the incident at 20:55.\n\nAfter resolving this incident, we added additional cloud regions for Docker jobs to make more capacity available.\n\n## Future Prevention and Process Improvement\n\nWe have already taken the following steps to improve job start times, including sourcing capacity from more places, reserving more capacity for large resource classes, and implementing more rapid provisioning.\n\n**We have added additional regions for Docker jobs.** We recently added additional cloud regions for Machine jobs, and we have now done the same for Docker jobs.\n\n**We will actively continue adding additional cloud regions for both Docker and Machine jobs**.\n\n**We have added additional Gen 2 clusters**. We configured several additional Gen 2 clusters to schedule jobs more quickly, and to make more total capacity available.\n\n**We have expanded the set of cloud instance types we use and have raised our minimum fleet size.** We have configured additional instance types. We have also increased the minimum fleet size for Gen 2 to meet current baseline demand.\n\n**We have modified our job scheduler to prioritize scheduling of larger resource classes.** We have changed how we schedule jobs to balance wait times more fairly across resource classes, and to bin-pack jobs more efficiently onto available instances.\n\n## Conclusion\n\nCustomer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. We thank our customers for their patience while our team worked to resolve this incident.\n\nPlease reach out to our support team with any questions or concerns.",
"postmortem_body_last_updated_at": "2026-10-22T23:31:44.584Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T23:31:44.584Z",
"reminder_intervals": null,
"resolved_at": "2026-10-21T20:54:46.671Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8g54j07z69lb",
"started_at": "2026-10-21T19:31:05.329Z",
"status": "resolved",
"updated_at": "2026-10-22T23:31:44.608Z"
},
{
"components": [
{
"created_at": "2017-11-09T08:22:00.885Z",
"description": "CircleCI jobs using the macos executor.",
"group_id": null,
"id": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T19:22:09.876Z"
}
],
"created_at": "2026-10-20T15:58:25.000Z",
"id": "rrrvc4lqplb5",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Wait times for macOS jobs on the m4pro.large resource class have returned to normal. We thank you for your patience while the backlogs cleared.",
"created_at": "2026-10-20T18:14:04.397Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T18:14:04.397Z",
"id": "85b06f5qg36t",
"incident_id": "rrrvc4lqplb5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T18:14:04.397Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Wait times for macOS jobs on the m4pro.large resource class are recovering. Jobs using Xcode 27.0 are now waiting about ~3.5 minutes on average, down from about ~40 minutes at the peak. Jobs using Xcode 26.6 and 26.4.1 are still working through a backlog, which is clearing. Wait times on m4pro.medium are normal. We will share another update soon as we continue to monitor.",
"created_at": "2026-10-20T17:31:54.658Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T17:31:54.658Z",
"id": "233q78brq5gc",
"incident_id": "rrrvc4lqplb5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T17:31:54.658Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Wait times for m4pro.large are still elevated due to heavy large demand. All m4pro.medium are normal. We're actively monitoring, we appreciate patience.",
"created_at": "2026-10-20T17:02:28.332Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T17:02:28.332Z",
"id": "v8gr9qxgsvmq",
"incident_id": "rrrvc4lqplb5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T17:02:28.332Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Customers running macOS jobs on the m4pro.large resource class are still seeing increased wait times, with some jobs waiting up to 30 minutes to start. We're continuing to monitor this and will share another update shortly.",
"created_at": "2026-10-20T16:32:17.257Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T16:32:17.000Z",
"id": "2yymsz0tzn0f",
"incident_id": "rrrvc4lqplb5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T16:34:00.192Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We're seeing increased wait times for some macOS jobs using the m4pro.large resource class. We're investigating and will provide another update soon. Thank you for your patience.",
"created_at": "2026-10-20T15:58:25.357Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T15:58:25.000Z",
"id": "ntxc3bthybzz",
"incident_id": "rrrvc4lqplb5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T16:31:38.541Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-20T16:32:17.000Z",
"name": "Increased wait times for macOS jobs on m4pro.large",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-20T18:14:04.397Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/pp6wqy3vlms3",
"started_at": "2026-10-20T15:58:25.000Z",
"status": "resolved",
"updated_at": "2026-10-20T18:14:04.419Z"
},
{
"components": [
{
"created_at": "2017-11-09T08:22:00.885Z",
"description": "CircleCI jobs using the macos executor.",
"group_id": null,
"id": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"page_id": "6w4r0ttlx5ft",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T19:22:09.876Z"
}
],
"created_at": "2026-10-14T14:47:48.002Z",
"id": "xc1s7vqky9rd",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Between approximately 14:00 UTC and 15:51 UTC on October 14, customers running macOS jobs that use Xcode 26.6.0 or Xcode 26.4 experienced delays starting jobs. From approximately 14:00 UTC, jobs that use Xcode 26.6.0 on the M4 Pro large resource class waited 30 minutes or more to start, and these delays recovered at approximately 15:12 UTC. From approximately 15:11 UTC, jobs that use Xcode 26.4 waited about 30 minutes to start. \n\nThe issue has been resolved and all affected functionality has returned to normal. We thank you for your patience while our team worked on implementing a fix.",
"created_at": "2026-10-14T16:01:16.750Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-14T16:01:16.750Z",
"id": "zfrjzr0703bt",
"incident_id": "xc1s7vqky9rd",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-14T16:01:16.750Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Delays starting macOS jobs that use Xcode 26.6.0 have recovered. Since approximately 15:11 UTC, macOS jobs that use Xcode 26.4 are experiencing delays starting, with wait times of about 30 minutes. \n\nThank you for your patience while our engineers work to resolve this. We will provide another update by 15:45 UTC.",
"created_at": "2026-10-14T15:17:16.952Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-14T15:17:16.952Z",
"id": "2h0nzgm2kfnl",
"incident_id": "xc1s7vqky9rd",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-14T15:17:16.952Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "bpfmy8wy8wq2",
"name": "macOS Jobs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Since approximately 14:00 UTC, customers running macOS jobs that use Xcode 26.6.0 on the M4 Pro large resource class are experiencing delays starting jobs. Wait times have reached 30 minutes or more. \n\nThank you for your patience while our engineers investigate. We will provide another update by 15:30 UTC.",
"created_at": "2026-10-14T14:47:48.245Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-14T14:47:48.245Z",
"id": "c6fbf1b0xrg5",
"incident_id": "xc1s7vqky9rd",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-14T14:47:48.245Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Delays starting macOS jobs that use Xcode 26.6.0",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-14T16:01:16.750Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/nhpcykblrb28",
"started_at": "2026-10-14T14:47:47.993Z",
"status": "resolved",
"updated_at": "2026-10-14T16:01:16.776Z"
},
{
"components": [
{
"created_at": "2017-07-10T21:13:25.145Z",
"description": null,
"group_id": null,
"id": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"page_id": "6w4r0ttlx5ft",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-14T00:26:03.788Z"
},
{
"created_at": "2017-08-11T03:29:32.538Z",
"description": null,
"group_id": null,
"id": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"page_id": "6w4r0ttlx5ft",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-14T00:23:12.766Z"
}
],
"created_at": "2026-10-11T17:37:45.504Z",
"id": "zlkzxwcyr7p2",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "Our maintenance has concluded, thank you for your patience.",
"created_at": "2026-10-11T20:22:43.368Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-11T20:22:43.368Z",
"id": "xbk4ytbs25k3",
"incident_id": "zlkzxwcyr7p2",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-11T20:22:43.368Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
}
],
"body": "The maintenance is still progressing. Our revised completion time is 20:30 UTC. We will continue to provide updates here.",
"created_at": "2026-10-11T19:26:07.288Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-11T19:26:07.288Z",
"id": "jyrsp473j49r",
"incident_id": "zlkzxwcyr7p2",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-11T19:26:07.288Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-11T17:40:08.395Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-11T17:40:08.395Z",
"id": "d1p89cj2kswx",
"incident_id": "zlkzxwcyr7p2",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-11T17:40:08.395Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are continuing to perform maintenance on our infrastructure. This should run October 11, 2026, from 17:30 UTC to 19:30 UTC. While we expect the impact to be minimal during this window, pipelines, workflows, jobs and notifications may briefly experience interruptions, including delays or failures in workflows and jobs starting. We will provide an update when the maintenance begins and when it has completed.",
"created_at": "2026-10-11T17:37:45.607Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-11T17:37:45.607Z",
"id": "j5xklzm7718m",
"incident_id": "zlkzxwcyr7p2",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-11T17:37:45.607Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Infrastructure Maintenance",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-11T20:22:43.368Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-11T17:40:00.000Z",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-11T20:40:00.000Z",
"shortlink": "https://stspg.io/f9nlzhzf8b8n",
"started_at": "2026-10-11T17:37:45.495Z",
"status": "completed",
"updated_at": "2026-10-11T20:22:43.416Z"
},
{
"components": [
{
"created_at": "2017-07-10T21:13:25.145Z",
"description": null,
"group_id": null,
"id": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"page_id": "6w4r0ttlx5ft",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-14T00:26:03.788Z"
},
{
"created_at": "2017-08-11T03:29:32.538Z",
"description": null,
"group_id": null,
"id": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"page_id": "6w4r0ttlx5ft",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-14T00:23:12.766Z"
}
],
"created_at": "2026-10-11T13:18:59.802Z",
"id": "n4hbmf9tzqzv",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-10-11T17:30:07.347Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-11T17:30:07.347Z",
"id": "8v6c7xvxc0ty",
"incident_id": "n4hbmf9tzqzv",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-11T17:30:07.347Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-11T13:30:07.340Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-11T13:30:07.340Z",
"id": "4wdjg37km4px",
"incident_id": "n4hbmf9tzqzv",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-11T13:30:07.340Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zxxnbynf4dkm",
"name": "Pipelines \u0026 Workflows",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing maintenance on October 11, 2026, from 13:30 UTC to 17:30 UTC. While we expect the impact to be minimal during this window, pipelines, workflows, jobs and notifications may briefly experience interruptions, including delays or failures in workflows and jobs starting. We will provide an update when the maintenance begins and when it has completed.",
"created_at": "2026-10-11T13:18:59.882Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-11T13:18:59.882Z",
"id": "rnmn6h9qycd6",
"incident_id": "n4hbmf9tzqzv",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-11T13:18:59.882Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Infrastructure Maintenance",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-11T17:30:07.347Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-11T13:30:00.000Z",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-11T19:30:00.000Z",
"shortlink": "https://stspg.io/77j4l074p7lv",
"started_at": "2026-10-11T13:18:59.795Z",
"status": "completed",
"updated_at": "2026-10-11T17:30:16.213Z"
},
{
"components": [
{
"created_at": "2025-05-02T13:47:11.283Z",
"description": "Public facing APIs",
"group_id": null,
"id": "t489m38xq83m",
"name": "CircleCI API",
"page_id": "6w4r0ttlx5ft",
"position": 6,
"showcase": true,
"start_date": "2025-05-02T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-14T00:22:39.024Z"
},
{
"created_at": "2017-08-11T03:29:32.538Z",
"description": null,
"group_id": null,
"id": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"page_id": "6w4r0ttlx5ft",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-14T00:23:12.766Z"
}
],
"created_at": "2026-10-09T11:33:00.290Z",
"id": "6cfhhlr4rmm0",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "Between 11:15 UTC and 23:47 UTC on October 9, customers experienced delays and failures across pipelines, workflows and jobs, along with delays in the CircleCI UI, API and notifications.\n\nFrom 11:15 to about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and Slack and email notifications were not sent. From about 13:00 to 14:40 UTC, workflows and jobs started with delays while we worked through the backlog.\n\nFrom about 15:15 to 20:48 UTC, workflows and jobs started with increasing delays, of up to about an hour, and the CircleCI UI and API fell behind by up to about 2 hours. During this time, some jobs failed to start and some in-progress workflows failed.\n\nFrom 20:48 to about 22:10 UTC, we temporarily paused starting new workflows and jobs so our systems could recover. New pipelines were accepted and queued during this time. From about 22:10 to 23:47 UTC, queued workflows and jobs resumed and were processed. Some of them had waited up to about 2 hours.\n\nThe issue has been resolved and all affected functionality has returned to normal. Usage charge data on the Project Usage page is not yet up to date and will be updated once our data backfills complete. Customers whose jobs failed during this window may rerun affected jobs. We thank you for your patience while our engineers worked on implementing a fix.",
"created_at": "2026-10-10T02:08:45.055Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-10T02:08:45.055Z",
"id": "tc0kt3x23wgk",
"incident_id": "6cfhhlr4rmm0",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-10T02:08:45.055Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Workflows and jobs are starting and running normally, and the CircleCI UI and API are up to date. We are continuing to monitor our systems closely to make sure they remain stable. \n\nWe will provide another update by 02:00 UTC.",
"created_at": "2026-10-10T01:23:13.373Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-10T01:23:13.373Z",
"id": "kx623r35xmcs",
"incident_id": "6cfhhlr4rmm0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-10T01:23:13.373Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Workflows and jobs have continued to start normally, within seconds, and the CircleCI UI and API remain up to date. Slack and email notifications delayed earlier in the incident are still being delivered. \n\nUsage charge data on the Project Usage page is stale. \n\nWe are continuing to monitor our systems closely and will provide another update by 01:15 UTC.",
"created_at": "2026-10-10T00:45:09.426Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-10T00:45:09.426Z",
"id": "1x9pgyt80fkg",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-10T00:45:09.426Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since our last update, the backlog of queued workflows and jobs has cleared, as of about 23:47 UTC. Workflows and jobs are now starting within seconds, and the CircleCI UI and API are up to date. We are monitoring our systems closely to make sure they remain stable. We will provide another update by 00:30 UTC.",
"created_at": "2026-10-09T23:59:55.337Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T23:59:55.337Z",
"id": "njsd66nx1hyw",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T23:59:55.337Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We are still working through a significant backlog of queued workflows and jobs. Wait times are improving but they remain longer than normal and some jobs have waited over an hour. The CircleCI UI and API remain up to date. Please avoid retriggering pipelines or workflows while the backlog clears, as this may create duplicates. We will provide another update by 00:00 UTC.",
"created_at": "2026-10-09T23:38:12.044Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T23:38:12.044Z",
"id": "slddcgnpwq08",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T23:38:12.044Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A large number of queued workflows and jobs have started, and the CircleCI UI and API have caught up and are showing updates normally with few delays. We are continuing to work through the remaining queued work. Jobs from the queue may have waited about an hour or more to start, and some Linux jobs are waiting up to about 10 minutes for capacity.\n\nPlease continue to avoid retriggering while queued work continues to process. \n\nWe will provide another update by 23:30 UTC.",
"created_at": "2026-10-09T23:00:06.282Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T23:00:06.282Z",
"id": "kn8lpnsykn0b",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T23:00:06.282Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since our last update, we have begun gradually resuming work, and some queued workflows and jobs are starting again. Most queued work is still waiting, and jobs that do start may have waited over an hour. We are increasing the rate carefully to avoid overloading our systems. Delayed updates continue to appear in the CircleCI UI and API.\n\nPlease continue to avoid retriggering pipelines or workflows, as this may create duplicates as queued work resumes. \n\nWe will provide another update by 23:00 UTC.",
"created_at": "2026-10-09T22:28:06.582Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T22:28:06.582Z",
"id": "jjc9y44qb8jb",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T22:28:06.582Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since our last update, most of the pipelines that were delayed earlier in the incident have now been processed, and their workflows are queued to start. New workflows and jobs remain paused while we prepare to resume work safely. Delayed updates continue to appear in the CircleCI UI and API. \n\nPlease continue to avoid retriggering pipelines or workflows, as this may create duplicates once work resumes. \n\nWe will provide another update by 22:30 UTC.",
"created_at": "2026-10-09T22:09:04.560Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T22:09:04.560Z",
"id": "4t1ztc23mt5p",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T22:09:04.560Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since about 20:48 UTC, new workflows and jobs are not starting. We have temporarily paused starting new work so that our systems can catch up on work already in progress, including delayed updates to the CircleCI UI and API. Older updates are now appearing in the UI and API as this backlog clears. New pipelines are still being accepted and queued\n\nPlease avoid retriggering pipelines or workflows during this time, as this may create duplicates once work resumes.\n\nWe will provide another update by 22:00 UTC.",
"created_at": "2026-10-09T21:22:37.747Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T21:22:37.747Z",
"id": "7d56g12hwjlm",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T21:22:37.747Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Jobs are starting faster and fewer are failing: most jobs are now starting within about 4 minutes, down from about 6 minutes, and the longest waits have dropped from about 30 minutes to about 20 minutes. \n\nPipelines from earlier in the incident are now being worked through. \n\nThe CircleCI UI is still behind, and some older updates may take longer to appear. Jobs may already have run even if the UI does not show them yet, so please avoid retriggering workflows that don't appear in the UI.\n\nWe will provide another update by 20:30 UTC.",
"created_at": "2026-10-09T20:06:36.341Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T20:06:36.341Z",
"id": "vxwn1x5xj418",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T20:06:36.341Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since our last update, jobs are starting faster: most are now starting within about 6 minutes, down from about 8 minutes. The CircleCI UI is catching up but still about an hour behind. New pipelines are being processed as they arrive. \n\nJobs may already have run even if the UI does not show them yet, so please avoid retriggering workflows that don't appear in the UI. \n\nWe will provide another update by 20:00 UTC.",
"created_at": "2026-10-09T19:29:54.594Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T19:29:54.594Z",
"id": "zznj02llzfvt",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T19:29:54.594Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "What's changed since our last update\n\n* Jobs are starting faster. Most are now starting within about 8 minutes, down from about 15 minutes, and the longest waits have dropped from over an hour to about 45 minutes.\n* The CircleCI UI is beginning to catch up, but it is still behind. Workflows and jobs may already have run, or may be running, even if the UI shows them as not started or does not show them at all.\n* Pipelines are being processed faster, but new pipelines are still delayed.\n\nStill ongoing\n* Some workflows and jobs are still failing.\n* Please avoid retriggering workflows that don't yet appear in the UI, as this may create duplicates.\n* Usage charge data on the Project Usage page is stale. \n\nNext update\nWe will provide another update by 19:30 UTC.",
"created_at": "2026-10-09T18:57:01.746Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T18:57:01.746Z",
"id": "phlrcb2qhh92",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T18:57:01.746Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "What's changed since our last update\n\n* The CircleCI UI is about an hour behind, and up to about 90 minutes for some updates. Workflows and jobs may already have run, or may be running, even if the UI shows them as not started or does not show them at all.\n* Jobs that are running are starting faster. Most are now starting within about 15 minutes, down from about 28 minutes, though some are still waiting over an hour.\n* New pipelines are increasingly delayed.\n* Some workflows are still failing. Workflows that could not be processed in time are being marked as failed.\n* Please avoid retriggering workflows that don't yet appear in the UI, as this may create duplicates.\n\nStill ongoing: \nUsage charge data on the Project Usage page is stale. \n\nNext update\nWe will provide another update by 19:00 UTC.",
"created_at": "2026-10-09T18:32:52.602Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T18:32:52.602Z",
"id": "t14nr135wscg",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T18:32:52.602Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog. Since about 15:15 UTC, workflows and jobs have again been starting with delays.\n\nWhat’s impacted\nCustomers triggering pipelines and running workflows and jobs, customers viewing pipelines, workflows and jobs in the CircleCI UI, and customers viewing usage data on the Project Usage page.\n\nWhat to expect\n- Since about 17:00 UTC, we are seeing an increase in delays. Most jobs are now taking about 25 minutes to start, and some are waiting up to about an hour.\n- Since about 17:15 UTC, some in-progress workflows have failed and a small number of jobs are failing to start.\n- New pipelines are being queued and are taking longer to start.\n- The CircleCI UI is more than an hour behind. New workflows and status changes may take that long to appear.\n- Please avoid retriggering workflows that don't yet appear in the UI, as this may create duplicates.\n- Usage charge data on the Project Usage page is currently stale. \n\nOur engineers continue to work to resolve the issue at hand. \n\nNext update\nWe will provide another update by 18:30 UTC.",
"created_at": "2026-10-09T18:08:03.166Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T18:08:03.166Z",
"id": "rr745bm3mdpr",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T18:08:03.166Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog. Since about 15:15 UTC, workflows and jobs have again been starting with delays.\n\nWhat’s impacted\nCustomers triggering pipelines and running workflows and jobs, customers viewing pipelines, workflows and jobs in the CircleCI UI, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n\n- Since about 17:00 UTC, delays have increased significantly. Most jobs are now taking about 12 minutes to start, and some are waiting up to about 30 minutes.\n- Since about 17:15 UTC, some jobs are failing to start. \n- Pipelines are being processed more slowly, so some may take longer to start. \n- The CircleCI UI is more than an hour behind. New workflows and status changes may take that long to appear. \n- Please avoid retriggering workflows that don’t yet appear in the UI, as this may create duplicates.\n- Usage charge data on the Project Usage page is currently stale. \n\nThank you for your patience while our engineers work to resolve this.\n\nNext update\nWe will provide another update by 18:00 UTC.",
"created_at": "2026-10-09T17:34:01.223Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T17:34:01.223Z",
"id": "ywrkgdr5qxpv",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T17:34:01.223Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog. Since about 15:15 UTC, workflows and jobs have again been starting with delays.\n\nWhat’s impacted\nCustomers running workflows and jobs, customers viewing pipelines, workflows and jobs in the CircleCI UI, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n\n- Delays starting workflows and jobs are holding steady. Most jobs are starting within about 4 minutes, and some are waiting up to about 8 minutes, down from about 25 minutes.\n- The CircleCI UI remains about 20 minutes behind on average, and some updates are taking longer. New workflows and status changes may take that long to appear.\n- Please avoid retriggering workflows that don’t yet appear in the UI, as this may create duplicates.\n- Usage charge data on the Project Usage page is currently stale. \n\nThank you for your patience while our engineers work to reduce these delays.\n\nNext update\nWe will provide another update by 17:30 UTC.",
"created_at": "2026-10-09T17:06:30.170Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T17:06:30.170Z",
"id": "m77xw2dm51zs",
"incident_id": "6cfhhlr4rmm0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T17:06:30.170Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog. Since about 15:15 UTC, workflows and jobs have again been starting with delays.\n\nWhat's impacted\nCustomers running workflows and jobs, customers viewing pipelines, workflows and jobs in the CircleCI UI, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n\n- Delays starting workflows and jobs have stopped increasing and are beginning to ease. Most jobs are starting within about 4 minutes, and some are waiting up to about 25 minutes.\n- The CircleCI UI is now about 20 minutes behind, down from about 30 minutes. New workflows and status changes may take that long to appear.\n- Please avoid retriggering workflows that don't yet appear in the UI, as this may create duplicates.\n- Usage charge data on the Project Usage page is currently stale. \n\nThank you for your patience while our engineers work to reduce these delays.\n\nNext update\nWe will provide another update by 17:00 UTC.",
"created_at": "2026-10-09T16:32:53.245Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T16:32:53.245Z",
"id": "zp01djn036zf",
"incident_id": "6cfhhlr4rmm0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T16:32:53.245Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog.\n\nWhat's impacted\nCustomers running workflows and jobs, customers viewing pipelines, workflows and jobs in the CircleCI UI, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n- Since about 15:15 UTC, workflows and jobs have been starting with increasing delays. Most jobs are currently starting within about 4 minutes, and some are waiting up to about 20 minutes.\n- The CircleCI UI remains about 30 minutes behind on average, and some updates are taking longer. New workflows and status changes may take that long to appear.\n- Please avoid retriggering workflows that don't yet appear in the UI, as this may create duplicates.\n- Usage charge data on the Project Usage page is currently stale. \n\nThank you for your patience while our engineers work to reduce these delays.\n\nNext update\nWe will provide another update by 16:30 UTC.",
"created_at": "2026-10-09T16:02:42.495Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T16:02:42.495Z",
"id": "ymxbztqs0prv",
"incident_id": "6cfhhlr4rmm0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T16:02:42.495Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog.\n\nWhat's impacted\nCustomers viewing pipelines, workflows and jobs in the CircleCI UI, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n- Workflows and jobs are running on time.\n- The CircleCI UI is about 30 minutes behind. New workflows and status changes may take that long to appear.\n- Our engineers are actively working to speed up processing so the UI can catch up.\n- Please avoid retriggering workflows that don't yet appear in the UI, as this may create duplicates.\n- Usage charge data on the Project Usage page is currently stale.\n\nThank you for your patience while our engineers work to reduce this delay.\n\nNext update\nWe will provide another update by 16:00 UTC.",
"created_at": "2026-10-09T15:32:03.279Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T15:32:03.279Z",
"id": "f9jb7ggjpsxy",
"incident_id": "6cfhhlr4rmm0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T15:32:03.279Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog.\n\nWhat’s impacted\nCustomers running workflows and jobs, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n\n- The backlog of pipelines and workflows from the incident has cleared.\n- Since about 14:45 UTC, most workflows and jobs are starting within about 15 seconds, and start times are continuing to return to normal.\n- Usage charge data on the Project Usage page is currently stale. \n\nThank you for your patience while our engineers monitor the recovery.\n\nNext update\nWe will provide another update by 15:20 UTC.",
"created_at": "2026-10-09T14:54:02.940Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T14:54:02.940Z",
"id": "bctws6py234x",
"incident_id": "6cfhhlr4rmm0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T14:54:02.940Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent. Between about 13:00 UTC and 14:40 UTC, workflows and jobs started with delays while we worked through the backlog.\n\nWhat’s impacted\nCustomers running workflows and jobs, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n\n- The backlog of workflows from pipelines triggered during the incident cleared at about 14:40 UTC.\n- Most jobs are now starting within about 30 seconds, and some are waiting up to about 3 minutes.\n- Usage charge data on the Project Usage page is currently stale. \n\nThank you for your patience while our engineers continue to work on the underlying issue.\n\nNext update\nWe will provide another update by 15:10 UTC.",
"created_at": "2026-10-09T14:43:52.603Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T14:43:52.603Z",
"id": "6l23l9vg00p7",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T14:43:52.603Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent.\n\nWhat's impacted\nCustomers running workflows and jobs, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n\n- Delays starting workflows and jobs are improving. Most jobs are now starting within about 3 minutes, down from about 10 minutes at 13:50 UTC.\n- Some workflows from pipelines triggered during the incident are still waiting up to about 40 minutes to start.\n- Working through the backlog is taking slightly longer than expected. We now expect to clear it by about 14:45 UTC. \n- Workflows from pipelines triggered during the incident are still waiting to start, so retriggering may result in duplicate workflows.\n- Usage charge data on the Project Usage page is currently stale. \n\nThank you for your patience while our engineers work to resolve this.\n\nNext update\nWe will provide another update by 15:00 UTC.",
"created_at": "2026-10-09T14:29:58.944Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T14:29:58.944Z",
"id": "76l1sfjdf53g",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T14:29:58.944Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Since 11:15 UTC, customers have experienced disruption to pipelines and notifications. Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent.\n\nWhat's impacted\nCustomers running workflows and jobs, and customers viewing usage data on the Project Usage page.\n\nWhat can you expect\n- Pipelines are being processed again, including those triggered during the incident.\n- Since about 13:25 UTC, workflows and jobs have been starting with increasing delays as we work through the backlog. Most jobs are currently starting within about 10 minutes, and some are waiting up to about 25 minutes.\n- We expect to work through this backlog by about 14:35 UTC.\n- Workflows from pipelines triggered during the incident are still waiting to start, so retriggering may result in duplicate workflows. \n- Usage charge data on the Project Usage page is currently stale.\n\nThank you for your patience while our engineers work to resolve this.\n\nNext update\nWe will provide another update by 14:30 UTC.",
"created_at": "2026-10-09T14:00:23.628Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T14:00:23.628Z",
"id": "nd0zqb74k0hw",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T14:00:23.628Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Between 11:15 UTC and about 13:00 UTC, pipelines were created but not processed, and outbound webhooks and transactional notifications (Slack and email) were not sent.\n\nWhat's impacted:\nCustomers triggering pipelines, and customers relying on outbound webhooks and email and Slack notifications.\n\nWhat can you expect:\n- Since about 13:00 UTC, pipelines are being processed again and jobs are starting.\n- Most jobs are currently starting within about 4 minutes, and some are waiting up to about 10 minutes. Pipelines triggered during the incident are still being processed and may take longer to start.\n- Outbound webhooks and notifications have resumed. \n- Pipelines are still being processed from the backlog, so retriggering may result in duplicate pipelines.\n\nThank you for your patience while our engineers work to resolve this.\n\nNext update:\nWe will provide another update by 14:00 UTC.",
"created_at": "2026-10-09T13:30:31.975Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T13:30:31.975Z",
"id": "88ds0hhr371x",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T13:30:31.975Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "What's impacted: \nCustomers triggering pipelines, and customers relying on outbound webhooks and email and Slack notifications.\n\nWhat can you expect\nSince 11:15 UTC:\n - Pipelines are still being created while this is ongoing but not yet processed, so retriggering may result in duplicate pipelines.\n - Outbound webhooks and transactional notifications (Slack and email) are not being sent.\n - Pipeline statuses may not be visible. \n - Usage charge data on the Project Usage page will be stale. \n\nThank you for your patience while our engineers work to resolve this.\n\nNext update: \nWe will provide another update by 13:30 UTC.",
"created_at": "2026-10-09T13:00:40.161Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T13:00:40.161Z",
"id": "yjnhth0t8yh8",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T13:00:40.161Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sw1j7xsbdlk4",
"name": "CircleCI UI",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "t489m38xq83m",
"name": "CircleCI API",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "There is a delay in our data infrastructure causing delays before build data and some notifications become visible. We are working on a mitigation.",
"created_at": "2026-10-09T11:33:00.542Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T11:33:00.542Z",
"id": "sfkwmky10ly8",
"incident_id": "6cfhhlr4rmm0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T11:33:00.542Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-09T14:54:02.940Z",
"name": "Delay in data appearing in UI and API",
"page_id": "6w4r0ttlx5ft",
"postmortem_body": "## Summary\n\nBetween 11:15 UTC and 23:47 UTC on October 9, CircleCI customers experienced hours-long delays and failures across pipelines, workflows and jobs, along with delays in the CircleCI UI, API and notifications.\n\n* From 11:15 to 13:00 UTC, pipelines were created but not processed, and outbound webhooks and Slack and email notifications were not sent.\n* From 13:00 to 14:40 UTC, workflows and jobs started with delays while we worked through the backlog.\n* From 15:15 to 20:48 UTC, workflows and jobs started with increasing delays, of up to about an hour, and the CircleCI UI and API fell behind by up to about 2 hours. During this time, some jobs failed to start and some in-progress workflows failed.\n* From 20:48 to 22:10 UTC, we temporarily paused starting new workflows and jobs so our systems could recover. New pipelines were accepted and queued during this time.\n* From 22:10 to 23:47 UTC, queued workflows and jobs resumed and were successfully processed. Some of them had waited up to about 2 hours.\n* By 23:47, the incident was mitigated.\n\nThe status page for this incident can be found at [October 9 - Delay in data appearing in UI and API](https://status.circleci.com/incidents/6cfhhlr4rmm0). Customers experienced a similar incident earlier in the week at [October 6 - Delays in pipelines, workflows, UI data, and notifications](https://status.circleci.com/incidents/z4hbq8h49j8c).\n\nMultiple CircleCI services use Temporal backed by Postgres as a reliable queueing mechanism, and both incidents had the same cause: the rate of writes to Temporal's Postgres instance were faster than Postgres’s transaction log could archive those writes. In both cases, this caused multi-hour delays in processing both pipelines and status notifications.\n\nWe evaluated several mitigations, then paused new workflow starts so the system could recover safely, and ramped back up gradually. We then throttled work back up until our systems returned to normal.\n\nTo permanently resolve the bottleneck, we have split work across multiple Temporal instances, each with its own independent database and worker clusters, and we load-balance across them. We had already scheduled infrastructure work starting the week of October 12 that would have removed this exact bottleneck and prevented these incidents. So starting 19:11 on October 9 our engineering team implemented, tested, and rolled out a simpler version of these changes within a few hours. We further added new alerting and tuned the underlying databases for better performance under high load.\n\nCustomers whose jobs failed during this window may rerun them.\n\n## Background\n\nTo run customer jobs, CircleCI executes a series of steps: we accept customer pipelines; we assign jobs for those pipelines to workers; we deliver status updates for those pipelines and jobs; and we charge customers for the credits those jobs consume. Each of these steps is implemented by an independent service, with reliable queues in between.\n\nHistorically we have implemented these reliable queues with RabbitMQ on top of durable storage, with exponential backoff and dead letter queues for reliability. Over the last year, we have increasingly reimplemented our reliable queues on [Temporal](https://temporal.io/), which provides far better durability and reliability guarantees over a simpler and more powerful programming model. Our build service \\(which turns a pipeline config into an executable plan\\), our workflow orchestrator service \\(which distributes tasks onto workers\\), our notification service \\(which delivers status updates\\), and our charging service \\(which charges customers for credits\\) all now use Temporal.\n\nTemporal stores the state for its workflows and activities \\(called “history”\\) in a database, and this is the only component in a Temporal instance that is synchronous and stateful. As a consequence, a Temporal instance’s ability to accept and execute workflows is dependent on the write rate of its backing database. We have also configured Temporal to delete completed workflows and activities from its history 3 days after they complete.\n\nAt the time of this incident, we used a single clustered Temporal instance backed by a Multi-AZ Postgres RDS instance for all of these reliable queues. While a Multi-AZ RDS instance is resilient to zone failures, it maintains database consistency via a single write-ahead-log \\(WAL\\). At the time we had configured our Postgres instance to store its WAL on a Dedicated Log Volume \\(DLV\\) so log writes did not compete with data writes. The database archives the log continuously to backup storage, and it can only reuse space on the volume after the log has been archived. If the database writes logs faster than it archives, the volume fills and Postgres has no space to the write its WAL log. Then the database stops accepting incoming writes and goes entirely read-only. An RDS DLV is limited to 1TB of storage and 3000 IOPS and per AWS these limits are hardcoded and cannot be increased.\n\n## What Happened\n\n_\\(All times UTC\\)_\n\nOne of the contributing factors to the October 6 incident was that we were processing a high rate of Temporal history deletions at the same time as incoming work, which in turn put pressure on the backing database. To resolve the October 6 incident, we changed cleanup settings on Temporal’s largest tables and moved the transaction log back to its dedicated volume. These changes eliminated the major source of transaction log writes, but did not consider the additional log writes from scheduled deletions. Because we have configured Temporal to delete completed workflows and activities from its history 3 days after they complete, the October 6 history was scheduled to be deleted on October 9.\n\nStarting at 08:00 on October 9, Temporal began processing those scheduled deletions, and as a consequence free space on the Postgres DLV began dropping precipitously, from 0.8TB at 08:00 to 0.1TB by 10:10. Our monitoring alerted at 10:28 and we declared an incident at 10:31. While our infrastructure team was investigating, at 11:15 the DLV ran out of space and the Postgres instance went read-only. This stopped all Temporal processing in its tracks.\n\nAt 11:42, we restarted Postgres to move the WAL log from the DLV onto the data volume and use the spare disk space and IOPS there. This maneuver had mitigated the incident on October 6, and we knew it would take roughly 60 minutes.\n\nWhile the database was restarting, we began implementing several load-shedding techniques upstream of Temporal:\n\n* We began investigating whether we should shed incoming workloads altogether to keep the rest moving, but ultimately decided that this would provide no value.\n* At 12:29, we flagged off credit charging updates, which represented 12% of Temporal activities. This left the Pipeline Usage page with stale charging data, until we would backfill it, but would not lose data or prevent pipelines from running.\n\nAt 12:59, the database was back up and began processing work that had queued since 11:15 and by 14:40, this backlog had cleared and jobs were starting on time. However, by 15:15, delays returned as database’s routing cleanup had fallen far behind on the tables Temporal uses to track queued and scheduled work. Those tables normally hold tens of thousands of deleted rows awaiting cleanup, but by late afternoon, they held tens of millions of rows. Every Temporal operation had to read past those rows, so the database spent much more processing time on each one and Temporal ran slower. This resulted in customers continuing to see hour-long delays in starting pipelines and in status updates.\n\nFor the next several hours we attempted various mitigations, including:\n\n* Manipulating Temporal configuration values to reduce database pressure. This had minimal effect.\n* Running manual vacuum jobs to clean up dead rows in Postgres. This ended up competing for CPU with queries themselves, and was ultimately counterproductive. We cancelled them at 17:45 to relieve the pressure on the database load.\n* We began trying to move all of the notification usecases to a separate Temporal instance. This represented 18% of Temporal activities. We successfully finished moving notifications by 17:07.\n* At 17:16, we successfully implemented an optimization in our representation of pipeline state, which reduced it by 5%. That in turn reduced the size of database writes and therefore data pressure on the Postgres WAL log.\n\nNone of these fully mitigated the issue.\n\nBy 20:48, we realized that Postgres was now thrashing on database CPU and was no longer bottlenecked on log writes. We further realized that our upstream systems were both submitting new work and also retrying work that had earlier been submitted and failed. Recognizing that we were in a classic retry storm, we paused all incoming and retry work and let it queue up, in order to allow the database to fully complete in-process work. We then slowly throttled up queued work from 22:07 to keep the database at a sustainable load.\n\nThis fully succeeded by 23:47 when all queued work had started, and we closed the incident at 02:08 on October 10.\n\n## Future Prevention and Process Improvement\n\n### Multiple Temporal Instances\n\nIn parallel with incident mitigation and resolution, starting at 19:11 on October 9, we also began standing up multiple sharded Temporal instances and implementing the ability to load-balance incoming work between them.\n\nEach of the multiple Temporal instances maintains its own independent database and worker clusters, and so is isolated from the others. As a postmortem action item from the October 6 incident, we had already scheduled infrastructure work that would have removed the bottleneck of a single Temporal instance and Postgres database, starting October 12 and expecting to take 2 weeks. On October 9, we realized that this work simply could not wait, so our engineering team implemented, tested, and rolled out a simpler version of the planned changes within a few hours.\n\nThese changes completed successfully at 01:45 on October 10.\n\n**In addition, we are changing how we store and monitor transaction logs.** Our Temporal database no longer uses the fixed-size log volume. We are adding alerts on how far log archiving falls behind and how fast the database writes logs, so we can act hours earlier than we did on October 9.\n\n**We are also revisiting the Temporal's deletion schedule.** Temporal deletes finished work a few days after it runs, so a busy day returns as database load later in the week. We are changing when and how quickly those deletions run.\n\n**We are changing how our systems retry when Temporal is slow.** On October 9, our systems retried failed work against Temporal and added to its load. As we committed after the October 6 incident, we are changing our systems to wait for Temporal to recover so work is not dropped.\n\n## Conclusion\n\nCustomer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. We thank our customers for their patience while our team worked to resolve these incidents.\n\nPlease reach out to our support team with any questions or concerns.",
"postmortem_body_last_updated_at": "2026-10-12T22:34:58.695Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-12T22:34:58.695Z",
"reminder_intervals": null,
"resolved_at": "2026-10-10T02:08:45.055Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/hkfkwz292bz8",
"started_at": "2026-10-09T11:33:00.284Z",
"status": "resolved",
"updated_at": "2026-10-12T22:34:58.722Z"
}
],
"page": {
"id": "6w4r0ttlx5ft",
"name": "CircleCI",
"url": "https://status.circleci.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}