Snapshot 11359
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"incidents": [
{
"created_at": "2026-09-09T21:30:35Z",
"id": "01M2418VWM3N4EZ9A8XF2A8XN8",
"impact": "major",
"incident_updates": [
{
"body": "The issue causing slow or failed data loading in our US-hosted service has been resolved. We're sorry for the disruption.",
"created_at": "2026-09-09T22:10:23Z",
"display_at": "2026-09-09T22:10:23Z",
"id": "01M243HQRM6EN39PCBJ8J8EJD9",
"incident_id": "01M2418VWM3N4EZ9A8XF2A8XN8",
"status": "resolved",
"updated_at": "2026-09-09T22:10:23Z"
},
{
"body": "We've applied a mitigation for the data-loading issue affecting our US-hosted service and are monitoring performance. We'll share another update once we've confirmed stability.",
"created_at": "2026-09-09T21:34:24Z",
"display_at": "2026-09-09T21:34:24Z",
"id": "01M241FV7V9Z3MW4STHDFM3XEZ",
"incident_id": "01M2418VWM3N4EZ9A8XF2A8XN8",
"status": "monitoring",
"updated_at": "2026-09-09T21:34:24Z"
},
{
"body": "We're investigating slow or failed data loading affecting some customers on our US-hosted service. Affected pages may load slowly, hang, or show errors.",
"created_at": "2026-09-09T21:30:35Z",
"display_at": "2026-09-09T21:30:35Z",
"id": "01M2418VWMRCNFW3V6XK9FN213",
"incident_id": "01M2418VWM3N4EZ9A8XF2A8XN8",
"status": "investigating",
"updated_at": "2026-09-09T21:30:35Z"
}
],
"name": "Slow or failed data loading in US region",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-09-09T22:10:23Z",
"status": "resolved",
"updated_at": "2026-09-09T22:10:23Z"
},
{
"created_at": "2026-09-04T15:19:49Z",
"id": "01M1PG2CDCKSZNZ1PS1RBDQ6FC",
"impact": "major",
"incident_updates": [
{
"body": "The issue causing elevated errors in the Braintrust control plane APIs has been resolved.",
"created_at": "2026-09-04T15:52:23Z",
"display_at": "2026-09-04T15:52:23Z",
"id": "01M1PHY05GPEDTYY7B96W1GFPR",
"incident_id": "01M1PG2CDCKSZNZ1PS1RBDQ6FC",
"status": "resolved",
"updated_at": "2026-09-04T15:52:23Z"
},
{
"body": "We have mitigated the issue causing elevated errors in the Braintrust control plane APIs, we are continuing to monitor closely.",
"created_at": "2026-09-04T15:34:05Z",
"display_at": "2026-09-04T15:34:05Z",
"id": "01M1PGWG89BA10Q0HS7G9FWRP3",
"incident_id": "01M1PG2CDCKSZNZ1PS1RBDQ6FC",
"status": "monitoring",
"updated_at": "2026-09-04T15:34:05Z"
},
{
"body": "We are investigating elevated errors affecting the Braintrust control plane APIs. Some users may experience failures when accessing projects, prompts, settings, or creating new resources.",
"created_at": "2026-09-04T15:19:49Z",
"display_at": "2026-09-04T15:19:49Z",
"id": "01M1PG2CDCKG5AYR6RJ45H1XRY",
"incident_id": "01M1PG2CDCKSZNZ1PS1RBDQ6FC",
"status": "investigating",
"updated_at": "2026-09-04T15:19:49Z"
}
],
"name": "Elevated 5XXs on Control Plane API",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-09-04T15:52:23Z",
"status": "resolved",
"updated_at": "2026-09-04T15:52:23Z"
},
{
"created_at": "2026-08-28T20:38:26Z",
"id": "01M151GQYSP7EFVRA513MFC98A",
"impact": "major",
"incident_updates": [
{
"body": "The load spike has subsided and 5xxs are mitigated",
"created_at": "2026-08-28T20:38:26Z",
"display_at": "2026-08-28T18:54:00Z",
"id": "01M151GQYSTE6AV079MJACK53F",
"incident_id": "01M151GQYSP7EFVRA513MFC98A",
"status": "resolved",
"updated_at": "2026-08-28T20:38:26Z"
},
{
"body": "The gateway encountered a load spike leading to 5xx errors.",
"created_at": "2026-08-28T20:38:26Z",
"display_at": "2026-08-28T18:41:00Z",
"id": "01M151GQYSF0CY51C95KZGZ853",
"incident_id": "01M151GQYSP7EFVRA513MFC98A",
"status": "investigating",
"updated_at": "2026-08-28T20:38:26Z"
}
],
"name": "Gateway 5xxs",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-08-28T18:54:00Z",
"status": "resolved",
"updated_at": "2026-08-28T20:38:26Z"
},
{
"created_at": "2026-08-28T18:24:47Z",
"id": "01M14SW1BKRQY52FMTNY410RVW",
"impact": "major",
"incident_updates": [
{
"body": "The deploy was rolled back and gateway 5xxs were mitigated.",
"created_at": "2026-08-28T18:24:48Z",
"display_at": "2026-08-28T15:55:00Z",
"id": "01M14SW1BKC92DZY4D97RWXP86",
"incident_id": "01M14SW1BKRQY52FMTNY410RVW",
"status": "resolved",
"updated_at": "2026-08-28T18:24:48Z"
},
{
"body": "A fresh gateway deploy was attempted and led to 5xx errors, and then was immediately rolled back.",
"created_at": "2026-08-28T18:24:48Z",
"display_at": "2026-08-28T15:47:00Z",
"id": "01M14SW1BKZ6MYEEMN0AQZP1BX",
"incident_id": "01M14SW1BKRQY52FMTNY410RVW",
"status": "investigating",
"updated_at": "2026-08-28T18:24:48Z"
}
],
"name": "Gateway 5xxs",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-08-28T15:55:00Z",
"status": "resolved",
"updated_at": "2026-08-28T18:24:47Z"
},
{
"created_at": "2026-08-28T18:21:24Z",
"id": "01M14SNT9T27W3CCVQ93MZ685B",
"impact": "major",
"incident_updates": [
{
"body": "The faulty deploy was rolled back and 5xxs were mitigated.",
"created_at": "2026-08-28T18:21:24Z",
"display_at": "2026-08-27T13:15:00Z",
"id": "01M14SNT9T3BWM5A1C3M6GP81B",
"incident_id": "01M14SNT9T27W3CCVQ93MZ685B",
"status": "resolved",
"updated_at": "2026-08-28T18:21:24Z"
},
{
"body": "A deployment caused the gateway to begin returning 5xxs for a portion of traffic.",
"created_at": "2026-08-28T18:21:24Z",
"display_at": "2026-08-27T01:50:00Z",
"id": "01M14SNT9TYN5MKE20EF22B2ZM",
"incident_id": "01M14SNT9T27W3CCVQ93MZ685B",
"status": "investigating",
"updated_at": "2026-08-28T18:21:24Z"
}
],
"name": "Gateway 5xxs",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-08-27T13:15:00Z",
"status": "resolved",
"updated_at": "2026-08-28T18:21:24Z"
},
{
"created_at": "2026-08-19T18:01:58Z",
"id": "01M0DJZRPAD4YF0EHTHQC2ZSSS",
"impact": "none",
"incident_updates": [
{
"body": "The upstream APT package mirror in AWS us-east-1 has recovered and we're no longer seeing impact when starting new brainstore instances.",
"created_at": "2026-08-19T22:35:25Z",
"display_at": "2026-08-19T22:35:25Z",
"id": "01M0E2MFKZ7518KQ0Y55YPTWK9",
"incident_id": "01M0DJZRPAD4YF0EHTHQC2ZSSS",
"status": "resolved",
"updated_at": "2026-08-19T22:35:25Z"
},
{
"body": "The upstream APT package mirror on AWS us-east-1 is degraded. **There is currently no downtime** and we are looking into alternative package mirrors in the meantime.\n\nPlease note that for **both the Braintrust-hosted data plane and self-hosted data planes:**\nEC2 instance replacement due to healthcheck failures or deploys may result in failed instance initialization and downtime until the upstream issue is resolved.\n",
"created_at": "2026-08-19T18:01:58Z",
"display_at": "2026-08-19T18:01:58Z",
"id": "01M0DJZRPATDNMGHF3MD54SN20",
"incident_id": "01M0DJZRPAD4YF0EHTHQC2ZSSS",
"status": "monitoring",
"updated_at": "2026-08-19T18:01:58Z"
}
],
"name": "Upstream package mirror is degraded, Brainstore is not currently affected",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-08-19T22:35:25Z",
"status": "resolved",
"updated_at": "2026-08-19T22:35:25Z"
},
{
"created_at": "2026-08-03T19:52:06Z",
"id": "01KZ4JXXZ535GBFN2PC70ZZMVM",
"impact": "major",
"incident_updates": [
{
"body": "The issue has been resolved and service has remained stable following the rollback. We will continue to monitor internally, but this incident is now considered resolved.",
"created_at": "2026-08-03T20:19:15Z",
"display_at": "2026-08-03T20:19:15Z",
"id": "01KZ4MFMYQ98NMBR1MR51PWG5Z",
"incident_id": "01KZ4JXXZ535GBFN2PC70ZZMVM",
"status": "resolved",
"updated_at": "2026-08-03T20:19:15Z"
},
{
"body": "The issue has been resolved. We are monitoring the service to confirm normal operation.",
"created_at": "2026-08-03T20:06:05Z",
"display_at": "2026-08-03T20:06:05Z",
"id": "01KZ4KQHR20RHFF3ZY3AWNEC7F",
"incident_id": "01KZ4JXXZ535GBFN2PC70ZZMVM",
"status": "monitoring",
"updated_at": "2026-08-03T20:06:05Z"
},
{
"body": "We identified an issue causing 5XXs introduced by a recent Gateway deployment and are currently rolling it back.",
"created_at": "2026-08-03T19:52:06Z",
"display_at": "2026-08-03T19:52:06Z",
"id": "01KZ4JXXZ5FAMDZE20WH1FNWKM",
"incident_id": "01KZ4JXXZ535GBFN2PC70ZZMVM",
"status": "identified",
"updated_at": "2026-08-03T19:52:06Z"
}
],
"name": "Gateway 5XXs",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-08-03T20:19:15Z",
"status": "resolved",
"updated_at": "2026-08-03T20:19:15Z"
},
{
"created_at": "2026-07-16T08:32:44Z",
"id": "01KXN0X1983TEAPFGBF40FGSA6",
"impact": "critical",
"incident_updates": [
{
"body": "We have reverted to an older configuration to avoid an AWS outage with Cloudfront.",
"created_at": "2026-07-16T09:01:09Z",
"display_at": "2026-07-16T09:01:09Z",
"id": "01KXN2H2W8N97YTRPZDY8R19PJ",
"incident_id": "01KXN0X1983TEAPFGBF40FGSA6",
"status": "resolved",
"updated_at": "2026-07-16T09:01:09Z"
},
{
"body": "The hosted EU data plane is currently experiencing a very high rate of 504 errors.",
"created_at": "2026-07-16T08:32:44Z",
"display_at": "2026-07-16T08:32:44Z",
"id": "01KXN0X198YWWTQF3B2E1Q62DK",
"incident_id": "01KXN0X1983TEAPFGBF40FGSA6",
"status": "investigating",
"updated_at": "2026-07-16T08:32:44Z"
}
],
"name": "High 504 error rate in EU data plane",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-07-16T09:01:09Z",
"status": "resolved",
"updated_at": "2026-07-16T09:01:09Z"
},
{
"created_at": "2026-06-12T18:38:22Z",
"id": "01KTYJ1HEN1HF53DVZDVY826DE",
"impact": "critical",
"incident_updates": [
{
"body": "A configuration change was identified as the root cause and reverted.",
"created_at": "2026-06-12T18:44:50Z",
"display_at": "2026-06-12T18:26:00Z",
"id": "01KTYJDD31Z1MPWZHKMESB95AE",
"incident_id": "01KTYJ1HEN1HF53DVZDVY826DE",
"status": "resolved",
"updated_at": "2026-06-12T18:47:32Z"
},
{
"body": "Our US data plane is encountering issues. ",
"created_at": "2026-06-12T18:43:51Z",
"display_at": "2026-06-12T18:16:00Z",
"id": "01KTYJBJW12QRRECWH6J3MGG51",
"incident_id": "01KTYJ1HEN1HF53DVZDVY826DE",
"status": "investigating",
"updated_at": "2026-06-12T18:47:04Z"
}
],
"name": "US Data plane outage",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-06-12T18:26:00Z",
"status": "resolved",
"updated_at": "2026-06-12T18:44:50Z"
},
{
"created_at": "2026-06-04T23:06:33Z",
"id": "01KTAE6V88PYB4JAJWC3GD4XA5",
"impact": "major",
"incident_updates": [
{
"body": "We have pushed a fix and the loading issues have been remediated. ",
"created_at": "2026-06-04T23:09:57Z",
"display_at": "2026-06-04T23:07:00Z",
"id": "01KTAED2CMFVQT5PGZ26F4TMVM",
"incident_id": "01KTAE6V88PYB4JAJWC3GD4XA5",
"status": "resolved",
"updated_at": "2026-06-04T23:15:14Z"
},
{
"body": "We're experiencing issues loading logs and some other pages in Braintrust. Our team is actively investigating and working to resolve. ",
"created_at": "2026-06-04T23:06:33Z",
"display_at": "2026-06-04T22:51:00Z",
"id": "01KTAE6V88D4AMJ7PAPZVPKSQN",
"incident_id": "01KTAE6V88PYB4JAJWC3GD4XA5",
"status": "investigating",
"updated_at": "2026-06-04T23:08:44Z"
}
],
"name": "Issues loading logs page",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-06-04T23:07:00Z",
"status": "resolved",
"updated_at": "2026-06-04T23:09:57Z"
},
{
"created_at": "2026-05-26T21:33:09Z",
"id": "01KSK39C10462RYGADSHB88DRM",
"impact": "major",
"incident_updates": [
{
"body": "Our auth provider has recovered.",
"created_at": "2026-06-02T06:08:17Z",
"display_at": "2026-05-26T22:11:00Z",
"id": "01KT3F4XPD7TG77JNH6FT4KDTQ",
"incident_id": "01KSK39C10462RYGADSHB88DRM",
"status": "resolved",
"updated_at": "2026-06-02T06:09:11Z"
},
{
"body": "Our upstream auth provider is having issues. Fresh logins to Braintrust will fail with an error.",
"created_at": "2026-05-26T21:33:09Z",
"display_at": "2026-05-26T21:33:09Z",
"id": "01KSK39C10VRSNH0J8XV6MFCYK",
"incident_id": "01KSK39C10462RYGADSHB88DRM",
"status": "investigating",
"updated_at": "2026-05-26T21:33:09Z"
}
],
"name": "Errors logging into Braintrust",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-05-26T22:11:00Z",
"status": "resolved",
"updated_at": "2026-06-02T06:08:17Z"
},
{
"created_at": "2026-06-04T01:23:21Z",
"id": "01KT83MKXJNJ4TWSD1M5E9R9EQ",
"impact": "minor",
"incident_updates": [
{
"body": "We determined the issue was caused by a role used by the automation service losing required permissions following a configuration change. The issue was reported on May 15 at approximately 1:00 PM PT and resolved on May 16 at 11:50 AM PT. To help prevent similar incidents, we have improved our alerting and added audit logging for role and permission changes.",
"created_at": "2026-06-04T01:23:21Z",
"display_at": "2026-05-16T18:50:00Z",
"id": "01KT83MKXJN18VXHZEH01RNV64",
"incident_id": "01KT83MKXJNJ4TWSD1M5E9R9EQ",
"status": "resolved",
"updated_at": "2026-06-04T01:23:21Z"
},
{
"body": "On May 15, some automations stopped running which impacted scorer executions. ",
"created_at": "2026-06-04T01:23:21Z",
"display_at": "2026-05-15T20:00:00Z",
"id": "01KT83MKXJ1N4DJKD3Z1K9N7BC",
"incident_id": "01KT83MKXJNJ4TWSD1M5E9R9EQ",
"status": "investigating",
"updated_at": "2026-06-04T01:23:21Z"
}
],
"name": "Automation degradation",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-05-16T18:50:00Z",
"status": "resolved",
"updated_at": "2026-06-04T01:23:21Z"
},
{
"created_at": "2026-05-05T23:35:29Z",
"id": "01KQX7Y8YTJ0P1ATY9H7Z39GD6",
"impact": "major",
"incident_updates": [
{
"body": "The issue was caused by our firewall provider unexpectedly blocking traffic despite safeguards we had in place to prevent this behavior. We are actively working with the vendor on additional mitigation measures to reduce the likelihood of this occurring again.",
"created_at": "2026-05-06T00:41:06Z",
"display_at": "2026-05-06T00:41:06Z",
"id": "01KQXBPDPJ2PDXB750YB3W7PXR",
"incident_id": "01KQX7Y8YTJ0P1ATY9H7Z39GD6",
"status": "resolved",
"updated_at": "2026-05-06T00:41:06Z"
},
{
"body": "We are no longer seeing issues. ",
"created_at": "2026-05-06T00:23:00Z",
"display_at": "2026-05-06T00:23:00Z",
"id": "01KQXAN8WD0R9J5Z3QKARSZ1Y1",
"incident_id": "01KQX7Y8YTJ0P1ATY9H7Z39GD6",
"status": "resolved",
"updated_at": "2026-05-06T00:23:00Z"
},
{
"body": "We are actively monitoring the fix. ",
"created_at": "2026-05-06T00:10:13Z",
"display_at": "2026-05-06T00:10:13Z",
"id": "01KQX9XVPT2E121WZPYMC9TSGH",
"incident_id": "01KQX7Y8YTJ0P1ATY9H7Z39GD6",
"status": "monitoring",
"updated_at": "2026-05-06T00:10:13Z"
},
{
"body": "We have identified the cause and are working with an upstream provider on a fix. ",
"created_at": "2026-05-06T00:08:04Z",
"display_at": "2026-05-06T00:08:04Z",
"id": "01KQX9SXWYM71VX3XFYNQZDVHX",
"incident_id": "01KQX7Y8YTJ0P1ATY9H7Z39GD6",
"status": "identified",
"updated_at": "2026-05-06T00:08:04Z"
},
{
"body": "We're observing a high rate of errors on the production control plane which is causing errors on braintrust.dev and data plane operations.",
"created_at": "2026-05-05T23:35:29Z",
"display_at": "2026-05-05T23:35:29Z",
"id": "01KQX7Y8YTEVHQAYXEJ5ZRS3SD",
"incident_id": "01KQX7Y8YTJ0P1ATY9H7Z39GD6",
"status": "investigating",
"updated_at": "2026-05-05T23:35:29Z"
}
],
"name": "High rate of errors on control plane",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-05-06T00:23:00Z",
"status": "resolved",
"updated_at": "2026-05-06T00:41:06Z"
},
{
"created_at": "2026-04-26T07:17:19Z",
"id": "01KQ4ACQD099F9V9ERBF67MHF0",
"impact": "critical",
"incident_updates": [
{
"body": "Provider has recovered. Error levels back to zero.",
"created_at": "2026-04-26T07:23:10Z",
"display_at": "2026-04-26T07:23:09Z",
"id": "01KQ4AQDRV6CYDAW36KF55AZZJ",
"incident_id": "01KQ4ACQD099F9V9ERBF67MHF0",
"status": "resolved",
"updated_at": "2026-04-26T07:23:10Z"
},
{
"body": "Our upstream auth provider is having an outage. Users unable to login to Braintrust.",
"created_at": "2026-04-26T07:17:19Z",
"display_at": "2026-04-26T07:17:19Z",
"id": "01KQ4ACQD0TAP4ZH7SJQXYWFZQ",
"incident_id": "01KQ4ACQD099F9V9ERBF67MHF0",
"status": "investigating",
"updated_at": "2026-04-26T07:17:19Z"
}
],
"name": "Control plane outage",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-04-26T07:23:09Z",
"status": "resolved",
"updated_at": "2026-04-26T07:23:10Z"
},
{
"created_at": "2026-04-20T03:33:53Z",
"id": "01KPMF79MVP73NTP8VJSBQ037Q",
"impact": "critical",
"incident_updates": [
{
"body": "We're no longer seeing impact from the connectivity issues and we are still investigating the root cause.",
"created_at": "2026-04-20T03:49:52Z",
"display_at": "2026-04-20T03:49:52Z",
"id": "01KPMG4HW4YF1RNMF41BE6XXHN",
"incident_id": "01KPMF79MVP73NTP8VJSBQ037Q",
"status": "resolved",
"updated_at": "2026-04-20T03:53:07Z"
},
{
"body": "We've updated our database connection method and are observing recovery. We're monitoring to look for remaining impact.",
"created_at": "2026-04-20T03:46:54Z",
"display_at": "2026-04-20T03:46:54Z",
"id": "01KPMFZ42HSPF82AZ8CYTJWXKT",
"incident_id": "01KPMF79MVP73NTP8VJSBQ037Q",
"status": "monitoring",
"updated_at": "2026-04-20T03:46:54Z"
},
{
"body": "We're currently experiencing an issue with database connectivity. We're currently attempting to mitigate by connecting through an alternative method.",
"created_at": "2026-04-20T03:33:53Z",
"display_at": "2026-04-20T03:33:53Z",
"id": "01KPMF79MVXP4SZ24P99P2XDBE",
"incident_id": "01KPMF79MVP73NTP8VJSBQ037Q",
"status": "identified",
"updated_at": "2026-04-20T03:33:53Z"
}
],
"name": "Control Plane Database Connectivity Issues",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-04-20T03:49:52Z",
"status": "resolved",
"updated_at": "2026-04-20T03:49:52Z"
},
{
"created_at": "2026-04-13T22:21:24Z",
"id": "01KP4EYT4FZNED9TT52AG7ERER",
"impact": "critical",
"incident_updates": [
{
"body": "Performance quickly returned to normal when the problematic query was identified and remediated, and we are implementing safeguards to prevent similar future issues.",
"created_at": "2026-04-13T22:21:24Z",
"display_at": "2026-04-13T21:51:00Z",
"id": "01KP4EYT4FHD3BMPP2MRJVG1NB",
"incident_id": "01KP4EYT4FZNED9TT52AG7ERER",
"status": "resolved",
"updated_at": "2026-04-13T22:21:57Z"
},
{
"body": "We experienced a brief period of degraded performance on the Braintrust hosted control plane due to elevated database load. The issue was caused by an internal traffic spike to a specific query pattern.",
"created_at": "2026-04-13T22:21:24Z",
"display_at": "2026-04-13T21:38:00Z",
"id": "01KP4EYT4F3WV9Q0TSNZDSDYFD",
"incident_id": "01KP4EYT4FZNED9TT52AG7ERER",
"status": "investigating",
"updated_at": "2026-04-13T22:26:57Z"
}
],
"name": "Control plane database degradation",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-04-13T21:51:00Z",
"status": "resolved",
"updated_at": "2026-04-13T22:21:24Z"
},
{
"created_at": "2026-03-31T17:52:50Z",
"id": "01KN2GDPG2GXVVAX49ZVABP4S1",
"impact": "major",
"incident_updates": [
{
"body": "Error rates have dropped and functionality is restored.",
"created_at": "2026-03-31T18:05:04Z",
"display_at": "2026-03-31T18:05:04Z",
"id": "01KN2H43S8Y6V5ERRZ03S6140V",
"incident_id": "01KN2GDPG2GXVVAX49ZVABP4S1",
"status": "resolved",
"updated_at": "2026-03-31T18:05:04Z"
},
{
"body": "We're seeing a high rate of 5xx errors",
"created_at": "2026-03-31T17:52:50Z",
"display_at": "2026-03-31T17:52:50Z",
"id": "01KN2GDPG2YMPWCK51MJ81GSRC",
"incident_id": "01KN2GDPG2GXVVAX49ZVABP4S1",
"status": "investigating",
"updated_at": "2026-03-31T17:52:50Z"
}
],
"name": "Elevated control plane errors",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-03-31T18:05:04Z",
"status": "resolved",
"updated_at": "2026-03-31T18:05:04Z"
},
{
"created_at": "2026-03-20T23:07:40Z",
"id": "01KM6R28MSWRM0SSNG2STQJT08",
"impact": "critical",
"incident_updates": [
{
"body": "Load issues have subsided",
"created_at": "2026-03-21T00:23:37Z",
"display_at": "2026-03-21T00:23:37Z",
"id": "01KM6WDB24MHECQKRA1ZBM14B4",
"incident_id": "01KM6R28MSWRM0SSNG2STQJT08",
"status": "resolved",
"updated_at": "2026-03-21T00:23:37Z"
},
{
"body": "DB upgrade complete. CPU usage appears to be at normal levels. Monitoring for now.",
"created_at": "2026-03-21T00:15:56Z",
"display_at": "2026-03-21T00:15:56Z",
"id": "01KM6VZ9FP2083CB2Q0P3171E6",
"incident_id": "01KM6R28MSWRM0SSNG2STQJT08",
"status": "monitoring",
"updated_at": "2026-03-21T00:15:56Z"
},
{
"body": "Database size is being increased. This will cause a short outage.",
"created_at": "2026-03-21T00:09:17Z",
"display_at": "2026-03-21T00:09:17Z",
"id": "01KM6VK3PEDYK0GR8NNPBAWR43",
"incident_id": "01KM6R28MSWRM0SSNG2STQJT08",
"status": "monitoring",
"updated_at": "2026-03-21T00:09:17Z"
},
{
"body": "Error rates have recovered by database usage is still high.",
"created_at": "2026-03-20T23:14:33Z",
"display_at": "2026-03-20T23:14:33Z",
"id": "01KM6REWP6Y68Y98GSZHA865E0",
"incident_id": "01KM6R28MSWRM0SSNG2STQJT08",
"status": "monitoring",
"updated_at": "2026-03-20T23:14:33Z"
},
{
"body": "The control plane is experiencing very high load.",
"created_at": "2026-03-20T23:07:40Z",
"display_at": "2026-03-20T23:07:40Z",
"id": "01KM6R28MSJEYSVC2YYC9Q8WVR",
"incident_id": "01KM6R28MSWRM0SSNG2STQJT08",
"status": "investigating",
"updated_at": "2026-03-20T23:07:40Z"
}
],
"name": "Database load issues with control plane",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-03-21T00:23:37Z",
"status": "resolved",
"updated_at": "2026-03-21T00:23:37Z"
},
{
"created_at": "2026-03-20T10:01:03Z",
"id": "01KM5B1Y7JP2W6PYVQAEFAYYV7",
"impact": "major",
"incident_updates": [
{
"body": "The authorization provider has completed mitigation work and is reporting a full recovery in their systems. We are no longer observing impact to users.",
"created_at": "2026-03-20T10:32:23Z",
"display_at": "2026-03-20T10:32:23Z",
"id": "01KM5CV9QPZ9D0DJPWG5N5ZCZJ",
"incident_id": "01KM5B1Y7JP2W6PYVQAEFAYYV7",
"status": "resolved",
"updated_at": "2026-03-20T10:32:23Z"
},
{
"body": "The authorization provider has applied a mitigation and we're seeing signs of recovery. We're continuing to monitor until all errors have subsided.",
"created_at": "2026-03-20T10:26:59Z",
"display_at": "2026-03-20T10:21:00Z",
"id": "01KM5CHD8KQ7V559SHEY8EV45J",
"incident_id": "01KM5B1Y7JP2W6PYVQAEFAYYV7",
"status": "monitoring",
"updated_at": "2026-03-20T10:27:13Z"
},
{
"body": "Degraded performance of our authorization provider is causing issues with accessing the control plane.",
"created_at": "2026-03-20T10:01:03Z",
"display_at": "2026-03-20T09:53:00Z",
"id": "01KM5B1Y7JJMG74EVDRMZ21CE0",
"incident_id": "01KM5B1Y7JP2W6PYVQAEFAYYV7",
"status": "investigating",
"updated_at": "2026-03-20T10:10:43Z"
}
],
"name": "Authorization service in degraded state",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-03-20T10:32:23Z",
"status": "resolved",
"updated_at": "2026-03-20T10:32:23Z"
},
{
"created_at": "2026-03-10T19:25:01Z",
"id": "01KKCKBD6F52BMMBSNR7DNYBXM",
"impact": "critical",
"incident_updates": [
{
"body": "Provider recovered",
"created_at": "2026-03-10T19:25:12Z",
"display_at": "2026-03-10T19:11:00Z",
"id": "01KKCKBR51WKHEGW2P1BAN5CHB",
"incident_id": "01KKCKBD6F52BMMBSNR7DNYBXM",
"status": "resolved",
"updated_at": "2026-03-10T19:25:43Z"
},
{
"body": "An upstream provider had a brief outage that caused high errors in our control plane",
"created_at": "2026-03-10T19:25:01Z",
"display_at": "2026-03-10T19:03:00Z",
"id": "01KKCKBD6FHT72M02E6WD4BWJ2",
"incident_id": "01KKCKBD6F52BMMBSNR7DNYBXM",
"status": "investigating",
"updated_at": "2026-03-10T19:25:38Z"
}
],
"name": "Control Plane Errors",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-03-10T19:11:00Z",
"status": "resolved",
"updated_at": "2026-03-12T18:22:07Z"
},
{
"created_at": "2026-03-10T16:18:23Z",
"id": "01KKC8NNS47E4WA4EZJAX2ZREM",
"impact": "major",
"incident_updates": [
{
"body": "The authorization provider has completed mitigation work and is reporting a full recovery in their systems. We are no longer observing impact to users.",
"created_at": "2026-03-10T16:38:10Z",
"display_at": "2026-03-10T16:38:10Z",
"id": "01KKC9SWHXPYHR3QSK42PH5PHW",
"incident_id": "01KKC8NNS47E4WA4EZJAX2ZREM",
"status": "resolved",
"updated_at": "2026-03-10T16:38:10Z"
},
{
"body": "The authorization provider has applied a mitigation and we're seeing signs of recovery. We're continuing to monitor until all errors have subsided.",
"created_at": "2026-03-10T16:35:54Z",
"display_at": "2026-03-10T16:35:54Z",
"id": "01KKC9NR03R8CQ9PZ25WGCS7MG",
"incident_id": "01KKC8NNS47E4WA4EZJAX2ZREM",
"status": "monitoring",
"updated_at": "2026-03-10T16:35:54Z"
},
{
"body": "Degraded performance of our authorization provider is causing issues with accessing the control plane. The service provider is investigating the root issue.\n\n ",
"created_at": "2026-03-10T16:18:23Z",
"display_at": "2026-03-10T16:18:23Z",
"id": "01KKC8NNS47RDSXYPZ2E7BA07E",
"incident_id": "01KKC8NNS47E4WA4EZJAX2ZREM",
"status": "investigating",
"updated_at": "2026-03-10T16:18:23Z"
}
],
"name": "Authorization service in degraded state",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-03-10T16:38:10Z",
"status": "resolved",
"updated_at": "2026-03-12T18:22:07Z"
},
{
"created_at": "2026-03-03T15:50:59Z",
"id": "01KJT6AERAAS7GHADPMJE7T8SN",
"impact": "major",
"incident_updates": [
{
"body": "Affected a small subset of users.",
"created_at": "2026-03-03T16:24:51Z",
"display_at": "2026-03-03T16:24:51Z",
"id": "01KJT88FBR8782DNR2P9C2JKCX",
"incident_id": "01KJT6AERAAS7GHADPMJE7T8SN",
"status": "resolved",
"updated_at": "2026-03-03T16:24:51Z"
},
{
"body": "S3 automations have stopped running on the expected schedule they were set to.",
"created_at": "2026-03-03T15:50:59Z",
"display_at": "2026-03-03T15:50:59Z",
"id": "01KJT6AERAE2AARS67KBNSGFMF",
"incident_id": "01KJT6AERAAS7GHADPMJE7T8SN",
"status": "investigating",
"updated_at": "2026-03-03T15:50:59Z"
}
],
"name": "S3 Export Automations failing",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-03-03T16:24:51Z",
"status": "resolved",
"updated_at": "2026-03-12T18:22:07Z"
},
{
"created_at": "2026-02-19T16:34:52Z",
"id": "01KHVC260RSW9Z7NJTJPF5PTDA",
"impact": "minor",
"incident_updates": [
{
"body": "We have not observed any recurrence of issues and our authorization provider has reported full recovery and stable performance across their entire system.",
"created_at": "2026-02-19T18:35:55Z",
"display_at": "2026-02-19T18:35:55Z",
"id": "01KHVJZTRTB8VPZRMSPJ344675",
"incident_id": "01KHVC260RSW9Z7NJTJPF5PTDA",
"status": "resolved",
"updated_at": "2026-02-19T18:35:55Z"
},
{
"body": "Impact for Braintrust users appears to have been resolved for the past 30 minutes. We'll continue to monitor until our authorization provider reports full resolution.",
"created_at": "2026-02-19T17:53:43Z",
"display_at": "2026-02-19T17:53:43Z",
"id": "01KHVGJJTRS5521GW8EN9CZ7T5",
"incident_id": "01KHVC260RSW9Z7NJTJPF5PTDA",
"status": "monitoring",
"updated_at": "2026-02-19T17:53:43Z"
},
{
"body": "We've observed improvement in authorization errors but our authorization provider is still reporting a degraded state. We're continuing to monitor.",
"created_at": "2026-02-19T17:20:13Z",
"display_at": "2026-02-19T17:20:13Z",
"id": "01KHVEN7RJDY7CJ4CAA5M2JJNW",
"incident_id": "01KHVC260RSW9Z7NJTJPF5PTDA",
"status": "investigating",
"updated_at": "2026-02-19T17:20:13Z"
},
{
"body": "Degraded performance of our authorization vendor is causing issues with user authorization accessing the control plane. The vendor has identified the issue and is working on a fix.",
"created_at": "2026-02-19T16:34:52Z",
"display_at": "2026-02-19T16:34:52Z",
"id": "01KHVC260RMW6P1YDF1J6KWHMJ",
"incident_id": "01KHVC260RSW9Z7NJTJPF5PTDA",
"status": "investigating",
"updated_at": "2026-02-19T16:34:52Z"
}
],
"name": "Authorization service in degraded state",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-02-19T18:35:55Z",
"status": "resolved",
"updated_at": "2026-03-12T18:22:07Z"
},
{
"created_at": "2026-02-18T21:32:16Z",
"id": "01KHSAP1GGAWAMCKXYGP6Z3AW2",
"impact": "minor",
"incident_updates": [
{
"body": "The issue has been resolved and you should be seeing no slowness in the Braintrust Control Plane UI. Please let us know if you have any issues.",
"created_at": "2026-02-18T21:35:30Z",
"display_at": "2026-02-18T21:35:30Z",
"id": "01KHSAVYQP14ZCTY8AG5BN72XE",
"incident_id": "01KHSAP1GGAWAMCKXYGP6Z3AW2",
"status": "resolved",
"updated_at": "2026-02-18T21:35:30Z"
},
{
"body": "At the current moment in time we have reports of hanging in the UI particularly in dataset or trace views.\n\nWe are currently narrowing down the code change causing the issue. Please stand by we are aware and currently working to resolve.",
"created_at": "2026-02-18T21:32:16Z",
"display_at": "2026-02-18T21:32:16Z",
"id": "01KHSAP1GGM7NDVA3H20235HT6",
"incident_id": "01KHSAP1GGAWAMCKXYGP6Z3AW2",
"status": "investigating",
"updated_at": "2026-02-18T21:32:16Z"
}
],
"name": "Control Plane UI Hanging",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-02-18T21:35:30Z",
"status": "resolved",
"updated_at": "2026-03-12T18:22:07Z"
},
{
"created_at": "2026-02-15T23:16:48Z",
"id": "01KHHSF8TDZBPR38CC12H8JC6C",
"impact": "critical",
"incident_updates": [
{
"body": "The root cause appeared to be an issue with an upstream dependency (database connection pooler), which we have replaced.\n\nWe will be writing a full post-mortem with more details.",
"created_at": "2026-02-16T01:08:08Z",
"display_at": "2026-02-16T01:08:08Z",
"id": "01KHHZV51P9FTPTQ91R9P82X2M",
"incident_id": "01KHHSF8TDZBPR38CC12H8JC6C",
"status": "resolved",
"updated_at": "2026-02-16T01:08:08Z"
},
{
"body": "We are no longer seeing issues",
"created_at": "2026-02-16T00:59:23Z",
"display_at": "2026-02-16T00:59:23Z",
"id": "01KHHZB3PN9R9QD97AWM4KE7FW",
"incident_id": "01KHHSF8TDZBPR38CC12H8JC6C",
"status": "resolved",
"updated_at": "2026-02-16T00:59:23Z"
},
{
"body": "DB upgrades have completed, and we have seen error rates go down. Continuing to monitor.",
"created_at": "2026-02-16T00:45:31Z",
"display_at": "2026-02-16T00:45:31Z",
"id": "01KHHYHQQRZB0DPRWVGWJE6HSM",
"incident_id": "01KHHSF8TDZBPR38CC12H8JC6C",
"status": "monitoring",
"updated_at": "2026-02-16T00:45:31Z"
},
{
"body": "We are upgrade DB resources, which may result in brief downtime",
"created_at": "2026-02-16T00:12:55Z",
"display_at": "2026-02-16T00:12:55Z",
"id": "01KHHWP131DPRFGPXR16N3WW46",
"incident_id": "01KHHSF8TDZBPR38CC12H8JC6C",
"status": "investigating",
"updated_at": "2026-02-16T00:12:55Z"
},
{
"body": "We are repairing the DB which may result in brief downtime.",
"created_at": "2026-02-16T00:11:26Z",
"display_at": "2026-02-16T00:11:26Z",
"id": "01KHHWKA0ZQ6M3362N9DQS8F4D",
"incident_id": "01KHHSF8TDZBPR38CC12H8JC6C",
"status": "investigating",
"updated_at": "2026-02-16T00:11:26Z"
},
{
"body": "We're observing a high rate of errors on the production control plane which is causing errors on braintrust.dev and data plane operations.",
"created_at": "2026-02-15T23:16:48Z",
"display_at": "2026-02-15T23:16:48Z",
"id": "01KHHSF8TD7H5DP664W1AB4TNT",
"incident_id": "01KHHSF8TDZBPR38CC12H8JC6C",
"status": "investigating",
"updated_at": "2026-02-15T23:16:48Z"
}
],
"name": "High rate of errors on control plane",
"page_id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"resolved_at": "2026-02-16T00:59:23Z",
"status": "resolved",
"updated_at": "2026-03-12T18:22:07Z"
}
],
"page": {
"id": "01JD5G3FK0E1SRAMQ5NFEETZ3J",
"name": "Braintrust",
"updated_at": "2026-09-09T22:10:23Z",
"url": "https://status.braintrust.dev/"
}
}