Snapshot 306735
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"incidents": [
{
"created_at": "2026-11-19T10:21:51Z",
"id": "01M7SN7CTQ4TN5R2TXN2W0VXDN",
"impact": "minor",
"incident_updates": [
{
"body": "Serverless platform now stable and operational.",
"created_at": "2026-11-19T11:14:40Z",
"display_at": "2026-11-19T11:14:40Z",
"id": "01M7SR8458YWE11ZD68XDYZVGX",
"incident_id": "01M7SN7CTQ4TN5R2TXN2W0VXDN",
"status": "resolved",
"updated_at": "2026-11-19T11:14:40Z"
},
{
"body": "We’ve applied a fix to resolve the issue and now we are investigating the Platform to ensure it’s stability. Serverless platform now in operational state.",
"created_at": "2026-11-19T11:03:25Z",
"display_at": "2026-11-19T11:03:24Z",
"id": "01M7SQKG0Z6QDAR96XT0WHM6Q3",
"incident_id": "01M7SN7CTQ4TN5R2TXN2W0VXDN",
"status": "monitoring",
"updated_at": "2026-11-19T11:03:25Z"
},
{
"body": "We are now experiencing issues with Serverless builds. The team is fixing that right now.",
"created_at": "2026-11-19T10:21:51Z",
"display_at": "2026-11-19T10:21:51Z",
"id": "01M7SN7CTQNJXQNQWABCTPNM0X",
"incident_id": "01M7SN7CTQ4TN5R2TXN2W0VXDN",
"status": "identified",
"updated_at": "2026-11-19T10:21:51Z"
}
],
"name": "Serverless platform has degraded operational status",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-19T11:14:40Z",
"status": "resolved",
"updated_at": "2026-11-19T11:14:41Z"
},
{
"created_at": "2026-11-18T12:49:25Z",
"id": "01M7QB8WW5XRF4HKFQ6PX4FQSG",
"impact": "major",
"incident_updates": [
{
"body": "The issue is now resolved. Affected models are operational.",
"created_at": "2026-11-18T13:04:00Z",
"display_at": "2026-11-18T13:04:00Z",
"id": "01M7QC3JS5Q46F13DHXC0DP1VG",
"incident_id": "01M7QB8WW5XRF4HKFQ6PX4FQSG",
"status": "resolved",
"updated_at": "2026-11-18T13:04:00Z"
},
{
"body": "We are now experiencing multiple models outage. Team is working right now to fix.",
"created_at": "2026-11-18T12:59:35Z",
"display_at": "2026-11-18T12:59:35Z",
"id": "01M7QBVG64XHRW92X9SE1CPFPS",
"incident_id": "01M7QB8WW5XRF4HKFQ6PX4FQSG",
"status": "identified",
"updated_at": "2026-11-18T12:59:35Z"
},
{
"body": "We are now experiencing the issue with deepseek model. Team is investigating.",
"created_at": "2026-11-18T12:49:25Z",
"display_at": "2026-11-18T12:49:25Z",
"id": "01M7QB8WW5XD10RPSRP7NHEW5Z",
"incident_id": "01M7QB8WW5XRF4HKFQ6PX4FQSG",
"status": "investigating",
"updated_at": "2026-11-18T12:49:25Z"
}
],
"name": "Multiple models outage",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-18T13:04:00Z",
"status": "resolved",
"updated_at": "2026-11-18T13:04:00Z"
},
{
"created_at": "2026-11-17T19:06:17Z",
"id": "01M7NEE7P538QZV77R4GHBKCR3",
"impact": "minor",
"incident_updates": [
{
"body": "Fix was released and after a period of monitoring we are marking this as closed.",
"created_at": "2026-11-17T20:08:32Z",
"display_at": "2026-11-17T20:08:32Z",
"id": "01M7NJ06VM5KMTFT72FNP6M3V2",
"incident_id": "01M7NEE7P538QZV77R4GHBKCR3",
"status": "resolved",
"updated_at": "2026-11-17T20:08:32Z"
},
{
"body": "We are aware and implementing a fix for the Flux 2 Klein 9B 500 errors that are being returned on inference requests.",
"created_at": "2026-11-17T19:06:17Z",
"display_at": "2026-11-17T19:06:17Z",
"id": "01M7NEE7P5CRF2R0TFX80ZNPBV",
"incident_id": "01M7NEE7P538QZV77R4GHBKCR3",
"status": "identified",
"updated_at": "2026-11-17T19:06:17Z"
}
],
"name": "Flux 2 Klein 9B experiencing 500 responses",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-17T20:08:32Z",
"status": "resolved",
"updated_at": "2026-11-17T20:08:32Z"
},
{
"created_at": "2026-11-17T16:44:50Z",
"id": "01M7N6B7HYE5N4ZJ43TQF6YMSA",
"impact": "minor",
"incident_updates": [
{
"body": "Issue has been resolved. And the platform now is stable.",
"created_at": "2026-11-17T18:15:52Z",
"display_at": "2026-11-17T18:15:52Z",
"id": "01M7NBHXP2H15PHTAEHXABFX7F",
"incident_id": "01M7N6B7HYE5N4ZJ43TQF6YMSA",
"status": "resolved",
"updated_at": "2026-11-17T18:15:52Z"
},
{
"body": "Configuration changes have been deployed, we are seeing autoscaling return to normal. We continue to monitor the situation.",
"created_at": "2026-11-17T17:16:52Z",
"display_at": "2026-11-17T17:16:52Z",
"id": "01M7N85WBFZ6957YBVWCXBTV9P",
"incident_id": "01M7N6B7HYE5N4ZJ43TQF6YMSA",
"status": "monitoring",
"updated_at": "2026-11-17T17:16:52Z"
},
{
"body": "We have identified the root cause, and currently in the process of deploying configuration changes to restore automatic scaling.",
"created_at": "2026-11-17T16:54:21Z",
"display_at": "2026-11-17T16:54:21Z",
"id": "01M7N6WN4ZZPRNNK749WXM4SCF",
"incident_id": "01M7N6B7HYE5N4ZJ43TQF6YMSA",
"status": "identified",
"updated_at": "2026-11-17T16:54:21Z"
},
{
"body": "We have identified an issue which is preventing workers from scaling. This impacts apps that scale to zero, preventing them from starting as well as bursting new workers from the configured minimum workers. We are investigating the underlying causes.",
"created_at": "2026-11-17T16:44:50Z",
"display_at": "2026-11-17T16:44:50Z",
"id": "01M7N6B7HYMYRV83BYAQGEZ0XF",
"incident_id": "01M7N6B7HYE5N4ZJ43TQF6YMSA",
"status": "investigating",
"updated_at": "2026-11-17T16:44:50Z"
}
],
"name": "Degraded worker autoscaling",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-17T18:15:52Z",
"status": "resolved",
"updated_at": "2026-11-17T18:15:52Z"
},
{
"created_at": "2026-11-14T11:53:49Z",
"id": "01M7CYG6RNW2YEX3BE4VDKKMS7",
"impact": "major",
"incident_updates": [
{
"body": "After close monitoring for the last ~1 hour we are resolved.",
"created_at": "2026-11-14T13:56:13Z",
"display_at": "2026-11-14T13:56:13Z",
"id": "01M7D5GA5PQWSBAARGEYGS96VX",
"incident_id": "01M7CYG6RNW2YEX3BE4VDKKMS7",
"status": "resolved",
"updated_at": "2026-11-14T13:56:13Z"
},
{
"body": "We have resolved the root cause issue and have implemented a fix to the infrastructure. We are now monitoring.",
"created_at": "2026-11-14T12:25:09Z",
"display_at": "2026-11-14T12:25:09Z",
"id": "01M7D09JKE16HJ1XCHK1AMZ2E7",
"incident_id": "01M7CYG6RNW2YEX3BE4VDKKMS7",
"status": "monitoring",
"updated_at": "2026-11-14T12:25:09Z"
},
{
"body": "We’re investigating an issue impacting **Inference**, where requests for several image-generation models are currently failing.\n\nOur team is working to bring the affected compute capacity back online and restore model availability.",
"created_at": "2026-11-14T11:53:49Z",
"display_at": "2026-11-14T11:53:49Z",
"id": "01M7CYG6RNSDWHCZNNSHMMR66H",
"incident_id": "01M7CYG6RNW2YEX3BE4VDKKMS7",
"status": "investigating",
"updated_at": "2026-11-14T11:53:49Z"
}
],
"name": "Investigating a production infrastructure issue",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-14T13:56:13Z",
"status": "resolved",
"updated_at": "2026-11-14T13:56:13Z"
},
{
"created_at": "2026-11-12T21:08:49Z",
"id": "01M78SEZSWN0K8EMJA14H1XET8",
"impact": "major",
"incident_updates": [
{
"body": "The fix is confirmed to be working. New workers are starting and running as expected.",
"created_at": "2026-11-12T21:26:08Z",
"display_at": "2026-11-12T21:26:08Z",
"id": "01M78TEPXN2RJYPAA44AMAP3KG",
"incident_id": "01M78SEZSWN0K8EMJA14H1XET8",
"status": "resolved",
"updated_at": "2026-11-12T21:26:08Z"
},
{
"body": "A fix has deployed and we are seeing platform recovery. Continuing to monitor the sitaution.",
"created_at": "2026-11-12T21:20:16Z",
"display_at": "2026-11-12T21:20:16Z",
"id": "01M78T3Z9XHFZHDM4XFJ0HY13X",
"incident_id": "01M78SEZSWN0K8EMJA14H1XET8",
"status": "monitoring",
"updated_at": "2026-11-12T21:20:16Z"
},
{
"body": "New workers are failing to start, we have identified the root cause and are deploying a fix.",
"created_at": "2026-11-12T21:08:49Z",
"display_at": "2026-11-12T21:08:49Z",
"id": "01M78SEZSWYS0DB4GK2NHY64BW",
"incident_id": "01M78SEZSWN0K8EMJA14H1XET8",
"status": "identified",
"updated_at": "2026-11-12T21:08:49Z"
}
],
"name": "Workers failing to start",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-12T21:26:08Z",
"status": "resolved",
"updated_at": "2026-11-12T21:26:08Z"
},
{
"created_at": "2026-11-12T19:52:17Z",
"id": "01M78N2VT1VG4J9T0ZF5P3R825",
"impact": "minor",
"incident_updates": [
{
"body": "A fix has been applied, and success rates are back to normal. Requests made during this time can be safely retried. We apologize for the disruption.",
"created_at": "2026-11-12T20:12:08Z",
"display_at": "2026-11-12T20:12:08Z",
"id": "01M78P76FHJ2YVSQK62Y57C1NE",
"incident_id": "01M78N2VT1VG4J9T0ZF5P3R825",
"status": "resolved",
"updated_at": "2026-11-12T20:12:08Z"
},
{
"body": "Since approximately 18:30 UTC, requests to some Google Gemini and OpenAI gpt-image models are failing with errors. We have identified the cause and are working on a fix. Other models and the API are not affected. ",
"created_at": "2026-11-12T19:52:17Z",
"display_at": "2026-11-12T19:52:17Z",
"id": "01M78N2VT1WBN8PDPGFGS12G1P",
"incident_id": "01M78N2VT1VG4J9T0ZF5P3R825",
"status": "identified",
"updated_at": "2026-11-12T19:52:17Z"
}
],
"name": "Elevated errors for Google Gemini and OpenAI image models",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-12T20:12:08Z",
"status": "resolved",
"updated_at": "2026-11-12T20:12:08Z"
},
{
"created_at": "2026-11-12T15:12:20Z",
"id": "01M78528NMFBRH51PJYTYDY299",
"impact": "major",
"incident_updates": [
{
"body": "We are seeing stability in the platform, apps are building and starting in a normal manner.",
"created_at": "2026-11-12T16:10:02Z",
"display_at": "2026-11-12T16:10:02Z",
"id": "01M788BXEFK1S5J3SVTMGEDZ8S",
"incident_id": "01M78528NMFBRH51PJYTYDY299",
"status": "resolved",
"updated_at": "2026-11-12T16:10:02Z"
},
{
"body": "We have deployed a fix for the underlying issue, we are seeing recovery and continue to monitor the situation.",
"created_at": "2026-11-12T15:28:10Z",
"display_at": "2026-11-12T15:28:10Z",
"id": "01M785Z8HP2PWP08ZK7NMYNJFY",
"incident_id": "01M78528NMFBRH51PJYTYDY299",
"status": "monitoring",
"updated_at": "2026-11-12T15:28:10Z"
},
{
"body": "We have identified an issue where workers are failing to start for specific apps. We have identified the issue and are working to resolve the root cause.",
"created_at": "2026-11-12T15:12:20Z",
"display_at": "2026-11-12T15:12:20Z",
"id": "01M78528NMXTNG28F2Q488WWQN",
"incident_id": "01M78528NMFBRH51PJYTYDY299",
"status": "identified",
"updated_at": "2026-11-12T15:12:20Z"
}
],
"name": "Worker failures impacting app start ups",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-12T16:10:02Z",
"status": "resolved",
"updated_at": "2026-11-12T16:10:02Z"
},
{
"created_at": "2026-11-11T18:37:37Z",
"id": "01M75YDDGYJK8BA0M4W08YBTV9",
"impact": "none",
"incident_updates": [
{
"body": "All systems have returned to normal.",
"created_at": "2026-11-11T20:58:44Z",
"display_at": "2026-11-11T20:58:44Z",
"id": "01M766FTN68QSA19KQ304TRX2A",
"incident_id": "01M75YDDGYJK8BA0M4W08YBTV9",
"status": "resolved",
"updated_at": "2026-11-11T20:58:44Z"
},
{
"body": "We’re investigating an issue causing requests for the **HiDream I1 Fast** model to fail.\n\nOur team is working to restore service as quickly as possible.",
"created_at": "2026-11-11T18:37:37Z",
"display_at": "2026-11-11T18:37:37Z",
"id": "01M75YDDGYW6P8KSWHK9015NBG",
"incident_id": "01M75YDDGYJK8BA0M4W08YBTV9",
"status": "investigating",
"updated_at": "2026-11-11T18:37:37Z"
}
],
"name": "HiDream I1 Fast model requests failing",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-11-11T20:58:44Z",
"status": "resolved",
"updated_at": "2026-11-11T20:58:44Z"
},
{
"created_at": "2026-10-29T16:57:50Z",
"id": "01M649HBQGH69V0A94FWTV0W37",
"impact": "minor",
"incident_updates": [
{
"body": "The issue with rate-limiting has been resolved.",
"created_at": "2026-10-29T19:19:13Z",
"display_at": "2026-10-29T19:19:13Z",
"id": "01M64HM8KKGB1CS75W0CQHZ2JP",
"incident_id": "01M649HBQGH69V0A94FWTV0W37",
"status": "resolved",
"updated_at": "2026-10-29T19:19:13Z"
},
{
"body": "Customers could experience the rate-limiting (429) response.\nWe are now taking action to process the load",
"created_at": "2026-10-29T16:57:50Z",
"display_at": "2026-10-29T16:57:50Z",
"id": "01M649HBQG2F58K6Q65180EPX6",
"incident_id": "01M649HBQGH69V0A94FWTV0W37",
"status": "investigating",
"updated_at": "2026-10-29T16:57:50Z"
}
],
"name": "Models experiencing delays due to increased traffic",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-10-29T19:19:13Z",
"status": "resolved",
"updated_at": "2026-10-29T19:19:13Z"
},
{
"created_at": "2026-10-28T17:31:08Z",
"id": "01M61S1M3MEBD92EJ0KXJ8W3SE",
"impact": "major",
"incident_updates": [
{
"body": "Issue resolved.",
"created_at": "2026-10-28T17:41:43Z",
"display_at": "2026-10-28T17:41:42Z",
"id": "01M61SMZX6Q06RZRFVH8702M3V",
"incident_id": "01M61S1M3MEBD92EJ0KXJ8W3SE",
"status": "resolved",
"updated_at": "2026-10-28T17:41:43Z"
},
{
"body": "Issue identified and a fix rolled out. API is back operational. The team continues to monitor.",
"created_at": "2026-10-28T17:33:07Z",
"display_at": "2026-10-28T17:33:07Z",
"id": "01M61S58MDKG7K3Z78J1XGNCVH",
"incident_id": "01M61S1M3MEBD92EJ0KXJ8W3SE",
"status": "monitoring",
"updated_at": "2026-10-28T17:33:07Z"
},
{
"body": "We are aware of the Serverless API experiencing an outage at the moment and the team is investigating.",
"created_at": "2026-10-28T17:31:08Z",
"display_at": "2026-10-28T17:31:08Z",
"id": "01M61S1M3MWJ2B3YW4MTZ7W9QR",
"incident_id": "01M61S1M3MEBD92EJ0KXJ8W3SE",
"status": "investigating",
"updated_at": "2026-10-28T17:31:08Z"
}
],
"name": "Serverless API is currently unreachable",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-10-28T17:41:42Z",
"status": "resolved",
"updated_at": "2026-10-28T17:41:43Z"
},
{
"created_at": "2026-10-19T15:46:13Z",
"id": "01M5ADF1DHY9T5YVAEQECSZFBV",
"impact": "none",
"incident_updates": [
{
"body": "Queue wait times have been resolved. Some requests may return HTTP Error 429, but that is temporary and should self-resolve.",
"created_at": "2026-10-19T16:51:05Z",
"display_at": "2026-10-19T16:51:05Z",
"id": "01M5AH5TWSX8GHFFEYH3WA6N6Q",
"incident_id": "01M5ADF1DHY9T5YVAEQECSZFBV",
"status": "resolved",
"updated_at": "2026-10-19T16:51:05Z"
},
{
"body": "Increased load and slower than normal response times on Z-Image Turbo family of models.",
"created_at": "2026-10-19T15:46:13Z",
"display_at": "2026-10-19T15:46:13Z",
"id": "01M5ADF1DHA0N4DFCY70DD2HVA",
"incident_id": "01M5ADF1DHY9T5YVAEQECSZFBV",
"status": "monitoring",
"updated_at": "2026-10-19T15:46:13Z"
}
],
"name": "Increased traffic on Z-Image Turbo Models",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-10-19T16:51:05Z",
"status": "resolved",
"updated_at": "2026-10-19T16:51:05Z"
},
{
"created_at": "2026-10-07T19:14:33Z",
"id": "01M4BWKWZ3RCKYZ1PQ13BVAVPN",
"impact": "major",
"incident_updates": [
{
"body": "Incident has been resolved, thank you for your patience.",
"created_at": "2026-10-07T20:52:33Z",
"display_at": "2026-10-07T20:52:33Z",
"id": "01M4C27B4YWHTXT7GBQTYY0MTH",
"incident_id": "01M4BWKWZ3RCKYZ1PQ13BVAVPN",
"status": "resolved",
"updated_at": "2026-10-07T20:52:33Z"
},
{
"body": "Network issue has been resolved, we are monitoring the queues until they are drained",
"created_at": "2026-10-07T19:59:21Z",
"display_at": "2026-10-07T19:59:21Z",
"id": "01M4BZ5XXPPDRQY10GHM13RVJC",
"incident_id": "01M4BWKWZ3RCKYZ1PQ13BVAVPN",
"status": "monitoring",
"updated_at": "2026-10-07T19:59:21Z"
},
{
"body": "One of our infrastructure vendors is currently experiencing network issues, which has caused some of our compute capacity to become unavailable.\nWe are actively migrating affected workloads to alternative infrastructure to restore full capacity as quickly as possible.\nWe apologise for the disruption and appreciate your patience while we work to fully resolve the issue.",
"created_at": "2026-10-07T19:16:40Z",
"display_at": "2026-10-07T19:16:39Z",
"id": "01M4BWQRD74X0GJPJMAP4R9R7Q",
"incident_id": "01M4BWKWZ3RCKYZ1PQ13BVAVPN",
"status": "identified",
"updated_at": "2026-10-07T19:16:40Z"
},
{
"body": "One of our infrastructure vendors is currently experiencing network issues, which has caused some of our compute capacity to become unavailable.\nWe are actively migrating affected workloads to alternative infrastructure to restore full capacity as quickly as possible.\nWe apologise for the disruption and appreciate your patience while we work to fully resolve the issue.",
"created_at": "2026-10-07T19:14:33Z",
"display_at": "2026-10-07T19:14:33Z",
"id": "01M4BWKWZ3Y47TJNJH3JECC1K5",
"incident_id": "01M4BWKWZ3RCKYZ1PQ13BVAVPN",
"status": "identified",
"updated_at": "2026-10-07T19:14:33Z"
}
],
"name": "Compute Capacity Degradation Due to Vendor Network Issue",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-10-07T20:52:33Z",
"status": "resolved",
"updated_at": "2026-10-07T20:52:33Z"
},
{
"created_at": "2026-09-28T16:30:54Z",
"id": "01M3MDNS86P1WX19531C0EDRVG",
"impact": "none",
"incident_updates": [
{
"body": "Degraded service resolved for FLUX.1 [dev]",
"created_at": "2026-09-28T20:03:08Z",
"display_at": "2026-09-28T20:03:08Z",
"id": "01M3MSTC5AFMM1VKMXYKRN2HH3",
"incident_id": "01M3MDNS86P1WX19531C0EDRVG",
"status": "resolved",
"updated_at": "2026-09-28T20:03:08Z"
},
{
"body": "We are experiencing high load and slower than normal response times on FLUX.1 [dev] models.",
"created_at": "2026-09-28T16:30:54Z",
"display_at": "2026-09-28T16:30:54Z",
"id": "01M3MDNS86C1PDS9JS9V57ZF3Q",
"incident_id": "01M3MDNS86P1WX19531C0EDRVG",
"status": "identified",
"updated_at": "2026-09-28T16:30:54Z"
}
],
"name": "Degraded performance of FLUX.1 [dev]",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-09-28T20:03:08Z",
"status": "resolved",
"updated_at": "2026-09-28T20:03:08Z"
},
{
"created_at": "2026-09-22T12:47:45Z",
"id": "01M34JGW3T244PBV5REE0EC7WS",
"impact": "minor",
"incident_updates": [
{
"body": "Incident has been resolved",
"created_at": "2026-09-22T13:17:26Z",
"display_at": "2026-09-22T13:17:25Z",
"id": "01M34M76JHT0FZ7EXATRZMVWSN",
"incident_id": "01M34JGW3T244PBV5REE0EC7WS",
"status": "resolved",
"updated_at": "2026-09-22T13:17:26Z"
},
{
"body": "Additional consumers joined the queue. The queue is stabilising; we will continue to monitor it a little longer.",
"created_at": "2026-09-22T12:59:20Z",
"display_at": "2026-09-22T12:59:20Z",
"id": "01M34K625GRNFZK3RBFKQ2T33Q",
"incident_id": "01M34JGW3T244PBV5REE0EC7WS",
"status": "monitoring",
"updated_at": "2026-09-22T12:59:20Z"
},
{
"body": "We have increased traffic towards the flux.1-dev model. We' are adjusting the consumers to satisfy the demand.",
"created_at": "2026-09-22T12:47:45Z",
"display_at": "2026-09-22T12:47:45Z",
"id": "01M34JGW3TYTHQ4TB7HXBDMSHM",
"incident_id": "01M34JGW3T244PBV5REE0EC7WS",
"status": "identified",
"updated_at": "2026-09-22T12:47:45Z"
}
],
"name": "Fast-flux increased traffic",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-09-22T13:17:25Z",
"status": "resolved",
"updated_at": "2026-09-22T13:17:26Z"
},
{
"created_at": "2026-09-10T19:21:31Z",
"id": "01M26C97XQ5Q2ACTG1MVVTD3KC",
"impact": "minor",
"incident_updates": [
{
"body": "We're continuing to monitor Seedance 2.5 performance. Generation times have improved and we'll keep watching closely. ",
"created_at": "2026-09-13T15:20:54Z",
"display_at": "2026-09-13T15:20:54Z",
"id": "01M2DNPTV2KNP9ECCHAHFYBH9C",
"incident_id": "01M26C97XQ5Q2ACTG1MVVTD3KC",
"status": "resolved",
"updated_at": "2026-09-13T15:20:54Z"
},
{
"body": "We've seen a meaningful improvement in Seedance 2.5 generation latency over the past several hours as we continue working with our upstream provider. Performance is trending back toward normal levels. ",
"created_at": "2026-09-11T11:33:26Z",
"display_at": "2026-09-11T11:33:26Z",
"id": "01M283WWNRSP2KMH9PTKGRP7RE",
"incident_id": "01M26C97XQ5Q2ACTG1MVVTD3KC",
"status": "monitoring",
"updated_at": "2026-09-11T11:33:26Z"
},
{
"body": "We’re aware of elevated latency on Seedance 2.5 due to capacity constraints with our upstream video generation provider. This is affecting generation times.\n\nWe’re actively working with the provider to get this resolved and will share updates as we have them..",
"created_at": "2026-09-10T19:21:31Z",
"display_at": "2026-09-10T19:21:31Z",
"id": "01M26C97XQSNFV5RADCRFGEPYY",
"incident_id": "01M26C97XQ5Q2ACTG1MVVTD3KC",
"status": "identified",
"updated_at": "2026-09-10T19:21:31Z"
}
],
"name": "Seedance 2.5 – Elevated Generation Latency",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-09-13T15:20:54Z",
"status": "resolved",
"updated_at": "2026-09-13T15:20:54Z"
},
{
"created_at": "2026-09-13T07:31:47Z",
"id": "01M2CTVVCTA2PBE18H6TMBG3F7",
"impact": "minor",
"incident_updates": [
{
"body": "The issue has been resolved. Queues are back to normal; requests are being processed.",
"created_at": "2026-09-13T07:42:10Z",
"display_at": "2026-09-13T07:42:10Z",
"id": "01M2CVEVFM4H4SK5N452XQ4SAF",
"incident_id": "01M2CTVVCTA2PBE18H6TMBG3F7",
"status": "resolved",
"updated_at": "2026-09-13T07:42:10Z"
},
{
"body": "We've identified the source of the problem, and we are working on a fix. We'll keep you posted as soon as possible.",
"created_at": "2026-09-13T07:31:47Z",
"display_at": "2026-09-13T07:31:47Z",
"id": "01M2CTVVCT34YNTMVNJZDCVAN3",
"incident_id": "01M2CTVVCTA2PBE18H6TMBG3F7",
"status": "identified",
"updated_at": "2026-09-13T07:31:47Z"
}
],
"name": "Flux2Klein models increased response times",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-09-13T07:42:10Z",
"status": "resolved",
"updated_at": "2026-09-13T07:42:10Z"
},
{
"created_at": "2026-09-09T18:18:06Z",
"id": "01M23P8DHBDJKH8JT0P1P72ETC",
"impact": "minor",
"incident_updates": [
{
"body": "Google service looks to have restored back to normal.",
"created_at": "2026-09-09T18:18:06Z",
"display_at": "2026-09-09T17:30:00Z",
"id": "01M23P8DHBWRAGEY0T8EMENHRN",
"incident_id": "01M23P8DHBDJKH8JT0P1P72ETC",
"status": "resolved",
"updated_at": "2026-09-09T18:18:06Z"
},
{
"body": "Google Omni Flash 1.1 is currently experiencing failures via up-stream Google services.",
"created_at": "2026-09-09T18:18:06Z",
"display_at": "2026-09-09T16:35:00Z",
"id": "01M23P8DHB6TMGV66W87M9E7K3",
"incident_id": "01M23P8DHBDJKH8JT0P1P72ETC",
"status": "identified",
"updated_at": "2026-09-09T18:18:06Z"
}
],
"name": "Google Omni Flash 1.1 experiencing failures",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-09-09T17:30:00Z",
"status": "resolved",
"updated_at": "2026-09-09T18:18:06Z"
},
{
"created_at": "2026-08-25T13:36:14Z",
"id": "01M0WJ5GG6QSP2NTJ0RHDWX630",
"impact": "minor",
"incident_updates": [
{
"body": "New consumers added, backlog resolved",
"created_at": "2026-08-25T14:41:39Z",
"display_at": "2026-08-25T14:41:39Z",
"id": "01M0WNX9GXN38Y85E37NABKF2D",
"incident_id": "01M0WJ5GG6QSP2NTJ0RHDWX630",
"status": "resolved",
"updated_at": "2026-08-25T14:41:39Z"
},
{
"body": "The load is processing, should be resolved soon.",
"created_at": "2026-08-25T14:07:46Z",
"display_at": "2026-08-25T14:07:46Z",
"id": "01M0WKZ8DVE3C90X2HVHTNHCCJ",
"incident_id": "01M0WJ5GG6QSP2NTJ0RHDWX630",
"status": "monitoring",
"updated_at": "2026-08-25T14:07:46Z"
},
{
"body": "We are experiencing high load for flux1-dev model. Currently we are adding more servers to handle the load.",
"created_at": "2026-08-25T13:36:14Z",
"display_at": "2026-08-25T13:36:14Z",
"id": "01M0WJ5GG6GDQZZT28VFP2JNW5",
"incident_id": "01M0WJ5GG6QSP2NTJ0RHDWX630",
"status": "identified",
"updated_at": "2026-08-25T13:36:14Z"
}
],
"name": "High load for flux1-dev model",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-08-25T14:41:39Z",
"status": "resolved",
"updated_at": "2026-08-25T14:41:39Z"
},
{
"created_at": "2026-08-24T17:34:15Z",
"id": "01M0TDCM7TJMFSPJBQB7ZJ08ZZ",
"impact": "minor",
"incident_updates": [
{
"body": "\u003e We updated routing to move Vercel traffic to our self-hosted capacity. The routing change is now working correctly, and Doubleword and our VLLM capacity are operating normally.",
"created_at": "2026-08-24T19:52:29Z",
"display_at": "2026-08-24T19:52:29Z",
"id": "01M0TN9Q2P6WVDKY23XKN9T85F",
"incident_id": "01M0TDCM7TJMFSPJBQB7ZJ08ZZ",
"status": "resolved",
"updated_at": "2026-08-24T19:52:29Z"
},
{
"body": "GLM 5.2 is currently experiencing degraded performance due to increased demand.",
"created_at": "2026-08-24T17:34:15Z",
"display_at": "2026-08-24T17:34:15Z",
"id": "01M0TDCM7T8RE8F8JN70RQ2WE3",
"incident_id": "01M0TDCM7TJMFSPJBQB7ZJ08ZZ",
"status": "monitoring",
"updated_at": "2026-08-24T17:34:15Z"
}
],
"name": "GLM 5.2 experiencing degraded performance",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-08-24T19:52:29Z",
"status": "resolved",
"updated_at": "2026-08-24T19:52:29Z"
},
{
"created_at": "2026-08-10T01:30:42Z",
"id": "01KZMMP851G4YFBAJ63XA2AXJN",
"impact": "major",
"incident_updates": [
{
"body": "We've monitored for the last 30 minutes and the service has resumed with no further issues.",
"created_at": "2026-08-10T02:31:17Z",
"display_at": "2026-08-10T02:31:17Z",
"id": "01KZMR55YSSWW88ZVBGA4QVG66",
"incident_id": "01KZMMP851G4YFBAJ63XA2AXJN",
"status": "resolved",
"updated_at": "2026-08-10T02:31:17Z"
},
{
"body": "Fixed rolled out and inference confirmed as working. We are monitoring.",
"created_at": "2026-08-10T02:10:59Z",
"display_at": "2026-08-10T02:10:59Z",
"id": "01KZMQ002PRV8GSKSQBYWRBHAA",
"incident_id": "01KZMMP851G4YFBAJ63XA2AXJN",
"status": "monitoring",
"updated_at": "2026-08-10T02:10:59Z"
},
{
"body": "We've identified the root cause and a fix is being rolled out. Google, OpenAI and Byteplus models are stating to process inference.",
"created_at": "2026-08-10T02:05:31Z",
"display_at": "2026-08-10T02:05:31Z",
"id": "01KZMPNZYYJJB7MTJAJ5K90AX8",
"incident_id": "01KZMMP851G4YFBAJ63XA2AXJN",
"status": "identified",
"updated_at": "2026-08-10T02:05:31Z"
},
{
"body": "We are investigating an issue impacting Google, Byteplus and OpenAI models at the moment.",
"created_at": "2026-08-10T01:30:42Z",
"display_at": "2026-08-10T01:30:42Z",
"id": "01KZMMP851KDTV3B53R324YHED",
"incident_id": "01KZMMP851G4YFBAJ63XA2AXJN",
"status": "investigating",
"updated_at": "2026-08-10T01:30:42Z"
}
],
"name": "Issue impacting Google, Byteplus and OpenAI models",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-08-10T02:31:17Z",
"status": "resolved",
"updated_at": "2026-08-10T02:31:17Z"
},
{
"created_at": "2026-08-05T21:08:39Z",
"id": "01KZ9W3HADD0NTB81QA5D5DAJ4",
"impact": "minor",
"incident_updates": [
{
"body": "Models performance has been stabilized.",
"created_at": "2026-08-05T21:59:41Z",
"display_at": "2026-08-05T21:59:41Z",
"id": "01KZ9Z0ZPVX49K4T1N8518SWMA",
"incident_id": "01KZ9W3HADD0NTB81QA5D5DAJ4",
"status": "resolved",
"updated_at": "2026-08-05T21:59:41Z"
},
{
"body": "The model performance is under monitoring right now. Team is working to be sure everything runs as it should.",
"created_at": "2026-08-05T21:23:20Z",
"display_at": "2026-08-05T21:23:20Z",
"id": "01KZ9WYE06NDNFYTK0KM1H15ZJ",
"incident_id": "01KZ9W3HADD0NTB81QA5D5DAJ4",
"status": "monitoring",
"updated_at": "2026-08-05T21:23:20Z"
},
{
"body": "We're currently experiencing issues with models. Expect delays in processing",
"created_at": "2026-08-05T21:08:39Z",
"display_at": "2026-08-05T21:08:39Z",
"id": "01KZ9W3HADW1810TSE198SZPSR",
"incident_id": "01KZ9W3HADD0NTB81QA5D5DAJ4",
"status": "identified",
"updated_at": "2026-08-05T21:08:39Z"
}
],
"name": "Some models may be slow in processing",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-08-05T21:59:41Z",
"status": "resolved",
"updated_at": "2026-08-05T21:59:41Z"
},
{
"created_at": "2026-08-05T13:19:07Z",
"id": "01KZ917SZCJZSP49AP6G1PQ9K9",
"impact": "minor",
"incident_updates": [
{
"body": "The issue has been resolved.\nThank you for the patience.",
"created_at": "2026-08-05T18:26:22Z",
"display_at": "2026-08-05T18:26:22Z",
"id": "01KZ9JTCTDE17X7757N10KRHYJ",
"incident_id": "01KZ917SZCJZSP49AP6G1PQ9K9",
"status": "resolved",
"updated_at": "2026-08-05T18:26:22Z"
},
{
"body": "Some models are intermittently slow or timing out due to a network issue at one of our infrastructure providers.\n\nThe issue comes in short bursts, and affected requests recover automatically once each burst passes. We are working with the provider on a fix and will post updates here.",
"created_at": "2026-08-05T15:51:33Z",
"display_at": "2026-08-05T15:51:33Z",
"id": "01KZ99YX6AMAAM2VZWC3CJKJG6",
"incident_id": "01KZ917SZCJZSP49AP6G1PQ9K9",
"status": "monitoring",
"updated_at": "2026-08-05T16:21:18Z"
},
{
"body": "We’ve migrated some of the consumers to a stable location. Queues are back to normal, we will keep monitoring this incident for a while",
"created_at": "2026-08-05T15:26:56Z",
"display_at": "2026-08-05T15:26:56Z",
"id": "01KZ98HTHG97BDT4MNFZGDNGFE",
"incident_id": "01KZ917SZCJZSP49AP6G1PQ9K9",
"status": "monitoring",
"updated_at": "2026-08-05T15:26:56Z"
},
{
"body": "We've identified the issue with our third-party provider and now preparing the remediation.",
"created_at": "2026-08-05T14:36:46Z",
"display_at": "2026-08-05T14:36:46Z",
"id": "01KZ95NZ1T0CJZ946JXNYZHREV",
"incident_id": "01KZ917SZCJZSP49AP6G1PQ9K9",
"status": "identified",
"updated_at": "2026-08-05T14:36:46Z"
},
{
"body": "The team is still investigating",
"created_at": "2026-08-05T13:46:27Z",
"display_at": "2026-08-05T13:46:27Z",
"id": "01KZ92SVC7RN9003AANHNFJK6A",
"incident_id": "01KZ917SZCJZSP49AP6G1PQ9K9",
"status": "identified",
"updated_at": "2026-08-05T13:46:27Z"
},
{
"body": "We're currently experiencing issues with some of the models.\nIssue sits on the infrastructure side, so team now is investigating.",
"created_at": "2026-08-05T13:19:07Z",
"display_at": "2026-08-05T13:19:07Z",
"id": "01KZ917SZCE3ZVF1CYBQC9RP3M",
"incident_id": "01KZ917SZCJZSP49AP6G1PQ9K9",
"status": "investigating",
"updated_at": "2026-08-05T13:19:07Z"
}
],
"name": "Some of the models has degraded performance or timeouts",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-08-05T18:26:22Z",
"status": "resolved",
"updated_at": "2026-08-05T18:26:22Z"
},
{
"created_at": "2026-08-04T15:58:51Z",
"id": "01KZ6QZJDY8N50WGH5313DZ8F2",
"impact": "minor",
"incident_updates": [
{
"body": "The incident now has been resolved.\nThe model DeepSeek-V4-Pro is now fully operational.",
"created_at": "2026-08-04T18:04:15Z",
"display_at": "2026-08-04T18:04:15Z",
"id": "01KZ6Z55ETX523ENW1AGN5C039",
"incident_id": "01KZ6QZJDY8N50WGH5313DZ8F2",
"status": "resolved",
"updated_at": "2026-08-04T18:04:15Z"
},
{
"body": "The node has been replaced.\nThe model is currently starting. Expect delay 10-30 mins",
"created_at": "2026-08-04T17:30:02Z",
"display_at": "2026-08-04T17:30:02Z",
"id": "01KZ6X6GTA56GS3AJB3WGSPFKZ",
"incident_id": "01KZ6QZJDY8N50WGH5313DZ8F2",
"status": "monitoring",
"updated_at": "2026-08-04T17:30:02Z"
},
{
"body": "The provider still hasn't fixed the node issue.\nOur team is finishing the node switch",
"created_at": "2026-08-04T17:16:37Z",
"display_at": "2026-08-04T17:16:37Z",
"id": "01KZ6WDYZHVP5Z961SQCNN8WJ5",
"incident_id": "01KZ6QZJDY8N50WGH5313DZ8F2",
"status": "identified",
"updated_at": "2026-08-04T17:16:37Z"
},
{
"body": "The provider still working on this issue\nOur team now is working to change the affected node",
"created_at": "2026-08-04T16:44:03Z",
"display_at": "2026-08-04T16:44:03Z",
"id": "01KZ6TJABBE6EZNQCZNGBSQ0YT",
"incident_id": "01KZ6QZJDY8N50WGH5313DZ8F2",
"status": "identified",
"updated_at": "2026-08-04T16:44:03Z"
},
{
"body": "We've identified the issue and it is associated with an up-stream DC provider.\nThe team now is working with to bring the GPU node back online.",
"created_at": "2026-08-04T16:28:47Z",
"display_at": "2026-08-04T16:28:47Z",
"id": "01KZ6SPBXCF7QV4AQX6WJV8QKE",
"incident_id": "01KZ6QZJDY8N50WGH5313DZ8F2",
"status": "identified",
"updated_at": "2026-08-04T16:28:47Z"
},
{
"body": "We are experiencing a timeouts with DeepSeek-V4-Pro vLLM model. The team are investigating. Updates will follow.",
"created_at": "2026-08-04T15:58:51Z",
"display_at": "2026-08-04T15:58:51Z",
"id": "01KZ6QZJDYDJBTJRCE3NPEJQAQ",
"incident_id": "01KZ6QZJDY8N50WGH5313DZ8F2",
"status": "investigating",
"updated_at": "2026-08-04T15:58:51Z"
}
],
"name": "DeepSeek-V4-Pro vLLM issue",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-08-04T18:04:15Z",
"status": "resolved",
"updated_at": "2026-08-04T18:04:15Z"
},
{
"created_at": "2026-07-31T17:38:40Z",
"id": "01KYWM3F609S471WSANTZQ5DEV",
"impact": "minor",
"incident_updates": [
{
"body": "queues are stable, we will continue to add consumers to the queue. Incident resolved.",
"created_at": "2026-07-31T18:06:39Z",
"display_at": "2026-07-31T18:06:39Z",
"id": "01KYWNPP7ZMEE9CM0TR1Q12KKA",
"incident_id": "01KYWM3F609S471WSANTZQ5DEV",
"status": "resolved",
"updated_at": "2026-07-31T18:06:39Z"
},
{
"body": "We have increased traffic towards flux-2-klein models. Adjusting the consumer capacity to match the traffic pattern.",
"created_at": "2026-07-31T17:38:41Z",
"display_at": "2026-07-31T17:38:40Z",
"id": "01KYWM3F60XPCE92WFP8KS13EN",
"incident_id": "01KYWM3F609S471WSANTZQ5DEV",
"status": "identified",
"updated_at": "2026-07-31T17:38:41Z"
}
],
"name": "Flux2klein - degraded performance",
"page_id": "01KSPXEHBNHPMWQ91TASD8370H",
"resolved_at": "2026-07-31T18:06:39Z",
"status": "resolved",
"updated_at": "2026-07-31T18:06:39Z"
}
],
"page": {
"id": "01KSPXEHBNHPMWQ91TASD8370H",
"name": "Runware",
"updated_at": "2026-11-19T11:14:40Z",
"url": "https://status.runware.ai/"
}
}