Snapshot 22906
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"incidents": [
{
"created_at": "2026-09-30T09:10:17Z",
"id": "01M3RS8E0MFN2SWBTNES3N7EBE",
"impact": "major",
"incident_updates": [
{
"body": "All projects ingesting as normal",
"created_at": "2026-09-30T15:30:23Z",
"display_at": "2026-09-30T15:30:23Z",
"id": "01M3SF0CX1CF8CCMBDEFKCRG85",
"incident_id": "01M3RS8E0MFN2SWBTNES3N7EBE",
"status": "resolved",
"updated_at": "2026-09-30T15:30:23Z"
},
{
"body": "Offending workload was isolated, performance should be back to normal for other projects",
"created_at": "2026-09-30T09:43:12Z",
"display_at": "2026-09-30T09:43:12Z",
"id": "01M3RV4PSHSQ13N7WPWC3BSTTJ",
"incident_id": "01M3RS8E0MFN2SWBTNES3N7EBE",
"status": "monitoring",
"updated_at": "2026-09-30T09:43:12Z"
},
{
"body": "S3 backpressure leads to S3 \"please slow down\" errors across projects with cm* prefix",
"created_at": "2026-09-30T09:10:17Z",
"display_at": "2026-09-30T09:10:17Z",
"id": "01M3RS8E0MTDNJJ677GM50CRVV",
"incident_id": "01M3RS8E0MFN2SWBTNES3N7EBE",
"status": "identified",
"updated_at": "2026-09-30T09:10:17Z"
}
],
"name": "Degraded ingestion for 20 projects",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-09-30T15:30:23Z",
"status": "resolved",
"updated_at": "2026-09-30T15:30:23Z"
},
{
"created_at": "2026-09-23T13:36:59Z",
"id": "01M377QR1YYCPPY6PHR89DFGSC",
"impact": "minor",
"incident_updates": [
{
"body": "The issue has been resolved.",
"created_at": "2026-09-23T13:43:42Z",
"display_at": "2026-09-23T13:43:42Z",
"id": "01M37840NDD0Y9EXD6M1JSV9BZ",
"incident_id": "01M377QR1YYCPPY6PHR89DFGSC",
"status": "resolved",
"updated_at": "2026-09-23T13:43:42Z"
},
{
"body": "We're observing elevated error rates across our In-App agent across multiple regions. Our team is investigating.",
"created_at": "2026-09-23T13:36:59Z",
"display_at": "2026-09-23T13:36:59Z",
"id": "01M377QR1YMS6EVKF1KDG706GX",
"incident_id": "01M377QR1YYCPPY6PHR89DFGSC",
"status": "investigating",
"updated_at": "2026-09-23T13:36:59Z"
}
],
"name": "Elevated error rates in In App Agent",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-09-23T13:43:42Z",
"status": "resolved",
"updated_at": "2026-09-23T13:43:42Z"
},
{
"created_at": "2026-09-16T20:20:04Z",
"id": "01M2NY0R9X77QMKQ6T4S6CSA3E",
"impact": "minor",
"incident_updates": [
{
"body": "Issue is resolved and OTEL ingestion pipeline is back to regular ingestion speed.",
"created_at": "2026-09-16T21:31:38Z",
"display_at": "2026-09-16T21:31:38Z",
"id": "01M2P23SQYR5DKMEXADFNJJ9KH",
"incident_id": "01M2NY0R9X77QMKQ6T4S6CSA3E",
"status": "resolved",
"updated_at": "2026-09-16T21:31:38Z"
},
{
"body": "We're investigating degraded performance in the ingestion flow",
"created_at": "2026-09-16T20:20:04Z",
"display_at": "2026-09-16T20:20:04Z",
"id": "01M2NY0R9XNN45ZGN6SHX6MTE5",
"incident_id": "01M2NY0R9X77QMKQ6T4S6CSA3E",
"status": "investigating",
"updated_at": "2026-09-16T20:20:04Z"
}
],
"name": "OTEL ingestion queue delay over 10 minutes in US",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-09-16T21:31:38Z",
"status": "resolved",
"updated_at": "2026-09-16T21:31:38Z"
},
{
"created_at": "2026-09-14T02:51:34Z",
"id": "01M2EX7FAKVNNVRJ7M6GZK665P",
"impact": "minor",
"incident_updates": [
{
"body": "The incident has been resolved.",
"created_at": "2026-09-14T06:10:22Z",
"display_at": "2026-09-14T06:10:22Z",
"id": "01M2F8KFKW5VQDHF5QAGP18P6X",
"incident_id": "01M2EX7FAKVNNVRJ7M6GZK665P",
"status": "resolved",
"updated_at": "2026-09-14T06:10:22Z"
},
{
"body": "We're making consistent progress in our backlog and are currently seeing delays of about 60 minutes.",
"created_at": "2026-09-14T05:14:36Z",
"display_at": "2026-09-14T05:14:36Z",
"id": "01M2F5DBRYYTMT9WHD6MP1B3AD",
"incident_id": "01M2EX7FAKVNNVRJ7M6GZK665P",
"status": "monitoring",
"updated_at": "2026-09-14T05:14:36Z"
},
{
"body": "We considerably increased throughput of the batch job to write data into v4 data model. We are currently catching up processing data.",
"created_at": "2026-09-14T03:52:08Z",
"display_at": "2026-09-14T03:52:08Z",
"id": "01M2F0PCCMQF1MA8JCYKGR35FR",
"incident_id": "01M2EX7FAKVNNVRJ7M6GZK665P",
"status": "investigating",
"updated_at": "2026-09-14T03:52:08Z"
},
{
"body": "We currently experience an ingestion delay of old SDKs by 3 hours. This is due to performance issues in our batch job which moves data into the v4 data model (\u003chttps://langfuse.com/changelog/2026-08-17-langfuse-v4\u003e).\n\n \n\nWe currently work on improving the batch job. All v4 compatible SDKs are not impacted by this. Upgrading SDKs is recommended.",
"created_at": "2026-09-14T02:51:34Z",
"display_at": "2026-09-14T02:51:34Z",
"id": "01M2EX7FAK8KVCZZGSB46WV64C",
"incident_id": "01M2EX7FAKVNNVRJ7M6GZK665P",
"status": "investigating",
"updated_at": "2026-09-14T02:51:34Z"
}
],
"name": "Delayed Ingestion for non v4 SDKs",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-09-14T06:10:22Z",
"status": "resolved",
"updated_at": "2026-09-14T06:10:22Z"
},
{
"created_at": "2026-09-08T17:26:03Z",
"id": "01M210WCHAJ4WKZWDBM4E9XYF5",
"impact": "minor",
"incident_updates": [
{
"body": "OTEL ingestion latency spike is resolved",
"created_at": "2026-09-08T19:00:55Z",
"display_at": "2026-09-08T19:00:55Z",
"id": "01M216A3BXTXGJDE2BET38TRQM",
"incident_id": "01M210WCHAJ4WKZWDBM4E9XYF5",
"status": "resolved",
"updated_at": "2026-09-08T19:00:55Z"
},
{
"body": "We're investigating degraded performance in the ingestion flow",
"created_at": "2026-09-08T17:26:03Z",
"display_at": "2026-09-08T17:26:03Z",
"id": "01M210WCHACAXQCF2TATV3T250",
"incident_id": "01M210WCHAJ4WKZWDBM4E9XYF5",
"status": "investigating",
"updated_at": "2026-09-08T17:26:03Z"
}
],
"name": "OTEL Ingestion Delay exceeds 10 minutes",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-09-08T19:00:55Z",
"status": "resolved",
"updated_at": "2026-09-08T19:00:55Z"
},
{
"created_at": "2026-09-02T16:02:22Z",
"id": "01M1HDPTQA3FVQTG8FZWA7DQWG",
"impact": "minor",
"incident_updates": [
{
"body": "Completely back to normal",
"created_at": "2026-09-02T16:56:09Z",
"display_at": "2026-09-02T16:56:09Z",
"id": "01M1HGSAKR93VB14VHZ31D9KDM",
"incident_id": "01M1HDPTQA3FVQTG8FZWA7DQWG",
"status": "resolved",
"updated_at": "2026-09-02T16:56:09Z"
},
{
"body": "Backlog is almost completely gone. Ingestion delay is \u003c 1 min.",
"created_at": "2026-09-02T16:49:28Z",
"display_at": "2026-09-02T16:49:28Z",
"id": "01M1HGD2X2YP13V1NG4JW7WXC8",
"incident_id": "01M1HDPTQA3FVQTG8FZWA7DQWG",
"status": "monitoring",
"updated_at": "2026-09-02T16:49:28Z"
},
{
"body": "A deployment rollout coincided with a traffic spike, resulting in an unusually large backlog. Adjusting our autoscaling to absorb the backlog faster.",
"created_at": "2026-09-02T16:24:35Z",
"display_at": "2026-09-02T16:24:35Z",
"id": "01M1HEZGW1JFZBNSDNS4Q4XHGE",
"incident_id": "01M1HDPTQA3FVQTG8FZWA7DQWG",
"status": "identified",
"updated_at": "2026-09-02T16:24:35Z"
},
{
"body": "We are observing OTEL ingestion backlog in US region",
"created_at": "2026-09-02T16:02:22Z",
"display_at": "2026-09-02T16:02:22Z",
"id": "01M1HDPTQAKPFXXZB1MB1V2NQF",
"incident_id": "01M1HDPTQA3FVQTG8FZWA7DQWG",
"status": "investigating",
"updated_at": "2026-09-02T16:02:22Z"
}
],
"name": "Degraded OTEL ingestion in US region",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-09-02T16:56:09Z",
"status": "resolved",
"updated_at": "2026-09-02T16:56:09Z"
},
{
"created_at": "2026-08-25T15:23:47Z",
"id": "01M0WRAE2W2JVQWVHE9K03SS8Z",
"impact": "minor",
"incident_updates": [
{
"body": "Issue resolved. Latency is back to normal levels.",
"created_at": "2026-08-25T16:28:41Z",
"display_at": "2026-08-25T16:28:41Z",
"id": "01M0WW1987QQDRX8PYEFPNTKCJ",
"incident_id": "01M0WRAE2W2JVQWVHE9K03SS8Z",
"status": "resolved",
"updated_at": "2026-08-25T16:28:41Z"
},
{
"body": "Ingestion delay of up to multiple minutes on the ingestion path. We have scaled up and are investigating the root cause.",
"created_at": "2026-08-25T15:23:47Z",
"display_at": "2026-08-25T15:23:47Z",
"id": "01M0WRAE2WZXR7SWY2C5T6A6FK",
"incident_id": "01M0WRAE2W2JVQWVHE9K03SS8Z",
"status": "monitoring",
"updated_at": "2026-08-25T15:23:47Z"
}
],
"name": "Performance degradation on ingestion path",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-25T16:28:41Z",
"status": "resolved",
"updated_at": "2026-08-25T16:28:41Z"
},
{
"created_at": "2026-08-25T10:16:41Z",
"id": "01M0W6R42TJZ789CC0QNQG971R",
"impact": "minor",
"incident_updates": [
{
"body": "Issue resolved since 12:30. Legacy ingestion pipeline is fully operational. ",
"created_at": "2026-08-25T13:42:56Z",
"display_at": "2026-08-25T13:42:56Z",
"id": "01M0WJHRYBTDJGP6PQVRQR019G",
"incident_id": "01M0W6R42TJZ789CC0QNQG971R",
"status": "resolved",
"updated_at": "2026-08-25T13:42:56Z"
},
{
"body": "The legacy ingestion queue is experiencing latency degradation up to 5 minutes. We have scaled up our infrastructure in response and are monitoring the situation.\n\nWe recommend to switch to the new OTEL ingestion queue.",
"created_at": "2026-08-25T10:16:41Z",
"display_at": "2026-08-25T10:16:41Z",
"id": "01M0W6R42TN2XT35JSHN37GPTZ",
"incident_id": "01M0W6R42TJZ789CC0QNQG971R",
"status": "monitoring",
"updated_at": "2026-08-25T10:16:41Z"
}
],
"name": "Latency degradation on legacy ingestion queue",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-25T13:42:56Z",
"status": "resolved",
"updated_at": "2026-08-25T13:42:56Z"
},
{
"created_at": "2026-08-20T08:06:36Z",
"id": "01M0F3AAW0WVSYJNF6A2JDW5TM",
"impact": "minor",
"incident_updates": [
{
"body": "All is back to normal",
"created_at": "2026-08-20T09:55:50Z",
"display_at": "2026-08-20T09:55:50Z",
"id": "01M0F9JBJZ23QHHF3M85NSJP7H",
"incident_id": "01M0F3AAW0WVSYJNF6A2JDW5TM",
"status": "resolved",
"updated_at": "2026-08-20T09:55:50Z"
},
{
"body": "After concurrency adjustments execution delay is back to normal. Monitoring.",
"created_at": "2026-08-20T09:40:33Z",
"display_at": "2026-08-20T09:40:33Z",
"id": "01M0F8PBK118XWVRXGKWNQSH37",
"incident_id": "01M0F3AAW0WVSYJNF6A2JDW5TM",
"status": "monitoring",
"updated_at": "2026-08-20T09:40:33Z"
},
{
"body": "Evaluations are taking longer to execute in EU",
"created_at": "2026-08-20T08:06:36Z",
"display_at": "2026-08-20T08:06:36Z",
"id": "01M0F3AAW01HCK8V626ED8ZNXZ",
"incident_id": "01M0F3AAW0WVSYJNF6A2JDW5TM",
"status": "investigating",
"updated_at": "2026-08-20T08:06:36Z"
}
],
"name": "Evaluations are delayed in EU",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-20T09:55:50Z",
"status": "resolved",
"updated_at": "2026-08-20T09:55:50Z"
},
{
"created_at": "2026-08-19T12:36:48Z",
"id": "01M0D0CCF6N91A79YYMFBGB9WJ",
"impact": "minor",
"incident_updates": [
{
"body": "Back to normal propagation delay",
"created_at": "2026-08-19T15:10:42Z",
"display_at": "2026-08-19T15:10:42Z",
"id": "01M0D965ZEYN4QTBSF9VEFEZX4",
"incident_id": "01M0D0CCF6N91A79YYMFBGB9WJ",
"status": "resolved",
"updated_at": "2026-08-19T15:10:42Z"
},
{
"body": "Latest major SDK versions are not impacted. Find upgrade docs [here](https://langfuse.com/docs/observability/sdk/upgrade-path).",
"created_at": "2026-08-19T13:19:25Z",
"display_at": "2026-08-19T13:19:25Z",
"id": "01M0D2TD6GF8AX0A2TF2D56B3J",
"incident_id": "01M0D0CCF6N91A79YYMFBGB9WJ",
"status": "monitoring",
"updated_at": "2026-08-19T14:12:42Z"
},
{
"body": "We have scaled our database infrastructure to match the throughput demand",
"created_at": "2026-08-19T12:36:48Z",
"display_at": "2026-08-19T10:30:00Z",
"id": "01M0D0CCF6TN370VDK0F1334WY",
"incident_id": "01M0D0CCF6N91A79YYMFBGB9WJ",
"status": "monitoring",
"updated_at": "2026-08-19T12:37:03Z"
}
],
"name": "Longer v3 -\u003e v4 propagation delay",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-19T15:10:42Z",
"status": "resolved",
"updated_at": "2026-08-19T15:10:42Z"
},
{
"created_at": "2026-08-18T12:06:47Z",
"id": "01M0AC8PM24HN9TVHRWR47G2R2",
"impact": "minor",
"incident_updates": [
{
"body": "Steady throughput for the last 30 minutes. Delay \u003c 5min.",
"created_at": "2026-08-18T13:25:37Z",
"display_at": "2026-08-18T13:25:37Z",
"id": "01M0AGS1YZ17VSP0S4EPZTEP8E",
"incident_id": "01M0AC8PM24HN9TVHRWR47G2R2",
"status": "resolved",
"updated_at": "2026-08-18T13:25:37Z"
},
{
"body": "our autoscaling responded slower than expected, but responded nonetheless. the backlog is draining",
"created_at": "2026-08-18T12:47:41Z",
"display_at": "2026-08-18T12:47:41Z",
"id": "01M0AEKK5R9Y76R70E6TEQ4VBQ",
"incident_id": "01M0AC8PM24HN9TVHRWR47G2R2",
"status": "monitoring",
"updated_at": "2026-08-18T12:47:41Z"
},
{
"body": "evaluation throughput was overwhelmed by a slow tenant/model workload, with lock-expiry retries multiplying work; the concurrent worker rollout prolonged the incident.",
"created_at": "2026-08-18T12:27:29Z",
"display_at": "2026-08-18T12:27:29Z",
"id": "01M0ADEK953AT42N5FWAMQTYV1",
"incident_id": "01M0AC8PM24HN9TVHRWR47G2R2",
"status": "identified",
"updated_at": "2026-08-18T12:27:29Z"
},
{
"body": "Eval execution in EU region is taking longer than usual",
"created_at": "2026-08-18T12:06:47Z",
"display_at": "2026-08-18T12:06:47Z",
"id": "01M0AC8PM22B1XPDDYYT9ADAFV",
"incident_id": "01M0AC8PM24HN9TVHRWR47G2R2",
"status": "investigating",
"updated_at": "2026-08-18T12:06:47Z"
}
],
"name": "Degraded LLMaJ execution in EU",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-18T13:25:37Z",
"status": "resolved",
"updated_at": "2026-08-18T13:25:37Z"
},
{
"created_at": "2026-08-15T17:28:52Z",
"id": "01M037G9MDS6PM7JNHCWZBFK89",
"impact": "minor",
"incident_updates": [
{
"body": "We incurred a temporary delay on the legacy ingestion path for v4. This delay has been resolved.\n\nOTEL ingestion has not been impacted.",
"created_at": "2026-08-15T18:05:17Z",
"display_at": "2026-08-15T18:05:17Z",
"id": "01M039JZDRHTVDBTQH9VP9CYCE",
"incident_id": "01M037G9MDS6PM7JNHCWZBFK89",
"status": "resolved",
"updated_at": "2026-08-15T18:05:17Z"
},
{
"body": "Currently the ingestion for the events table is delayed in the US environment. This happens only for data from non-latest major SDK versions.\n\nWe are investigating this and have started a mitigation. The data should be up to date soon.",
"created_at": "2026-08-15T17:28:52Z",
"display_at": "2026-08-15T17:28:52Z",
"id": "01M037G9MD3EJ2AV9RMHRQZW24",
"incident_id": "01M037G9MDS6PM7JNHCWZBFK89",
"status": "investigating",
"updated_at": "2026-08-15T17:53:15Z"
}
],
"name": "prod-us: Delayed ingestion for prod-us",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-15T18:05:17Z",
"status": "resolved",
"updated_at": "2026-08-15T18:05:17Z"
},
{
"created_at": "2026-08-12T11:56:38Z",
"id": "01KZTX9SG4T9GAZMQ2KGWWD649",
"impact": "minor",
"incident_updates": [
{
"body": "The issue is resolved now",
"created_at": "2026-08-12T13:37:15Z",
"display_at": "2026-08-12T13:37:15Z",
"id": "01KZV320W3KSNMTE0QZQKY1C0E",
"incident_id": "01KZTX9SG4T9GAZMQ2KGWWD649",
"status": "resolved",
"updated_at": "2026-08-12T13:37:15Z"
},
{
"body": "Previous latency increase of the eval execution is resolved. Slight delays may still occur.",
"created_at": "2026-08-12T12:39:18Z",
"display_at": "2026-08-12T12:39:18Z",
"id": "01KZTZQX6JW1HKK0R8T1ZDVZ73",
"incident_id": "01KZTX9SG4T9GAZMQ2KGWWD649",
"status": "monitoring",
"updated_at": "2026-08-12T12:39:18Z"
},
{
"body": "We are seeing an increase in the eval execution. We have scaled up our infrastructure and are monitoring.",
"created_at": "2026-08-12T11:56:38Z",
"display_at": "2026-08-12T11:56:38Z",
"id": "01KZTX9SG4SVE22V669RPWBT8K",
"incident_id": "01KZTX9SG4T9GAZMQ2KGWWD649",
"status": "monitoring",
"updated_at": "2026-08-12T11:56:38Z"
}
],
"name": "Latency increase eval execution up to 40 minutes is resolved",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-12T13:37:15Z",
"status": "resolved",
"updated_at": "2026-08-12T13:37:15Z"
},
{
"created_at": "2026-08-03T18:25:40Z",
"id": "01KZ4DZNVKPJ4Z40WVCGJAMPT4",
"impact": "minor",
"incident_updates": [
{
"body": "This issue is resolved and we're back to regular processing times.",
"created_at": "2026-08-03T19:03:40Z",
"display_at": "2026-08-03T19:03:40Z",
"id": "01KZ4G58330P700D3FS9V485D2",
"incident_id": "01KZ4DZNVKPJ4Z40WVCGJAMPT4",
"status": "resolved",
"updated_at": "2026-08-03T19:03:40Z"
},
{
"body": "We see an improvement and the current delay is below 5min. We continue to investigate and scale.",
"created_at": "2026-08-03T18:56:58Z",
"display_at": "2026-08-03T18:56:58Z",
"id": "01KZ4FRZV9GZ0674T9WGJHA7YR",
"incident_id": "01KZ4DZNVKPJ4Z40WVCGJAMPT4",
"status": "investigating",
"updated_at": "2026-08-03T18:56:58Z"
},
{
"body": "We're currently observing about 10min ingestion delays. Our team is investigating.",
"created_at": "2026-08-03T18:25:40Z",
"display_at": "2026-08-03T18:25:40Z",
"id": "01KZ4DZNVK7HC9A8VRMPXX50WF",
"incident_id": "01KZ4DZNVKPJ4Z40WVCGJAMPT4",
"status": "investigating",
"updated_at": "2026-08-03T18:25:40Z"
}
],
"name": "[US] Ingestion Delays",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-08-03T19:03:40Z",
"status": "resolved",
"updated_at": "2026-08-03T19:03:40Z"
},
{
"created_at": "2026-07-14T19:12:41Z",
"id": "01KXH0QCAJ354J5ETHE76SNG49",
"impact": "minor",
"incident_updates": [
{
"body": "We resolved the performance bottleneck. All APIs are responding as expected as of now.",
"created_at": "2026-07-14T20:19:30Z",
"display_at": "2026-07-14T20:19:30Z",
"id": "01KXH4HQHRJ0G4RGTYPRC1W3FW",
"incident_id": "01KXH0QCAJ354J5ETHE76SNG49",
"status": "resolved",
"updated_at": "2026-07-14T20:19:30Z"
},
{
"body": "We experience occasional API errors with 502 Bad Gateway. The team is investigating the root cause currently.",
"created_at": "2026-07-14T19:12:41Z",
"display_at": "2026-07-14T19:12:41Z",
"id": "01KXH0QCAJ55DQDH42YBBSRFS2",
"incident_id": "01KXH0QCAJ354J5ETHE76SNG49",
"status": "investigating",
"updated_at": "2026-07-14T19:12:41Z"
}
],
"name": "502 BAD Gateway errors on UI",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-07-14T20:19:30Z",
"status": "resolved",
"updated_at": "2026-07-14T20:19:30Z"
},
{
"created_at": "2026-07-02T07:43:01Z",
"id": "01KWGWFYKT9X16PY7090RSYWYE",
"impact": "minor",
"incident_updates": [
{
"body": "The issue has been resolved.",
"created_at": "2026-07-02T08:03:36Z",
"display_at": "2026-07-02T08:03:36Z",
"id": "01KWGXNMR21M3YN2Y3XVQGV9E0",
"incident_id": "01KWGWFYKT9X16PY7090RSYWYE",
"status": "resolved",
"updated_at": "2026-07-02T08:03:36Z"
},
{
"body": "We're observing elevated latencies and error rates across read APIs and some frontend routes. Ingestion is not affected.",
"created_at": "2026-07-02T07:43:01Z",
"display_at": "2026-07-02T07:43:01Z",
"id": "01KWGWFYKTKFVTDDH7AXGM0E56",
"incident_id": "01KWGWFYKT9X16PY7090RSYWYE",
"status": "investigating",
"updated_at": "2026-07-02T07:43:01Z"
}
],
"name": "[EU] Elevated latencies and error rates",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-07-02T08:03:36Z",
"status": "resolved",
"updated_at": "2026-07-02T08:03:36Z"
},
{
"created_at": "2026-06-26T09:32:34Z",
"id": "01KW1MC79474ZC9EJ6NZAQ7T0F",
"impact": "minor",
"incident_updates": [
{
"body": "The incident has been resolved.",
"created_at": "2026-06-26T09:51:12Z",
"display_at": "2026-06-26T09:51:12Z",
"id": "01KW1NEANFZK250APB2ZZ6BZPF",
"incident_id": "01KW1MC79474ZC9EJ6NZAQ7T0F",
"status": "resolved",
"updated_at": "2026-06-26T09:51:12Z"
},
{
"body": "We found a patch and currently work through the backlog. Things should recover within the next 10min.",
"created_at": "2026-06-26T09:43:54Z",
"display_at": "2026-06-26T09:43:54Z",
"id": "01KW1N0YZQ0GF3K18SDRGZTAV3",
"incident_id": "01KW1MC79474ZC9EJ6NZAQ7T0F",
"status": "monitoring",
"updated_at": "2026-06-26T09:43:54Z"
},
{
"body": "We're currently experiencing elevated ingestion times in the EU environment. Some events are processed with a delay of about 15 minutes. Our team is investigating.",
"created_at": "2026-06-26T09:32:34Z",
"display_at": "2026-06-26T09:32:34Z",
"id": "01KW1MC794KC93Y0K9Z919NPD0",
"incident_id": "01KW1MC79474ZC9EJ6NZAQ7T0F",
"status": "investigating",
"updated_at": "2026-06-26T09:32:34Z"
}
],
"name": "[EU] Elevated ingestion times",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-06-26T09:51:12Z",
"status": "resolved",
"updated_at": "2026-06-26T09:51:12Z"
},
{
"created_at": "2026-06-16T17:36:00Z",
"id": "01KV8R27KJ44QJGY5R958J3Q0J",
"impact": "minor",
"incident_updates": [
{
"body": "All latencies are back to normal",
"created_at": "2026-06-16T18:12:46Z",
"display_at": "2026-06-16T18:12:46Z",
"id": "01KV8T5HR2H9C7KV7NHKMDM9AX",
"incident_id": "01KV8R27KJ44QJGY5R958J3Q0J",
"status": "resolved",
"updated_at": "2026-06-16T18:12:46Z"
},
{
"body": "A hiccup during a planned database maintenance. Identified and resolved. Presently monitoring.",
"created_at": "2026-06-16T17:36:00Z",
"display_at": "2026-06-16T17:36:00Z",
"id": "01KV8R27KJXTSP800AGSRZB98D",
"incident_id": "01KV8R27KJ44QJGY5R958J3Q0J",
"status": "monitoring",
"updated_at": "2026-06-16T17:36:00Z"
}
],
"name": "Elevated API and tracing latency in the US region",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-06-16T18:12:46Z",
"status": "resolved",
"updated_at": "2026-06-16T18:12:46Z"
},
{
"created_at": "2026-06-10T14:47:23Z",
"id": "01KTS015KHNRNVR87Y5Y5BGHQ6",
"impact": "minor",
"incident_updates": [
{
"body": "We scaled our ingestion and are processing data in time.",
"created_at": "2026-06-10T15:16:21Z",
"display_at": "2026-06-10T15:16:21Z",
"id": "01KTS1P6NWV2P2WWAPE4YH0QRY",
"incident_id": "01KTS015KHNRNVR87Y5Y5BGHQ6",
"status": "resolved",
"updated_at": "2026-06-10T15:16:21Z"
},
{
"body": "We currently have a bottleneck in our ingestion pipeline which causes a delay of ingested data of 12 minutes. We currently work to scale the pipeline.",
"created_at": "2026-06-10T14:47:23Z",
"display_at": "2026-06-10T14:47:23Z",
"id": "01KTS015KH7GM8WT765VT0ZEKR",
"incident_id": "01KTS015KHNRNVR87Y5Y5BGHQ6",
"status": "investigating",
"updated_at": "2026-06-10T14:47:23Z"
}
],
"name": "Ingestion delay",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-06-10T15:16:21Z",
"status": "resolved",
"updated_at": "2026-06-10T15:16:21Z"
},
{
"created_at": "2026-06-08T20:55:30Z",
"id": "01KTMG9RPS9AV1T412GFAN7QWD",
"impact": "minor",
"incident_updates": [
{
"body": "Upgrade successfully resolved.\n",
"created_at": "2026-06-08T20:55:44Z",
"display_at": "2026-06-08T20:55:44Z",
"id": "01KTMGA6EGYXWMYBF7JZ3SHSE7",
"incident_id": "01KTMG9RPS9AV1T412GFAN7QWD",
"status": "resolved",
"updated_at": "2026-06-08T20:55:44Z"
},
{
"body": "We just executed a database upgrade which caused a downtime of 5 minutes.",
"created_at": "2026-06-08T20:55:30Z",
"display_at": "2026-06-08T20:55:30Z",
"id": "01KTMG9RPS814TH6YQ4KE9X7EQ",
"incident_id": "01KTMG9RPS9AV1T412GFAN7QWD",
"status": "investigating",
"updated_at": "2026-06-08T20:55:30Z"
}
],
"name": "Database upgrdae",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-06-08T20:55:44Z",
"status": "resolved",
"updated_at": "2026-06-08T20:55:44Z"
},
{
"created_at": "2026-06-08T18:57:36Z",
"id": "01KTM9HWN7GW0M4Z4E1GXXYXE4",
"impact": "major",
"incident_updates": [
{
"body": "We scaled our infrastructure and are processing media uploads without errors.",
"created_at": "2026-06-10T14:45:43Z",
"display_at": "2026-06-08T19:55:00Z",
"id": "01KTRZY3T0JY77V0XR01T24VXM",
"incident_id": "01KTM9HWN7GW0M4Z4E1GXXYXE4",
"status": "resolved",
"updated_at": "2026-06-10T14:46:20Z"
},
{
"body": "We scaled our underlying infrastructure and are processing media uploads without errors.",
"created_at": "2026-06-08T19:54:31Z",
"display_at": "2026-06-08T19:54:31Z",
"id": "01KTMCT3ASDC0KHD2D535MW0WT",
"incident_id": "01KTM9HWN7GW0M4Z4E1GXXYXE4",
"status": "investigating",
"updated_at": "2026-06-08T19:54:31Z"
},
{
"body": "We observe that our api/public/media endpoint currently returns 500 status errors as our underlying database has performance issues.",
"created_at": "2026-06-08T18:57:36Z",
"display_at": "2026-06-08T18:57:36Z",
"id": "01KTM9HWN7KG00C17NP0FBZ0PM",
"incident_id": "01KTM9HWN7GW0M4Z4E1GXXYXE4",
"status": "investigating",
"updated_at": "2026-06-08T18:57:36Z"
}
],
"name": "Our media API returns 500 errors",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-06-08T19:55:00Z",
"status": "resolved",
"updated_at": "2026-06-10T14:45:43Z"
},
{
"created_at": "2026-06-02T15:44:12Z",
"id": "01KT4G3EJ32XHSVCA1GWWXC6Q5",
"impact": "minor",
"incident_updates": [
{
"body": "This issue has been resolved.",
"created_at": "2026-06-02T16:55:53Z",
"display_at": "2026-06-02T16:55:53Z",
"id": "01KT4M6Q48MW955XFNWBN037QS",
"incident_id": "01KT4G3EJ32XHSVCA1GWWXC6Q5",
"status": "resolved",
"updated_at": "2026-06-02T16:55:53Z"
},
{
"body": "We've changed infrastructure capacity and observe that this mitigates the problems. We continue to observe the situation.",
"created_at": "2026-06-02T15:56:49Z",
"display_at": "2026-06-02T15:56:49Z",
"id": "01KT4GTJ5FV9X7RB4XJGAG3K8Y",
"incident_id": "01KT4G3EJ32XHSVCA1GWWXC6Q5",
"status": "monitoring",
"updated_at": "2026-06-02T15:56:49Z"
},
{
"body": "We're investigating elevated loading times within the Langfuse UI and our APIs.",
"created_at": "2026-06-02T15:44:12Z",
"display_at": "2026-06-02T15:44:12Z",
"id": "01KT4G3EJ3Z6KP9V88R40ARNR5",
"incident_id": "01KT4G3EJ32XHSVCA1GWWXC6Q5",
"status": "investigating",
"updated_at": "2026-06-02T15:44:12Z"
}
],
"name": "[EU] Elevated latencies in UI",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-06-02T16:55:53Z",
"status": "resolved",
"updated_at": "2026-06-02T16:55:53Z"
},
{
"created_at": "2026-06-01T11:53:57Z",
"id": "01KT1GH4K1EESQTRGS4RGBDGQ9",
"impact": "minor",
"incident_updates": [
{
"body": "The underlying incident with our infrastructure (https://statuspage.incident.io/clickhousecloud/incidents/01KT1G25S9PBKM7VJEB146680G) has been resolved.",
"created_at": "2026-06-01T14:13:26Z",
"display_at": "2026-06-01T14:13:26Z",
"id": "01KT1RGHBYTYVETESVG4JMRJEF",
"incident_id": "01KT1GH4K1EESQTRGS4RGBDGQ9",
"status": "resolved",
"updated_at": "2026-06-01T14:13:26Z"
},
{
"body": "We're investigating an issue with elevated latencies and error rates in the EU environment. ",
"created_at": "2026-06-01T11:53:57Z",
"display_at": "2026-06-01T11:53:57Z",
"id": "01KT1GH4K15JRMZNTBRA36XDA3",
"incident_id": "01KT1GH4K1EESQTRGS4RGBDGQ9",
"status": "investigating",
"updated_at": "2026-06-01T11:53:57Z"
}
],
"name": "[EU] Elevated error rates and delayed inserts",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-06-01T14:13:26Z",
"status": "resolved",
"updated_at": "2026-06-01T14:13:26Z"
},
{
"created_at": "2026-05-28T16:14:27Z",
"id": "01KSQNV81DX0PQQTSJX6FAM1E4",
"impact": "minor",
"incident_updates": [
{
"body": "We've fully caught up and process LLM as a judge in realtime again.",
"created_at": "2026-05-28T17:13:36Z",
"display_at": "2026-05-28T17:13:36Z",
"id": "01KSQS7HRS902NDAHD9YFGDGFS",
"incident_id": "01KSQNV81DX0PQQTSJX6FAM1E4",
"status": "resolved",
"updated_at": "2026-05-28T17:13:36Z"
},
{
"body": "We're currently investigating delays in our LLM as a judge execution times.",
"created_at": "2026-05-28T16:14:27Z",
"display_at": "2026-05-28T16:14:27Z",
"id": "01KSQNV81D0G240MFA3ESM02BG",
"incident_id": "01KSQNV81DX0PQQTSJX6FAM1E4",
"status": "investigating",
"updated_at": "2026-05-28T16:14:27Z"
}
],
"name": "[US] Delayed LLM as a judge evaluation",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-05-28T17:13:36Z",
"status": "resolved",
"updated_at": "2026-05-28T17:13:36Z"
},
{
"created_at": "2026-05-27T17:17:57Z",
"id": "01KSN72STM8HKPGS5TFGAKY3XK",
"impact": "minor",
"incident_updates": [
{
"body": "We have scaled our infra to drain the queue backlog. Evals are executed without delay again.",
"created_at": "2026-05-27T18:36:20Z",
"display_at": "2026-05-27T18:36:20Z",
"id": "01KSNBJAZNFSZRRFKESWRWYHXN",
"incident_id": "01KSN72STM8HKPGS5TFGAKY3XK",
"status": "resolved",
"updated_at": "2026-05-27T18:36:20Z"
},
{
"body": "We continue to monitor the situation",
"created_at": "2026-05-27T18:01:47Z",
"display_at": "2026-05-27T18:01:47Z",
"id": "01KSN9K2F83PPXHFXS3Y3HS2Z6",
"incident_id": "01KSN72STM8HKPGS5TFGAKY3XK",
"status": "monitoring",
"updated_at": "2026-05-27T18:01:47Z"
},
{
"body": "We believe only a single customer is affected and it is caused by LLM's provider rate limits. Since we monitor worst delay across our system this initially looked as a bigger issue.",
"created_at": "2026-05-27T18:01:25Z",
"display_at": "2026-05-27T18:01:25Z",
"id": "01KSN9JCEJFSBXT7EH07HKQBQH",
"incident_id": "01KSN72STM8HKPGS5TFGAKY3XK",
"status": "identified",
"updated_at": "2026-05-27T18:01:25Z"
},
{
"body": "We are investigating the root cause",
"created_at": "2026-05-27T17:17:57Z",
"display_at": "2026-05-27T13:17:00Z",
"id": "01KSN72STMB4V93SBBCGMYEN4G",
"incident_id": "01KSN72STM8HKPGS5TFGAKY3XK",
"status": "investigating",
"updated_at": "2026-05-27T17:19:11Z"
}
],
"name": "LLM as a Judge execution is delayed",
"page_id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"resolved_at": "2026-05-27T18:36:20Z",
"status": "resolved",
"updated_at": "2026-05-27T18:36:20Z"
}
],
"page": {
"id": "01KS5BHY7AHH8P8EFW4ZENDP28",
"name": "Langfuse",
"updated_at": "2026-09-30T15:30:23Z",
"url": "https://status.langfuse.com/"
}
}