Snapshot 187283
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"g0bngj08wv8y",
"jjlrqc267101",
"40dchhyksdvj",
"t6h34yfknmlh",
"9hy6lprl8r0t",
"zrrw478rdn15",
"2q58rx5kry8s"
],
"created_at": "2019-12-31T16:18:08.086Z",
"description": null,
"group_id": null,
"id": "llp3sllc5hjw",
"name": "Cloud Serverless: GCP",
"page_id": "n3wh2xdrss43",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:39:59.386Z"
},
{
"components": [
"ctk1fsr984vm",
"k52g03svy9zh",
"v67g7nh6g19q",
"yt2p73k6pqk3",
"yvxg3vrg03k4",
"4mrmc30hpgys",
"yp14rldkqsdq"
],
"created_at": "2019-12-31T17:57:43.720Z",
"description": null,
"group_id": null,
"id": "09ylz7krtf65",
"name": "Cloud Serverless: AWS, EU-Central",
"page_id": "n3wh2xdrss43",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:40:09.636Z"
},
{
"components": [
"dgy77st8jft6",
"cgsnpmzn9fjf",
"qxk5y7fprtqj",
"km2z6lhxpfhw",
"17l4mp9mwv0f",
"jb60j6mdkq58",
"1bl9f588302w"
],
"created_at": "2019-12-31T18:00:20.303Z",
"description": null,
"group_id": null,
"id": "h6vxc920l94g",
"name": "Cloud Serverless: AWS, US-West-2-1",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:45:01.758Z"
},
{
"components": [
"8s2qg6x6dng8",
"lbjl9xq165fg",
"hyz83gc6vg3l",
"fyz9hwtq1mqv",
"123cwc23rt2q",
"fs5zrnkp4v7z",
"rq81z7768g5j"
],
"created_at": "2022-01-18T21:08:50.907Z",
"description": null,
"group_id": null,
"id": "40sp5lzmz7fn",
"name": "Cloud Serverless: AWS, US-West-2-2",
"page_id": "n3wh2xdrss43",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:45:57.451Z"
},
{
"components": [
"w01bjntdyy1j",
"sczwv7kzm4p2",
"tysqmv0jshxk",
"lxlld9w8w1v5",
"dffqk1y71qb1",
"1yyjpwp8981v",
"qkdtwp91b97d"
],
"created_at": "2020-10-12T16:51:23.625Z",
"description": null,
"group_id": null,
"id": "8ggny1z5y5jl",
"name": "Cloud Serverless: AWS, US-East-1",
"page_id": "n3wh2xdrss43",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:46:21.024Z"
},
{
"components": [
"8j9skdxr31qy",
"z7nzzsv5g701",
"88dhppfqt709",
"ht7rt34nqb8k",
"2r0h41zy26nm",
"pr6qsy5bgmqc",
"46dk5x679l26"
],
"created_at": "2020-08-06T15:34:56.688Z",
"description": null,
"group_id": null,
"id": "zk71dc4www40",
"name": "Cloud Serverless: Azure, W. Europe",
"page_id": "n3wh2xdrss43",
"position": 6,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:48:31.889Z"
},
{
"components": [
"zpjwlqwtyfxw",
"z7w7xl6r5295",
"042r04zhy82r",
"6zwn5nlnqt0k",
"0qxv2zcm4sg0",
"ckm9v77cszvm",
"d5kvgl1j0xz6"
],
"created_at": "2020-10-07T20:04:22.890Z",
"description": null,
"group_id": null,
"id": "y7vrcb204xph",
"name": "Cloud Serverless: Azure, East US",
"page_id": "n3wh2xdrss43",
"position": 7,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:47:00.809Z"
},
{
"components": [
"nskyglhbzt03",
"mfvbkqy7wkqr",
"cxm8xqyrh1tp"
],
"created_at": "2024-08-15T21:57:24.010Z",
"description": null,
"group_id": null,
"id": "cpj3f450vs2v",
"name": "Cloud Dedicated",
"page_id": "n3wh2xdrss43",
"position": 8,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:48:17.300Z"
},
{
"components": [
"4tfn1s94kpwq",
"3znbywl030qk"
],
"created_at": "2019-12-31T18:16:40.618Z",
"description": "These services provide functionality across all providers and regions.",
"group_id": null,
"id": "hd39m4t0zywd",
"name": "Other Services",
"page_id": "n3wh2xdrss43",
"position": 9,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-08-21T16:48:17.313Z"
}
],
"incidents": [
{
"components": [
{
"created_at": "2020-08-06T15:36:01.931Z",
"description": null,
"group_id": "zk71dc4www40",
"id": "88dhppfqt709",
"name": "API Queries",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T18:17:38.945Z"
}
],
"created_at": "2026-09-29T05:28:12.389Z",
"id": "q52cftvv6fnl",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We appreciate the continued patience while we worked through this incident. \nWe have been monitoring the previous fixes for the last 24 hours and all changes have responded well during sustained traffic. The region is stable and we are resolving this incident. An RCA is in progress and will be added to our status page as soon as its available.",
"created_at": "2026-10-02T18:17:38.987Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-02T18:17:38.987Z",
"id": "t819049n12cf",
"incident_id": "q52cftvv6fnl",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-02T18:17:38.987Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The cluster remains stable, and we are continuing to monitor it.",
"created_at": "2026-10-01T08:13:30.571Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-01T08:13:30.571Z",
"id": "zz5v1mstg5wz",
"incident_id": "q52cftvv6fnl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-01T08:13:30.571Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have increased isolation in the storage layer for the impacted region to provide greater amounts of resilience and to allow for faster recovery. We are continuing to investigate heightened memory consumption and page faults within the storage layer. Query success rates and response durations have returned to normal levels. We will continue work to increase the resiliency of this region over the next 24 hours and will carefully monitor for any increase in query latency.",
"created_at": "2026-09-30T17:16:46.712Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T17:16:46.712Z",
"id": "2hrplnm4ymhj",
"incident_id": "q52cftvv6fnl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T17:16:46.712Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Performance has returned to normal levels following an update. We will continue to monitor.",
"created_at": "2026-09-30T14:24:27.641Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T14:24:27.641Z",
"id": "tgtnczq2r068",
"incident_id": "q52cftvv6fnl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T14:24:27.641Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have made several recent changes to improve query performance and we continue to monitor the region closely.",
"created_at": "2026-09-30T13:44:45.487Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T13:44:45.487Z",
"id": "bg79p9ztxc6z",
"incident_id": "q52cftvv6fnl",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T13:44:45.487Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-09-30T12:08:49.315Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T12:08:49.315Z",
"id": "9zndsrsdhsdw",
"incident_id": "q52cftvv6fnl",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T12:08:49.315Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-09-30T10:49:34.213Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T10:49:34.213Z",
"id": "qw9yhnzzgc6n",
"incident_id": "q52cftvv6fnl",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T10:49:34.213Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are still having issues in the region, the engineering team is working on it.",
"created_at": "2026-09-30T08:50:06.120Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T08:50:06.120Z",
"id": "v1tzlww8tllf",
"incident_id": "q52cftvv6fnl",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T08:50:06.120Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have identified the issue and the team is actively working on implementing a fix.",
"created_at": "2026-09-30T07:53:39.157Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T07:53:39.157Z",
"id": "2x33qljzfh1q",
"incident_id": "q52cftvv6fnl",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T07:53:39.157Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are working on resolving the ongoing intermittent query performance. \n\nDuring peak usage periods, large or complex queries may run slower than normal. \n\nWrites and data durability are not affected at this time.\n\nWe have deployed several improvements overnight and further changes are in progress.",
"created_at": "2026-09-30T07:34:55.022Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-30T07:34:55.022Z",
"id": "z2d8jv0njxvz",
"incident_id": "q52cftvv6fnl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-30T07:34:55.022Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Query performance remains stable, and we continue to monitor closely as traffic increases. We will continue to assess performance and provide updates.",
"created_at": "2026-09-29T23:31:39.791Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T23:31:39.791Z",
"id": "vhqf1v59rll0",
"incident_id": "q52cftvv6fnl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T23:31:39.791Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Query performance remains stable, and we continue to monitor closely. Additional mitigation and stabilization work is ongoing.",
"created_at": "2026-09-29T18:33:55.579Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T18:33:55.579Z",
"id": "g3n0y3ymshhy",
"incident_id": "q52cftvv6fnl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T18:33:55.579Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Query performance has returned to normal levels. Planned stabilisation work is still ongoing, and we are monitoring closely.",
"created_at": "2026-09-29T16:39:16.891Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T16:39:16.891Z",
"id": "2dxl93hx7w4s",
"incident_id": "q52cftvv6fnl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T16:39:16.891Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing work to remediate immediate query performance degradation, and investigating actions for a permanent fix.",
"created_at": "2026-09-29T15:25:18.270Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T15:25:18.270Z",
"id": "462dwbdxtjch",
"incident_id": "q52cftvv6fnl",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T15:25:18.270Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-09-29T11:34:16.264Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T11:34:16.264Z",
"id": "3xtqkx2d5f4q",
"incident_id": "q52cftvv6fnl",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T11:34:16.264Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-09-29T09:09:03.331Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T09:09:03.331Z",
"id": "ynk78rsrtb3w",
"incident_id": "q52cftvv6fnl",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T09:09:03.331Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue and will provide further updates as soon as available.",
"created_at": "2026-09-29T06:38:24.885Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T06:38:24.885Z",
"id": "1x94mng4b331",
"incident_id": "q52cftvv6fnl",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T06:38:24.885Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating an issue affecting query performance in Azure West Europe.",
"created_at": "2026-09-29T05:28:12.501Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-29T05:28:12.501Z",
"id": "0l5kxs76nszl",
"incident_id": "q52cftvv6fnl",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-29T05:28:12.501Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-29T16:39:16.891Z",
"name": "Query Performance Degradation in Azure West Europe",
"page_id": "n3wh2xdrss43",
"postmortem_body": "# RCA \n\n# Query Performance Degradation in Azure West Europe, September 27 to October 1, 2026\n\n## Summary\n\nFrom the evening of September 27 to the morning of September 30 \\(UTC\\), customers in the Azure West Europe region experienced slow queries and timeouts. Writes were accepted throughout, but on some partitions, new data was not immediately queryable. Earlier incidents on this cluster \\(September 7, 11, 15 and 16\\) produced the same symptom with different triggers and have their own RCAs. We believe the condition described herein was the root cause of all prior incidents.\n\n## Cause\n\nAll data in the cluster is stored on redundant pods across many physical nodes. These pods read time-series data through memory-mapped files. The host’s operating system keeps the most frequently accessed parts of those files in a page cache built from memory not consumed by the pods themselves. In this region, each pod holds about 365 GB of data. The cache on a host could not hold the data its pods were querying, so reads that would normally come from cache became page faults served from disk. Each pod read about 90 MB/s from disk, thus a host with ten pods reached roughly 990 MB/s, which is near the throughput ceiling on each host.\n\nMemory consumption looked normal on every metric we monitor. The storage process itself uses little memory, and the cache takes the remainder, so a shortage would not appear as memory pressure or shortfall. At the throughput ceiling, index lookups on the write path were delayed by disk reads, causing writes to fall behind and pods to fail their replication-lag health check. Pods in this region spent 8.9% of the day past the lag threshold, against 0.09% in a comparable region. At peak, both replicas of the busiest partitions fell behind together, creating access delays which were most acutely visible through the query path.\n\nSlow disk caused health and readiness checks to time out, so healthy pods were removed from rotation and restarted. Each restart took 30 to 85 minutes to recover, during which the pod's partner carried the load alone, with the rebuild adding even more disk load. A file-reference safeguard was also causing 50\\+ pod restarts a day where a normal rate would be 0.\n\n## Remediation\n\nWhile early changes helped, and resolved many trouble indicators, they did not solve the root cause. We doubled query-layer capacity, restarted pods, raised the readiness timeouts, raised cleanup timeouts to avoid safeguard restarts, and doubled recovery concurrency after a restart. We ruled out a controller lock, a per-disk limit, and a single tenant's load.\n\nThe root cause fix was to raise each storage pod's resource request so that each host holds fewer pods. Each pod now has twice as much memory for page cache and disk throughput. Within 1 hour of this change, query abandonment fell from 31% to 0.1%. Page faults and disk reads dropped sharply. During the next morning's peak, abandonment was 0.06%, within normal limits.\n\n## Timeline \\(UTC\\)\n\nSep 27 22:58 - Query queue depth alert fires; incident opened. Query-layer capacity increased over the next hours without lasting effect\n\nSep 28 07:00 - Degradation at morning peak; storage pods fall behind on writes and fail health checks; stuck pods restarted one at a time\n\nSep 28 11:33 - Restart loop identified: slow disk causes health checks to trigger restart; each restart takes 30 to 85 minutes to recover\n\nSep 28 13:45 - Lost file-reference cleanup identified as a second source of forced restarts\n\nSep 29 08:58 - Readiness probe timeout raised and rolled out fleet-wide\n\nSep 29 11:10 - Throughput appears to recover; degradation returns, with 36% of queries abandoned\n\nSep 29 16:20 - Queue clears after recovery-concurrency and probe changes\n\nSep 30 00:25 - Cleanup timeout raised to 24 hours; forced restarts stop\n\nSep 30 07:00 - Degradation at peak; abandonment about 55%\n\nSep 30 11:59 - Analysis shows cache capacity per pod is too small and disk reads are at the host ceiling\n\nSep 30 12:43 - Rollout begins to spread pods across more nodes\n\nSep 30 13:21 - Query performance returns to normal; abandonment down to 0.1% within the hour\n\nSep 30 14:20 - Page faults and disk reads confirmed sharply lower after rollout\n\nOct 1 07:00 to 09:00 - Morning peak with 0.06% abandonment versus 40% to 55% the day before\n\nOct 1 09:48 - Incident declared resolved internally\n\n## Future mitigation\n\nWe added warning alerts for page-fault rate, cache coverage, disk reads on storage volumes, and query abandonment. Each alert is designed so that cache pressure appears before causing external visibility. Runbooks now suggest major page faults as a leading indicator of storage performance degradation, with details on “relief valves” available to tune. We are continuing to track down the storage-engine reference leak and moving the disk check out of the health path. Neither leak nor check is a risk factor for triggering this type of incident; they are conditions observed during diagnosis and remediation that warrant investigation.",
"postmortem_body_last_updated_at": "2026-10-06T19:11:28.710Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-06T19:11:28.710Z",
"reminder_intervals": null,
"resolved_at": "2026-10-02T18:17:38.987Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/wl8wp3k886fx",
"started_at": "2026-09-29T05:28:12.381Z",
"status": "resolved",
"updated_at": "2026-10-06T19:11:28.736Z"
},
{
"components": [
{
"created_at": "2020-08-06T15:36:01.931Z",
"description": null,
"group_id": "zk71dc4www40",
"id": "88dhppfqt709",
"name": "API Queries",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T18:17:38.945Z"
}
],
"created_at": "2026-09-28T08:33:13.999Z",
"id": "dg41w6sbdk5b",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-09-28T20:19:14.813Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-28T20:19:14.813Z",
"id": "zjw8s9yln67x",
"incident_id": "dg41w6sbdk5b",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-28T20:19:14.813Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-09-28T18:01:24.078Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-28T18:01:24.078Z",
"id": "26l7ygt0sxym",
"incident_id": "dg41w6sbdk5b",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-28T18:01:24.078Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-09-28T15:08:38.481Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-28T15:08:38.481Z",
"id": "4j1p6lrzn9lx",
"incident_id": "dg41w6sbdk5b",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-28T15:08:38.481Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-09-28T13:10:13.328Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-28T13:10:13.328Z",
"id": "ntw656928vjy",
"incident_id": "dg41w6sbdk5b",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-28T13:10:13.328Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-09-28T11:45:06.078Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-28T11:45:06.078Z",
"id": "5xcq61vdypth",
"incident_id": "dg41w6sbdk5b",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-28T11:45:06.078Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Some queries may be slower than normal and recently written data may take longer to appear in results. Data ingestion and durability are not affected. We have identified the cause and mitigation is underway",
"created_at": "2026-09-28T09:54:38.250Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-28T09:54:38.250Z",
"id": "4rqtx8g768nh",
"incident_id": "dg41w6sbdk5b",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-28T09:54:38.250Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-09-28T08:33:14.107Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-28T08:33:14.107Z",
"id": "z14935916hhy",
"incident_id": "dg41w6sbdk5b",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-28T08:33:14.107Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-28T18:01:24.078Z",
"name": "Degraded Query Performance in Azure West Europe",
"page_id": "n3wh2xdrss43",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-28T20:19:14.813Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/nt8fvq72j34s",
"started_at": "2026-09-28T08:33:13.989Z",
"status": "resolved",
"updated_at": "2026-09-28T20:19:14.838Z"
},
{
"components": [
{
"created_at": "2020-08-06T15:36:01.931Z",
"description": null,
"group_id": "zk71dc4www40",
"id": "88dhppfqt709",
"name": "API Queries",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T18:17:38.945Z"
}
],
"created_at": "2026-09-16T10:08:27.330Z",
"id": "6q8vcr976d2k",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-09-16T16:57:11.451Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T16:57:11.451Z",
"id": "893yfwlh82g8",
"incident_id": "6q8vcr976d2k",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T16:57:11.451Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-09-16T12:56:48.326Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T12:56:48.326Z",
"id": "mvwwsygx68yp",
"incident_id": "6q8vcr976d2k",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T12:56:48.326Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-09-16T11:26:47.067Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T11:26:47.067Z",
"id": "xr6nnz3fkjks",
"incident_id": "6q8vcr976d2k",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T11:26:47.067Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-09-16T10:08:27.413Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T10:08:27.413Z",
"id": "dpqxm5l0ww8r",
"incident_id": "6q8vcr976d2k",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T10:08:27.413Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-16T12:56:48.326Z",
"name": "Query performance degradation in Azure Westeurope.",
"page_id": "n3wh2xdrss43",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-16T16:57:11.451Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/p7rlkkfdswnm",
"started_at": "2026-09-16T10:08:27.320Z",
"status": "resolved",
"updated_at": "2026-09-16T16:57:11.466Z"
},
{
"components": [
{
"created_at": "2024-08-02T18:28:03.617Z",
"description": null,
"group_id": "hd39m4t0zywd",
"id": "3znbywl030qk",
"name": "Auth0 User Authentication",
"page_id": "n3wh2xdrss43",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-11T16:37:46.266Z"
}
],
"created_at": "2026-09-11T16:32:25.112Z",
"id": "48z2y7pyl5fd",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "3znbywl030qk",
"name": "Other Services - Auth0 User Authentication",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-09-11T19:26:26.397Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-11T19:26:26.397Z",
"id": "lk9d14jjlr0f",
"incident_id": "48z2y7pyl5fd",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-11T19:26:26.397Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3znbywl030qk",
"name": "Other Services - Auth0 User Authentication",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The engineering team has resolved the issue and is continuing to monitor the health of the SSO Login Service",
"created_at": "2026-09-11T16:37:46.297Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-09-11T16:37:46.297Z",
"id": "kschwjr3vdhs",
"incident_id": "48z2y7pyl5fd",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-11T16:37:46.297Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3znbywl030qk",
"name": "Other Services - Auth0 User Authentication",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "The SSO Login Service is currently experiencing downtime and the team is currently looking into it",
"created_at": "2026-09-11T16:32:25.198Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-11T16:32:25.198Z",
"id": "ftd7w34c297t",
"incident_id": "48z2y7pyl5fd",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-11T16:32:25.198Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-11T16:37:46.297Z",
"name": "SSO Login Service Availability (other login methods not affected)",
"page_id": "n3wh2xdrss43",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-11T19:26:26.397Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/7dt3n0grpnx2",
"started_at": "2026-09-11T16:32:25.103Z",
"status": "resolved",
"updated_at": "2026-09-11T19:26:26.417Z"
},
{
"components": [
{
"created_at": "2020-08-06T15:36:01.931Z",
"description": null,
"group_id": "zk71dc4www40",
"id": "88dhppfqt709",
"name": "API Queries",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T18:17:38.945Z"
}
],
"created_at": "2026-09-11T08:45:49.624Z",
"id": "020h6gmr70rt",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-09-11T09:46:51.096Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-11T09:46:51.096Z",
"id": "2nmxvxby7343",
"incident_id": "020h6gmr70rt",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-11T09:46:51.096Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-09-11T09:27:31.289Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-11T09:27:31.289Z",
"id": "45v450jv8ypl",
"incident_id": "020h6gmr70rt",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-11T09:27:31.289Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-09-11T08:45:49.729Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-11T08:45:49.729Z",
"id": "d115j48xdm8n",
"incident_id": "020h6gmr70rt",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-11T08:45:49.729Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-11T09:27:31.289Z",
"name": "Query performance degradation in Azure Westeurope",
"page_id": "n3wh2xdrss43",
"postmortem_body": "# **Root Cause Analysis: Query Latency on September 11, 2026**\n\nRegion: Azure West Europe Cluster: prod01-eu-west-1\n\n\n\n\n\n**Incident report: query latency in Azure West Europe, September 11, 2026**\n\nOn the morning of September 11, some customers in Azure West Europe saw slower than usual query responses. Read latency was elevated and there were occasional query timeouts. Performance returned to normal at 08:52 UTC. We're sorry for the disruption.\n\n**What happened**\n\nOn September 10, a single workload ran a sustained volume of deletes. That activity left a large backlog of delete markers on every storage partition in the region.\n\nDelete markers are normally cleared by a scheduled delete-compaction process. Until that process ran, each hourly retention pass had to work through the full backlog. Passes that usually take about three minutes were taking 10 to 15 minutes across the cluster. While a partition was in one of these longer passes it responded more slowly, and queries that spanned it were delayed.\n\nAt 07:55 UTC on September 11, one storage partition entered a self-recovery restart. Its companion replica was in the middle of a retention pass at the time, so for a short period that partition had no healthy replica available to serve queries. This was not the root cause, but the overlap added to the impact on queries touching that partition.\n\nAlerts on query queue depth fired at 08:14 and we opened an incident. An operator restarted the companion replica at 08:25. The impaired partition completed its recovery at 08:47. The query queue then drained without further intervention, and by 08:52 it was back to normal and the alerts had cleared.\n\n**Timeline** \\(UTC\\)\n\nTime Event\n\nSep 10 - Sustained high delete volume from a single workload\n\nSep 11, 07:55 - Storage partition restarted and entered extended recovery\n\n08:14 - Query queue-depth alerts triggered; incident opened\n\n08:25 - Companion replica restarted\n\n08:47 - Impaired partition completed recovery\n\n08:52 - Query queue returned to normal; alerts cleared\n\n**Changes we are making**\n\nThere are four pieces of work under way. None of them is complete yet, and we won't consider them validated until we have confirmed the effect in production.\n\n* Retention enforcement is moving to an off-peak window. Longer passes should then fall outside business-hours query load.\n* Full compactions will run more frequently, so a delete backlog can clear between scheduled runs instead of accumulating.\n* Per-partition compaction throughput is being increased.\n* New alerting and telemetry will tell us whether the changes above are having the intended effect.",
"postmortem_body_last_updated_at": "2026-09-28T13:16:15.162Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-28T13:16:15.162Z",
"reminder_intervals": null,
"resolved_at": "2026-09-11T09:46:51.096Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/p3644c1wj0zk",
"started_at": "2026-09-11T08:45:49.611Z",
"status": "resolved",
"updated_at": "2026-09-28T13:16:15.191Z"
},
{
"components": [
{
"created_at": "2020-08-06T15:36:01.931Z",
"description": null,
"group_id": "zk71dc4www40",
"id": "88dhppfqt709",
"name": "API Queries",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T18:17:38.945Z"
}
],
"created_at": "2026-09-07T10:11:46.600Z",
"id": "g937nc4t9z2k",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-09-07T16:03:18.940Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-07T16:03:18.940Z",
"id": "gq8w8yjwxkv2",
"incident_id": "g937nc4t9z2k",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-07T16:03:18.940Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-09-07T15:37:57.442Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-07T15:37:57.442Z",
"id": "9ygqznr67clb",
"incident_id": "g937nc4t9z2k",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-07T15:37:57.442Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-09-07T14:52:38.551Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-07T14:52:38.551Z",
"id": "f3dhn1q2k12x",
"incident_id": "g937nc4t9z2k",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-07T14:52:38.551Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-09-07T13:11:42.333Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-07T13:11:42.333Z",
"id": "v5x70mymkmsg",
"incident_id": "g937nc4t9z2k",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-07T13:11:42.333Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-09-07T11:13:04.141Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-07T11:13:04.141Z",
"id": "5rlm0hkdtncw",
"incident_id": "g937nc4t9z2k",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-07T11:13:04.141Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-09-07T10:11:46.701Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-07T10:11:46.701Z",
"id": "8fdlf0qxjqr3",
"incident_id": "g937nc4t9z2k",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-07T10:11:46.701Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-07T15:37:57.442Z",
"name": "Query performance degradation in Azure West Europe",
"page_id": "n3wh2xdrss43",
"postmortem_body": "# **Root Cause Analysis: Query Latency on September 7, 2026**\n\nRegion: Azure West Europe Cluster: prod01-eu-west-1\n\n## **Summary and impact**\n\nOn September 7, 2026, some customers in Azure West Europe experienced increased query latency. Query success rates and write operations were not affected.\n\nThree storage replicas fell behind after increased delete activity coincided with scheduled retention processing. Recovery required staged restarts and changes to compaction and retention processing. Query latency returned to normal at 15:50 UTC, and the incident was resolved at 16:16 UTC.\n\n## **Cause**\n\nA sudden burst of delete requests from one organization coincided with a scheduled retention pass on three storage replicas. The combined load exhausted their compaction capacity, causing them to fall behind in processing incoming data.\n\nThe replicas did not recover on their own after the delete burst stopped. Each retention pass began immediately after the preceding pass finished, leaving no window to clear the backlog. A steady stream of delete requests from a second organization also contributed to the slow recovery.\n\n## **Recovery actions**\n\nThe team contacted the organization issuing the delete burst, and the burst stopped at 10:10 UTC. The affected replicas were restarted in stages, and compaction was temporarily paused to prioritize clearing the backlog.\n\nThe retention schedule was also lengthened to give replicas recovery time between passes. Backlogs subsequently decreased across all affected replicas, and query latency returned to normal.\n\n## **Timeline**\n\nAll times are UTC.\n\nTime Event\n\n09:40 - Delete burst began; storage replicas fell behind.\n\n10:01 - Alerts triggered; incident opened.\n\n10:04 - Cause identified; originating organization contacted; status page published.\n\n10:10 - Delete burst stopped.\n\n10:15 - Staged restart of affected replicas began.\n\n11:14 - First replica fully recovered; the other two continued clearing their backlogs.\n\n12:11 - Compaction was paused on the most-affected replica.\n\n14:29 - Retention schedule was lengthened to allow recovery between passes.\n\n15:29 - Backlogs were decreasing on all affected replicas; status page moved to monitoring.\n\n15:50 - Query latency returned to normal.\n\n16:16 - Incident resolved.\n\n## **Follow-up work and Mitigations**\n\nThe retention scheduling change was applied during the incident response. Additional work described in the investigation includes:\n\n* Continued investigation into why the affected replicas did not self-recover after the load was removed.\n* Evaluation of scheduling changes to prevent consecutive retention passes while a replica is behind.\n* Consideration of per-organization delete-rate limits, in addition to existing circuit-breaker controls.\n\nThe scheduling safeguards and rate limits remain under evaluation; they are not described as completed fixes; however, we continue to look for ways to provide a healthy and stable environment.",
"postmortem_body_last_updated_at": "2026-09-28T13:13:20.314Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-28T13:13:20.314Z",
"reminder_intervals": null,
"resolved_at": "2026-09-07T16:03:18.940Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/ctrr38xqs81c",
"started_at": "2026-09-07T10:11:46.589Z",
"status": "resolved",
"updated_at": "2026-09-28T13:13:20.341Z"
},
{
"components": [
{
"created_at": "2020-08-06T15:36:01.931Z",
"description": null,
"group_id": "zk71dc4www40",
"id": "88dhppfqt709",
"name": "API Queries",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-02T18:17:38.945Z"
}
],
"created_at": "2026-08-25T13:49:50.103Z",
"id": "mpp8x4qjqgft",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-08-25T15:41:54.986Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T15:41:54.986Z",
"id": "mqsbn87xght8",
"incident_id": "mpp8x4qjqgft",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T15:41:54.986Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-08-25T14:29:24.100Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T14:29:24.100Z",
"id": "kt6zhmwl7nhp",
"incident_id": "mpp8x4qjqgft",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T14:29:24.100Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "88dhppfqt709",
"name": "Cloud Serverless: Azure, W. Europe - API Queries",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-08-25T13:49:50.195Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-25T13:49:50.195Z",
"id": "mbngp0wcyv5t",
"incident_id": "mpp8x4qjqgft",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-25T13:49:50.195Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-08-25T14:29:24.100Z",
"name": "Query performance degradation in Azure Westeurope",
"page_id": "n3wh2xdrss43",
"postmortem_body": "# **Root Cause Analysis: Query Latency on August 25, 2026**\n\nRegion: Azure West Europe Cluster: prod01-eu-west-1\n\n## **Summary and impact**\n\nOn August 25, 2026, some customers in Azure West Europe experienced increased query latency and query timeouts. Additional query capacity helped reduce the initial queue, but full recovery required a manual restart of a stalled storage partition. Query latency returned to normal at 15:35 UTC.\n\n## **Cause**\n\nA surge in query volume triggered scaling that quickly exhausted available resources. Shortly after the initial trigger, a storage partition did not complete startup successfully.\n\nAlthough additional capacity brought the query queue back to a nominal state, customer latency signals showed that recovery was incomplete. We discovered the stalled partition required manual intervention to complete initialization. One half of the partition remained online to serve queries and writes while its companion half was impaired.\n\nThe underlying reason for the partition startup failure remains under investigation.\n\n## **Recovery actions**\n\nThe team allocated additional query resources, then manually inspected and restarted the impaired partition after identifying the incomplete recovery. The partition fully recovered at 15:32 UTC, and query latency returned to normal at 15:35 UTC.\n\n## **Timeline**\n\nAll times are UTC.\n\nTime Event\n\n13:13 - Scaling alerts triggered; incident opened.\n\n13:33 - Additional resources were added and consumed.\n\n13:35 - Query queue returned to a nominal state.\n\n14:11 - Impaired partition attempted a self-repair restart.\n\n14:40 - Customer latency signals indicated incomplete recovery; investigation resumed.\n\n15:24 - Impaired partition was manually inspected and restarted.\n\n15:32 - Impaired partition fully recovered.\n\n15:35 - Query latency returned to normal.\n\n## **Future Mitigations**\n\nImprovements to the partition management workflow are in progress to strengthen resilience and automate recovery. Investigation into the startup failure continues.\n\nThese improvements are ongoing work, rather than completed preventive fixes. We continue to strive for a healthy platform for all customers on our cloud infrastructure.",
"postmortem_body_last_updated_at": "2026-09-28T13:09:25.806Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-28T13:01:31.960Z",
"reminder_intervals": null,
"resolved_at": "2026-08-25T15:41:54.986Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/rrn627w0gk7s",
"started_at": "2026-08-25T13:49:50.092Z",
"status": "resolved",
"updated_at": "2026-09-28T13:09:25.809Z"
},
{
"components": [
{
"created_at": "2020-10-07T20:04:57.190Z",
"description": null,
"group_id": "y7vrcb204xph",
"id": "042r04zhy82r",
"name": "API Queries",
"page_id": "n3wh2xdrss43",
"position": 3,
"showcase": true,
"start_date": "2020-10-07T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-05-27T16:58:45.249Z"
}
],
"created_at": "2026-05-27T15:51:52.865Z",
"id": "fcdpj47wpgtm",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "042r04zhy82r",
"name": "Cloud Serverless: Azure, East US - API Queries",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-05-27T16:58:45.280Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-27T16:58:45.280Z",
"id": "xqry6wt6jfvk",
"incident_id": "fcdpj47wpgtm",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-27T16:58:45.280Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "042r04zhy82r",
"name": "Cloud Serverless: Azure, East US - API Queries",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-05-27T16:00:33.526Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-27T16:00:33.526Z",
"id": "0vjm568179cm",
"incident_id": "fcdpj47wpgtm",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-27T16:00:33.526Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "042r04zhy82r",
"name": "Cloud Serverless: Azure, East US - API Queries",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-05-27T15:51:52.946Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-27T15:51:52.946Z",
"id": "xhhf5v0h3b0c",
"incident_id": "fcdpj47wpgtm",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-27T15:51:52.946Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-05-27T16:00:33.526Z",
"name": "Increased Error rates in Azure eastus",
"page_id": "n3wh2xdrss43",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-05-27T16:58:45.280Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/yzpp1z4brvmg",
"started_at": "2026-05-27T15:51:52.855Z",
"status": "resolved",
"updated_at": "2026-05-27T16:58:45.295Z"
},
{
"components": [
{
"created_at": "2019-12-31T17:57:43.729Z",
"description": null,
"group_id": "09ylz7krtf65",
"id": "k52g03svy9zh",
"name": "API Writes",
"page_id": "n3wh2xdrss43",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-14T19:00:07.240Z"
}
],
"created_at": "2026-05-12T12:43:01.185Z",
"id": "g2qb3m2bwbpc",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "k52g03svy9zh",
"name": "Cloud Serverless: AWS, EU-Central - API Writes",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-05-14T19:00:07.315Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-14T19:00:07.315Z",
"id": "ydddb442xh57",
"incident_id": "g2qb3m2bwbpc",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-14T19:00:07.315Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "k52g03svy9zh",
"name": "Cloud Serverless: AWS, EU-Central - API Writes",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-05-14T17:00:07.309Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-14T17:00:07.309Z",
"id": "qw1lsns70qkn",
"incident_id": "g2qb3m2bwbpc",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-14T17:00:07.309Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Maintenance will begin as scheduled in 60 minutes.",
"created_at": "2026-05-14T16:00:09.465Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-14T16:00:09.465Z",
"id": "3fv6gxbhpr28",
"incident_id": "g2qb3m2bwbpc",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-14T16:00:09.465Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "k52g03svy9zh",
"name": "Cloud Serverless: AWS, EU-Central - API Writes",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will perform essential maintenance of your InfluxDB Cloud Serverless service.\n\nImpact during the maintenance:\n - Most writes continue normally (buffered in the ingester)\n - Writes that introduce new measurements, tags, or fields may fail\n - Reads will continue to be processed\n\nIn order to minimize the business impact of this maintenance activity, we've scheduled the change to take place during the time window outlined below:\n \t 2026- May 14th – 17:00 – 19:00 UTC\n \t\nYou can use this tool to determine your equivalent local time:\n - https://everytimezone.com/s/e3fd5562 \n\n If you have questions, please contact InfluxData Ops team by opening a case via the Support portal\n\n Thank you.\n InfluxData Ops team",
"created_at": "2026-05-12T12:43:01.242Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-12T12:43:01.242Z",
"id": "wq08w3fn9dhh",
"incident_id": "g2qb3m2bwbpc",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-12T12:43:01.242Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "InfluxDB Cloud Serverless Scheduled Maintenance Notification, May 14, 2026",
"page_id": "n3wh2xdrss43",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-05-14T19:00:07.315Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-05-14T17:00:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": "2026-05-14T16:00:09.584Z",
"scheduled_until": "2026-05-14T19:00:00.000Z",
"shortlink": "https://stspg.io/5dbnpcn6205g",
"started_at": "2026-05-12T12:43:01.176Z",
"status": "completed",
"updated_at": "2026-05-14T19:00:07.345Z"
},
{
"components": [
{
"created_at": "2020-10-12T16:51:41.531Z",
"description": null,
"group_id": "8ggny1z5y5jl",
"id": "sczwv7kzm4p2",
"name": "API Writes",
"page_id": "n3wh2xdrss43",
"position": 2,
"showcase": true,
"start_date": "2020-10-12T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-05-13T12:00:07.204Z"
}
],
"created_at": "2026-05-07T16:31:41.861Z",
"id": "2m7mhv7jyntr",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "sczwv7kzm4p2",
"name": "Cloud Serverless: AWS, US-East-1 - API Writes",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-05-13T12:00:07.335Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-13T12:00:07.335Z",
"id": "dfswsqjf178j",
"incident_id": "2m7mhv7jyntr",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-13T12:00:07.335Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sczwv7kzm4p2",
"name": "Cloud Serverless: AWS, US-East-1 - API Writes",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-05-13T10:00:07.246Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-13T10:00:07.246Z",
"id": "qj56hfj0dj83",
"incident_id": "2m7mhv7jyntr",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-13T10:00:07.246Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Maintenance will begin as scheduled in 60 minutes.",
"created_at": "2026-05-13T09:00:08.511Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-13T09:00:08.511Z",
"id": "b8vyl63wg4w8",
"incident_id": "2m7mhv7jyntr",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-13T09:00:08.511Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sczwv7kzm4p2",
"name": "Cloud Serverless: AWS, US-East-1 - API Writes",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will perform essential maintenance of your InfluxDB Cloud Serverless service.\n\nImpact during the maintenance:\n - Most writes continue normally (buffered in the ingester)\n - Writes that introduce new measurements, tags, or fields may fail\n - Reads will continue to be processed\n\nIn order to minimize the business impact of this maintenance activity, we've scheduled the change to take place during the time window outlined below:\n \t2026- May 13th – 10:00 – 12:00 UTC\n\nYou can use this tool to determine your equivalent local time:\n - https://everytimezone.com/s/3fad8441\n \nIf you have questions, please contact InfluxData Ops team by opening a case via the Support portal\n Thank you.\n InfluxData Ops team",
"created_at": "2026-05-07T16:31:41.923Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-07T16:31:41.923Z",
"id": "v208287d7ytd",
"incident_id": "2m7mhv7jyntr",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-07T16:31:41.923Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "InfluxDB Cloud Serverless Scheduled Maintenance Notification, May 13, 2026",
"page_id": "n3wh2xdrss43",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-05-13T12:00:07.335Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-05-13T10:00:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": "2026-05-13T09:00:08.756Z",
"scheduled_until": "2026-05-13T12:00:00.000Z",
"shortlink": "https://stspg.io/j3qjwfgjtnyg",
"started_at": "2026-05-07T16:31:41.852Z",
"status": "completed",
"updated_at": "2026-05-13T12:00:07.371Z"
}
],
"page": {
"id": "n3wh2xdrss43",
"name": "InfluxDB Cloud",
"url": "https://status.influxdata.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}