Snapshot 259757
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
},
{
"created_at": "2022-02-21T13:42:59.965+01:00",
"description": null,
"group_id": null,
"id": "1lvk2cwdr0tj",
"name": "Ecosystem",
"page_id": "23nynrl74h10",
"position": 3,
"showcase": true,
"start_date": "2022-02-21T00:00:00.000+01:00",
"status": "operational",
"updated_at": "2025-09-24T14:12:23.718+02:00"
},
{
"components": [
"2ptc81hhwfch",
"3f5xytkdd5t8",
"jl20cssbdh1n",
"vwvy39kdljk7"
],
"created_at": "2021-11-20T17:54:59.638+01:00",
"description": null,
"group_id": null,
"id": "n2st5j0v6fnc",
"name": "Third-Party APIs",
"page_id": "23nynrl74h10",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2022-02-21T13:45:19.561+01:00"
}
],
"incidents": [
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
}
],
"created_at": "2026-11-10T14:01:12.203+01:00",
"id": "fshw60wh60ty",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-11-12T03:00:08.897+01:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-12T03:00:08.897+01:00",
"id": "g317gbrcf252",
"incident_id": "fshw60wh60ty",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-12T03:00:08.897+01:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-11-11T23:00:09.620+01:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-11T23:00:09.620+01:00",
"id": "zzl9gjwl73yy",
"incident_id": "fshw60wh60ty",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T23:00:09.620+01:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Maintenance will begin as scheduled in 60 minutes.",
"created_at": "2026-11-11T22:00:08.159+01:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-11T22:00:08.159+01:00",
"id": "c2g8myvc54q8",
"incident_id": "fshw60wh60ty",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T22:00:08.159+01:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Our hoster is performing a scheduled update to our servers' storage infrastructure. This may temporarily affect the availability or performance of Tideways.\n\nService interruptions are possible during the maintenance window. No action is required from customers.",
"created_at": "2026-11-10T14:01:12.515+01:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-10T14:01:12.515+01:00",
"id": "cg3m52qqnc2c",
"incident_id": "fshw60wh60ty",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-10T14:01:12.515+01:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Storage Maintenance Window",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-12T03:00:08.897+01:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-11-11T23:00:00.000+01:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": "2026-11-11T22:00:08.501+01:00",
"scheduled_until": "2026-11-12T03:00:00.000+01:00",
"shortlink": "https://stspg.io/47rnnnz0ys3n",
"started_at": "2026-11-10T14:01:12.175+01:00",
"status": "completed",
"updated_at": "2026-11-12T03:00:09.564+01:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
}
],
"created_at": "2026-10-21T09:25:20.850+02:00",
"id": "5y3h5m2g27g5",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Updates went smooth, everything looks good on our end. We will keep monitoring our systems closely for a few more hours.\n\nThank you for your patience and best of luck with your updates.",
"created_at": "2026-10-21T16:10:05.766+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T16:10:05.766+02:00",
"id": "h3shynsg0t8s",
"incident_id": "5y3h5m2g27g5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T16:10:05.766+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Packages have been released and are currently rolling out on our systems. We will keep monitoring the situation and keep you posted.",
"created_at": "2026-10-21T15:46:58.010+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T15:46:58.000+02:00",
"id": "vv695k09ctzg",
"incident_id": "5y3h5m2g27g5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T16:08:45.283+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-21T15:43:37.512+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T15:43:37.512+02:00",
"id": "dgkgtsdqb7pr",
"incident_id": "5y3h5m2g27g5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T15:43:37.512+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "under_maintenance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "degraded_performance",
"old_status": "under_maintenance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-21T09:25:44.089+02:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-21T09:25:44.000+02:00",
"id": "fp0fp8h7b2zx",
"incident_id": "5y3h5m2g27g5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T09:26:30.434+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Tideways operations team is aware of the upcoming critical PHP security release and preparing for responses as more information and patches are becoming available.\n\nWe are updating you here on this page about our progress.\n\nTideways is not only monitoring your PHP applications, the server side of Tideways is also written in PHP itself. So as the day continues we are not ruling out to shut down Tideways for a short period of time should this become necessary.\n\nRelated:\n* https://www.php.net/archive/2026.php#2026-10-20-1\n* https://phpnews.net/php-critical-vulnerabilities",
"created_at": "2026-10-21T09:25:21.114+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T09:25:21.000+02:00",
"id": "cd264z5gnvqq",
"incident_id": "5y3h5m2g27g5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T15:43:20.691+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-21T15:46:58.000+02:00",
"name": "Pending PHP Security Release",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-21T16:10:05.766+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/rrpxf406gmnb",
"started_at": "2026-10-21T09:25:21.000+02:00",
"status": "resolved",
"updated_at": "2026-10-21T16:10:05.787+02:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
}
],
"created_at": "2026-10-21T03:44:09.099+02:00",
"id": "ndl7512p7h5d",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "Recovered on its own for now.",
"created_at": "2026-10-21T03:51:40.010+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T03:51:40.010+02:00",
"id": "b44lq6vmfqjj",
"incident_id": "ndl7512p7h5d",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T03:51:40.010+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "Like this afternoon, we are seeing network issues in our data center, leading to some requests failing and some working. We have contacted the hosting provider to fix the problem.",
"created_at": "2026-10-21T03:44:09.211+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-21T03:44:09.211+02:00",
"id": "99wr64yd59z7",
"incident_id": "ndl7512p7h5d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T03:44:09.211+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Network failures",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-21T03:51:40.010+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/fh343m2ct9dp",
"started_at": "2026-10-21T03:44:09.091+02:00",
"status": "resolved",
"updated_at": "2026-10-21T03:51:40.033+02:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
}
],
"created_at": "2026-10-20T15:21:23.032+02:00",
"id": "8y6bnw13k4r4",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "All queues have caught up. Data is up-to-date and everything is operational.",
"created_at": "2026-10-20T18:19:20.554+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T18:19:20.554+02:00",
"id": "0rl9ylbvrnvc",
"incident_id": "8y6bnw13k4r4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T18:19:20.554+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We estimate the remaining backlog to be cleared by 4-5pm UTC. Other than that, everything is back to normal operations. Thank you for your patience.",
"created_at": "2026-10-20T17:03:33.898+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T17:03:33.898+02:00",
"id": "zzwkzm07l2zq",
"incident_id": "8y6bnw13k4r4",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T17:03:33.898+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The measurements data backlog has been cleared (section \"Performance\"), while trace processing is still lagging. If current performance keeps up, we expect the outstanding traces to be processed in the next hours. If anything in this prediction changes, we will update you with the new information.",
"created_at": "2026-10-20T16:46:41.936+02:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-20T16:46:41.936+02:00",
"id": "md5yxn20t6kq",
"incident_id": "8y6bnw13k4r4",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T16:46:41.936+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "partial_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "degraded_performance",
"old_status": "partial_outage"
}
],
"body": "System performance is still not back to 100 percent but seems to have stabilized. Backlogs still need quite some time to catch up, we will keep you posted.",
"created_at": "2026-10-20T16:18:57.793+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T16:18:57.793+02:00",
"id": "z3mq15zc7qx1",
"incident_id": "8y6bnw13k4r4",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T16:18:57.793+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Stability is still not fully there, but incoming data is processed and the backlog is slowly shrinking. We keep monitoring the situation and keep you updated.",
"created_at": "2026-10-20T15:56:08.727+02:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-20T15:56:08.727+02:00",
"id": "wb93605qgqq0",
"incident_id": "8y6bnw13k4r4",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T15:56:08.727+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently experiencing a partial outage of our hosting infrastructure. We are in contact with our hosting partner to resolve the issue.",
"created_at": "2026-10-20T15:21:23.294+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-20T15:21:23.294+02:00",
"id": "dny9qk7l36mj",
"incident_id": "8y6bnw13k4r4",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-20T15:21:23.294+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-20T15:56:08.727+02:00",
"name": "Partial Outage Data Center",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-20T18:19:20.554+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/vffbcn1771c1",
"started_at": "2026-10-20T15:21:23.024+02:00",
"status": "resolved",
"updated_at": "2026-10-20T18:19:20.720+02:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
}
],
"created_at": "2026-10-16T15:53:09.350+02:00",
"id": "5m116yxdh2vp",
"impact": "critical",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "Database is responsive again.",
"created_at": "2026-10-16T16:33:36.991+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T16:33:36.991+02:00",
"id": "4ffbfh3x4tfs",
"incident_id": "5m116yxdh2vp",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T16:33:36.991+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "A maintenance query rolled out during a deployment caused more load than expected, overloading the server. The query was aborted and is now being rolled back, still taking time to process. We hopefully expect the process to be completed in a few minutes.",
"created_at": "2026-10-16T16:10:30.191+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T16:10:30.191+02:00",
"id": "mspsp0g9n1ld",
"incident_id": "5m116yxdh2vp",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T16:10:30.191+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "major_outage",
"old_status": "operational"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-16T15:53:18.563+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T15:53:18.563+02:00",
"id": "mz3m6plpd051",
"incident_id": "5m116yxdh2vp",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T15:53:18.563+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-16T15:53:09.414+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T15:53:09.414+02:00",
"id": "165qqdvszhrf",
"incident_id": "5m116yxdh2vp",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T15:53:09.414+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "MySQL primary database server under extreme load and not responding",
"page_id": "23nynrl74h10",
"postmortem_body": "## **Summary**\n\nA maintenance query to delete old entries in a large table was not adequately optimized and required a lot of load. This brought the primary database server under heavy load. Cancelling the query lead to a rollback that also took a while for the system to recover finally.\n\nCustomers experienced a dataloss of incoming data between 13:42 and 14:28 UTC time on October 16th, 2026. \n\nWe are very sorry for the inconvenience and you can be assured that we are following up on this incident to make sure a similar mistake does not happen again and is otherwise softened by better automatic recovery and retry mechanisms.\n\n## **Timeline**\n\n* **13:30 UTC** - Maintenance query to large table was started.\n* **13:42 UTC** - Database server load increased to a level that made UI/Ingest unavailable.\n* **13:49 UTC** - query was killed and started to roll back.\n* **14:28 UTC** - system recovered and load normalized.\n\n## **Root Cause**\n\nAn unoptimized maintenance SQL query to cleanup a large table caused high load.\n\n## **Impact**\n\n* ~45 minute total data gap across customers \\(13:42–14:28 UTC\\)\n* Possible transient UI errors during the incident window\n\n## **Follow-ups**\n\n* Current defined best-practices from Ops team were not followed for the maintenance query, so work is being done on reviewing them in the team and potentially surfacing them during code-reviews automatically.\n* We are investigating how the retry mechanism of daemons from customer machines worked and if we can optimize and improve it for the future.",
"postmortem_body_last_updated_at": "2026-10-16T17:16:59.608+02:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-16T17:16:59.608+02:00",
"reminder_intervals": null,
"resolved_at": "2026-10-16T16:33:36.991+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/ljpj7bw9sxqm",
"started_at": "2026-10-16T15:53:09.343+02:00",
"status": "resolved",
"updated_at": "2026-10-16T17:16:59.633+02:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
}
],
"created_at": "2026-10-09T16:52:43.868+02:00",
"id": "hy1218f3bv5c",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Trace processing speeds are back to normal. The backlog is currently processed and will be fully caught within the next 30-60 minutes.",
"created_at": "2026-10-09T18:15:42.015+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T18:15:42.015+02:00",
"id": "g25y1r8vq3m2",
"incident_id": "hy1218f3bv5c",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T18:15:42.015+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We confirmed the I/O latency issues and are working on a quick solution. Processing speed has already improved and the backlog is no longer growing. We are monitoring the situation and keep you updated of any changes to the situation.",
"created_at": "2026-10-09T18:00:57.819+02:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-09T18:00:57.819+02:00",
"id": "zvchbf8lw649",
"incident_id": "hy1218f3bv5c",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T18:00:57.819+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Trace processing is currently delayed. We suspect degraded I/O performance on our servers and are working with our hosting provider to investigate.",
"created_at": "2026-10-09T16:52:44.128+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-09T16:52:44.128+02:00",
"id": "7cg88r5xcm5y",
"incident_id": "hy1218f3bv5c",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-09T16:52:44.128+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-09T18:00:57.819+02:00",
"name": "Trace ingest pipeline slowed down",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-09T18:15:42.015+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/k9pgzng1vn50",
"started_at": "2026-10-09T16:52:43.860+02:00",
"status": "resolved",
"updated_at": "2026-10-09T18:15:42.038+02:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
}
],
"created_at": "2026-09-21T16:20:14.521+02:00",
"id": "4tjz6yxb8q67",
"impact": "critical",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "After recovering our MySQL database server, everything is back online. Sorry for the interruption.",
"created_at": "2026-09-21T16:32:39.325+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-21T16:32:39.325+02:00",
"id": "zlptkl8h362y",
"incident_id": "4tjz6yxb8q67",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-21T16:32:39.325+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "major_outage",
"old_status": "operational"
}
],
"body": "The MySQL database server is currently unavailable, leading to an outage of all services of Tideways.",
"created_at": "2026-09-21T16:20:14.607+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-21T16:20:14.607+02:00",
"id": "91rcg4xfxdbv",
"incident_id": "4tjz6yxb8q67",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-21T16:20:14.607+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Downtime due to database outage",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-21T16:32:39.325+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/t7ng26yw2r44",
"started_at": "2026-09-21T16:20:14.514+02:00",
"status": "resolved",
"updated_at": "2026-09-21T16:32:39.342+02:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
}
],
"created_at": "2026-07-08T10:30:20.184+02:00",
"id": "qgs08y0g21vb",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Trace collection rate is back to 100%",
"created_at": "2026-07-08T15:34:42.887+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-08T15:34:42.887+02:00",
"id": "kyj12xjlsfjc",
"incident_id": "qgs08y0g21vb",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-08T15:34:42.887+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Following performance issues in our data collection pipeline (https://status.tideways.io/incidents/fg5p17f0rsrl) we have temporarily reduced the number of traces collected to roughly 80%. We expect to lift this limit on July, 8th around 11:00-14:00 CEST, when additional servers have been provisioned.",
"created_at": "2026-07-08T10:30:20.257+02:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-08T10:30:20.257+02:00",
"id": "q1cshtrlphz1",
"incident_id": "qgs08y0g21vb",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-08T10:30:20.257+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-08T10:30:20.257+02:00",
"name": "Reduced number of sampled traces",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-07-08T15:34:42.887+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/zky30ppkbn5g",
"started_at": "2026-07-08T10:30:20.175+02:00",
"status": "resolved",
"updated_at": "2026-07-08T15:34:42.902+02:00"
},
{
"components": [],
"created_at": "2026-07-07T12:00:00.000+02:00",
"id": "nl333gk7q3wx",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": null,
"body": "Following performance issues in our data collection pipeline (https://status.tideways.io/incidents/fg5p17f0rsrl) we have temporarily reduced the number of traces collected to roughly 80%. We expect to lift this limit on July, 8th around 11:00-14:00 CEST, when additional servers have been provisioned.",
"created_at": "2026-07-08T10:28:21.502+02:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-07T12:00:00.000+02:00",
"id": "55j82zp2whb2",
"incident_id": "nl333gk7q3wx",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-08T10:28:21.502+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Reduced number of sampled traces",
"page_id": "23nynrl74h10",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-07-07T12:00:00.000+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/pn1c2zr9x8nr",
"started_at": "2026-07-08T10:28:21.448+02:00",
"status": "resolved",
"updated_at": "2026-07-08T10:29:32.557+02:00"
},
{
"components": [
{
"created_at": "2021-11-20T16:09:55.147+01:00",
"description": "APIs communicating with tideways-daemon, tideways-proxy",
"group_id": null,
"id": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"page_id": "23nynrl74h10",
"position": 1,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:07.735+01:00"
},
{
"created_at": "2021-11-20T16:10:35.843+01:00",
"description": "UI @ https://app.tideways.io",
"group_id": null,
"id": "z7h3bvm9nw01",
"name": "User Interface",
"page_id": "23nynrl74h10",
"position": 2,
"showcase": true,
"start_date": "2021-08-24T00:00:00.000+02:00",
"status": "operational",
"updated_at": "2026-11-12T03:00:08.300+01:00"
}
],
"created_at": "2026-07-06T15:45:42.367+02:00",
"id": "fg5p17f0rsrl",
"impact": "critical",
"impact_override": "critical",
"incident_updates": [
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "All queues have fully caught up, no delays remain.",
"created_at": "2026-07-06T20:11:34.229+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-06T20:11:34.229+02:00",
"id": "y75hb9ds6l9q",
"incident_id": "fg5p17f0rsrl",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-06T20:11:34.229+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The trace queue still needs some time to fully catch up. We'll keep monitoring the situation and will update you as it progresses.",
"created_at": "2026-07-06T18:10:59.342+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-06T18:10:59.342+02:00",
"id": "tz8y1z4rmbsv",
"incident_id": "fg5p17f0rsrl",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-06T18:10:59.342+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "degraded_performance",
"old_status": "partial_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Measurement data has caught up and is now up to date. Trace data is still delayed by roughly 30-45 minutes. We're continuing to work on increasing processing throughput, but it may take some time before traces fully catch up as well.",
"created_at": "2026-07-06T17:51:22.354+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-06T17:51:22.354+02:00",
"id": "mt95yq8k0p58",
"incident_id": "fg5p17f0rsrl",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-06T17:51:22.354+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "We've identified the root cause: elevated write latency on one of our OpenSearch nodes is causing the processing backlog. We're working with our hosting provider to resolve it and will post updates as we have them.",
"created_at": "2026-07-06T16:15:45.248+02:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-06T16:15:45.248+02:00",
"id": "mw336x809px9",
"incident_id": "fg5p17f0rsrl",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-06T16:15:45.248+02:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "jmwgvv6q6j1x",
"name": "Ingest / Data Collection",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "z7h3bvm9nw01",
"name": "User Interface",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating an issue regarding our data collection queues, leading to memory shortages. This might also involve error message in our UI.",
"created_at": "2026-07-06T15:45:42.545+02:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-06T15:45:42.545+02:00",
"id": "qnj6q50bpr0p",
"incident_id": "fg5p17f0rsrl",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-06T15:45:42.545+02:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-06T18:10:59.342+02:00",
"name": "Data processing outage",
"page_id": "23nynrl74h10",
"postmortem_body": "## Summary\n\nElevated write latency on an OpenSearch node caused a backlog in our data collection queues, leading to memory shortages and a short gap in incoming data \\(13:30–13:45 UTC\\) as well as delayed processing of measurement and trace data afterward.\n\n## Timeline\n\n* **~10:00 UTC** — OpenSearch node write latency slowly starts increasing\n* **12:15 UTC —** Issue internally detected, investigation started\n* **13:30–13:45 UTC** — Data collection gap \\(no data ingested\\)\n* **13:45 UTC** — Informing our customers about known issues \\(queue backlog, memory shortages, possible UI errors\\)\n* **14:15 UTC** — Root cause identified: elevated write latency on an OpenSearch node; engaged hosting provider\n* **15:51 UTC** — Measurement data normalized; trace data still delayed ~30–45 min\n* **16:10 UTC** — Trace queue still catching up\n* **18:11 UTC** — All queues fully caught up, incident resolved\n\n## Root Cause\n\nAn OpenSearch node experienced elevated write latency, causing data collection queues to back up and exhaust available memory, which blocked further ingestion for a short window.\n\n## Impact\n\n* ~15 minute total data gap across customers \\(13:30–13:45 UTC\\)\n* Delayed \\(not lost\\) measurement and trace data for several hours afterward\n* Possible transient UI errors during the incident window\n\n## Resolution\n\nWorked with the hosting provider to resolve the underlying OpenSearch node latency issue; queues drained and fully caught up by 18:11 UTC.\n\nAdditionally, increased the number of nodes in the OpenSearch cluster, massively reducing the risk of another write-performance degradation causing similar issues in the future.\n\n## Follow-ups\n\n* Optimizing write operations in the ingest pipeline to reduce the impact of similar performance degradations\n* Improving alerting configuration to catch these issues earlier",
"postmortem_body_last_updated_at": "2026-07-13T15:55:36.822+02:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-07-13T15:55:36.822+02:00",
"reminder_intervals": null,
"resolved_at": "2026-07-06T20:11:34.229+02:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/z8mm38z0qm8v",
"started_at": "2026-07-06T15:45:42.356+02:00",
"status": "resolved",
"updated_at": "2026-10-16T16:35:48.903+02:00"
}
],
"page": {
"id": "23nynrl74h10",
"name": "Tideways",
"url": "https://status.tideways.io"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}