Third Party Index

Snapshot 11033

Document
Registry listing
URL
https://status.honeycomb.io/index.json
Fetched
HTTP status
200
Content type
application/json; charset=utf-8
Fetch mode
api
Size
44200 bytes
SHA-256 (raw)
778e853e1465282bd6d46dc194b398aaebd263e38359aaa4380d04aea9d796ba
SHA-256 (normalized text)
7cf4c2330926576807c4f80c437ec75f25d103c6cd12265960948b6cf425396f

Normalized text

Scripts and page chrome removed; this is what change detection compares.

{
  "components": [
    {
      "created_at": "2017-10-13T17:30:35.373-07:00",
      "description": "POST endpoint to submit events",
      "group_id": null,
      "id": "m8wyq69d24y8",
      "name": "api.honeycomb.io - US1 Event Ingest",
      "page_id": "9xjvdr2lr7rv",
      "position": 1,
      "showcase": true,
      "start_date": null,
      "status": "operational",
      "updated_at": "2026-06-24T12:06:10.569-07:00"
    },
    {
      "created_at": "2017-10-16T19:58:26.301-07:00",
      "description": "Describes the ability to run new queries on existing data, independent of data ingest.",
      "group_id": null,
      "id": "27rh766x05ds",
      "name": "ui.honeycomb.io - US1 Querying",
      "page_id": "9xjvdr2lr7rv",
      "position": 2,
      "showcase": true,
      "start_date": null,
      "status": "operational",
      "updated_at": "2026-08-10T13:35:25.626-07:00"
    },
    {
      "created_at": "2017-10-13T17:30:35.393-07:00",
      "description": "Web app interface for Honeycomb",
      "group_id": null,
      "id": "pw59z78swkns",
      "name": "ui.honeycomb.io - US1 App Interface",
      "page_id": "9xjvdr2lr7rv",
      "position": 3,
      "showcase": true,
      "start_date": null,
      "status": "operational",
      "updated_at": "2026-08-21T08:25:14.570-07:00"
    },
    {
      "created_at": "2021-11-23T12:51:23.891-08:00",
      "description": null,
      "group_id": null,
      "id": "p5q31vg0lzrn",
      "name": "ui.honeycomb.io - US1 Trigger \u0026 SLO Alerting",
      "page_id": "9xjvdr2lr7rv",
      "position": 4,
      "showcase": true,
      "start_date": "2021-11-23T00:00:00.000-08:00",
      "status": "operational",
      "updated_at": "2026-06-24T12:06:53.203-07:00"
    },
    {
      "created_at": "2024-04-10T04:46:50.485-07:00",
      "description": null,
      "group_id": null,
      "id": "ryhsgbnrzg2l",
      "name": "ui.honeycomb.io - US1 Activity Log",
      "page_id": "9xjvdr2lr7rv",
      "position": 5,
      "showcase": true,
      "start_date": "2024-03-27T00:00:00.000-07:00",
      "status": "operational",
      "updated_at": "2026-06-29T17:47:15.291-07:00"
    },
    {
      "created_at": "2026-06-18T11:17:57.239-07:00",
      "description": null,
      "group_id": null,
      "id": "plh6yjykc52q",
      "name": "ui.honeycomb.io - US1 Enhance",
      "page_id": "9xjvdr2lr7rv",
      "position": 6,
      "showcase": true,
      "start_date": "2026-01-01T00:00:00.000-08:00",
      "status": "operational",
      "updated_at": "2026-06-18T12:15:29.365-07:00"
    },
    {
      "created_at": "2024-01-24T08:00:37.417-08:00",
      "description": "POST endpoint to submit events",
      "group_id": null,
      "id": "r7lr006x58vg",
      "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
      "page_id": "9xjvdr2lr7rv",
      "position": 7,
      "showcase": true,
      "start_date": "2023-10-01T00:00:00.000-07:00",
      "status": "operational",
      "updated_at": "2026-06-25T11:31:29.039-07:00"
    },
    {
      "created_at": "2024-01-24T08:01:57.417-08:00",
      "description": "Web app interface for Honeycomb",
      "group_id": null,
      "id": "458ypjm16xt0",
      "name": "ui.eu1.honeycomb.io - EU1 App Interface",
      "page_id": "9xjvdr2lr7rv",
      "position": 8,
      "showcase": true,
      "start_date": "2023-10-01T00:00:00.000-07:00",
      "status": "operational",
      "updated_at": "2026-08-21T08:25:14.592-07:00"
    },
    {
      "created_at": "2024-01-24T08:02:34.673-08:00",
      "description": "Describes the ability to run new queries on existing data, independent of data ingest.",
      "group_id": null,
      "id": "pnss6nhdwd65",
      "name": "ui.eu1.honeycomb.io - EU1 Querying",
      "page_id": "9xjvdr2lr7rv",
      "position": 9,
      "showcase": true,
      "start_date": "2023-10-01T00:00:00.000-07:00",
      "status": "operational",
      "updated_at": "2026-07-28T11:30:16.271-07:00"
    },
    {
      "created_at": "2024-01-24T08:03:13.237-08:00",
      "description": null,
      "group_id": null,
      "id": "sdnpgrc81h4v",
      "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
      "page_id": "9xjvdr2lr7rv",
      "position": 10,
      "showcase": true,
      "start_date": "2023-10-01T00:00:00.000-07:00",
      "status": "operational",
      "updated_at": "2026-07-28T11:30:16.289-07:00"
    },
    {
      "created_at": "2024-04-10T04:48:05.760-07:00",
      "description": null,
      "group_id": null,
      "id": "zf99j8cvb7g9",
      "name": "ui.eu1.honeycomb.io - EU1 Activity Log",
      "page_id": "9xjvdr2lr7rv",
      "position": 11,
      "showcase": true,
      "start_date": "2024-03-27T00:00:00.000-07:00",
      "status": "operational",
      "updated_at": "2026-06-18T11:18:10.583-07:00"
    },
    {
      "created_at": "2026-06-18T11:18:32.483-07:00",
      "description": null,
      "group_id": null,
      "id": "6zprzdf74q84",
      "name": "ui.eu1.honeycomb.io - EU1 Enhance",
      "page_id": "9xjvdr2lr7rv",
      "position": 12,
      "showcase": true,
      "start_date": "2026-01-01T00:00:00.000-08:00",
      "status": "operational",
      "updated_at": "2026-06-18T12:16:04.270-07:00"
    },
    {
      "created_at": "2020-06-29T08:40:36.995-07:00",
      "description": "Marketing, sales, and documentation.",
      "group_id": null,
      "id": "1hk92bykzr99",
      "name": "www.honeycomb.io",
      "page_id": "9xjvdr2lr7rv",
      "position": 13,
      "showcase": false,
      "start_date": null,
      "status": "operational",
      "updated_at": "2026-07-02T11:13:29.892-07:00"
    },
    {
      "created_at": "2024-07-18T14:13:28.920-07:00",
      "description": "The Sandbox is an interactive demo environment that works without signup or setup.",
      "group_id": null,
      "id": "c8z7tj0cvd8j",
      "name": "Sandbox",
      "page_id": "9xjvdr2lr7rv",
      "position": 14,
      "showcase": false,
      "start_date": "2024-07-18T00:00:00.000-07:00",
      "status": "operational",
      "updated_at": "2026-06-18T11:18:46.585-07:00"
    }
  ],
  "incidents": [
    {
      "components": [
        {
          "created_at": "2017-10-13T17:30:35.393-07:00",
          "description": "Web app interface for Honeycomb",
          "group_id": null,
          "id": "pw59z78swkns",
          "name": "ui.honeycomb.io - US1 App Interface",
          "page_id": "9xjvdr2lr7rv",
          "position": 3,
          "showcase": true,
          "start_date": null,
          "status": "operational",
          "updated_at": "2026-08-21T08:25:14.570-07:00"
        },
        {
          "created_at": "2024-01-24T08:01:57.417-08:00",
          "description": "Web app interface for Honeycomb",
          "group_id": null,
          "id": "458ypjm16xt0",
          "name": "ui.eu1.honeycomb.io - EU1 App Interface",
          "page_id": "9xjvdr2lr7rv",
          "position": 8,
          "showcase": true,
          "start_date": "2023-10-01T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-08-21T08:25:14.592-07:00"
        }
      ],
      "created_at": "2026-08-21T08:21:56.370-07:00",
      "id": "nhny0dmssfmp",
      "impact": "minor",
      "impact_override": null,
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "pw59z78swkns",
              "name": "ui.honeycomb.io - US1 App Interface",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "458ypjm16xt0",
              "name": "ui.eu1.honeycomb.io - EU1 App Interface",
              "new_status": "operational",
              "old_status": "operational"
            }
          ],
          "body": "This incident has been resolved.",
          "created_at": "2026-08-21T08:59:39.680-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-08-21T08:59:39.680-07:00",
          "id": "q9tt6vmcj0l6",
          "incident_id": "nhny0dmssfmp",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-08-21T08:59:39.680-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "pw59z78swkns",
              "name": "ui.honeycomb.io - US1 App Interface",
              "new_status": "operational",
              "old_status": "degraded_performance"
            },
            {
              "code": "458ypjm16xt0",
              "name": "ui.eu1.honeycomb.io - EU1 App Interface",
              "new_status": "operational",
              "old_status": "degraded_performance"
            }
          ],
          "body": "We have deployed the fix and are seeing recovery of normal Canvas functionality.",
          "created_at": "2026-08-21T08:25:14.622-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-08-21T08:25:14.622-07:00",
          "id": "p6s160x37r0w",
          "incident_id": "nhny0dmssfmp",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-08-21T08:25:14.622-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "pw59z78swkns",
              "name": "ui.honeycomb.io - US1 App Interface",
              "new_status": "degraded_performance",
              "old_status": "operational"
            },
            {
              "code": "458ypjm16xt0",
              "name": "ui.eu1.honeycomb.io - EU1 App Interface",
              "new_status": "degraded_performance",
              "old_status": "operational"
            }
          ],
          "body": "Canvas is not loading for customers in the US and EU regions and we have identified what we believe to be the cause and are rolling out a fix.",
          "created_at": "2026-08-21T08:21:56.493-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-08-21T08:21:56.493-07:00",
          "id": "yrbqq5n3x2md",
          "incident_id": "nhny0dmssfmp",
          "status": "identified",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-08-21T08:21:56.493-07:00",
          "wants_twitter_update": false
        }
      ],
      "metadata": {},
      "monitoring_at": "2026-08-21T08:25:14.622-07:00",
      "name": "Canvas not loading",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-08-21T08:59:39.680-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/8395mp4pdgq3",
      "started_at": "2026-08-21T08:21:56.359-07:00",
      "status": "resolved",
      "updated_at": "2026-08-21T08:59:39.699-07:00"
    },
    {
      "components": [
        {
          "created_at": "2017-10-16T19:58:26.301-07:00",
          "description": "Describes the ability to run new queries on existing data, independent of data ingest.",
          "group_id": null,
          "id": "27rh766x05ds",
          "name": "ui.honeycomb.io - US1 Querying",
          "page_id": "9xjvdr2lr7rv",
          "position": 2,
          "showcase": true,
          "start_date": null,
          "status": "operational",
          "updated_at": "2026-08-10T13:35:25.626-07:00"
        }
      ],
      "created_at": "2026-08-10T06:01:42.951-07:00",
      "id": "fqv171nyvhyf",
      "impact": "minor",
      "impact_override": null,
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "27rh766x05ds",
              "name": "ui.honeycomb.io - US1 Querying",
              "new_status": "operational",
              "old_status": "degraded_performance"
            }
          ],
          "body": "The fix is in place and we've confirmed resolution. Queries and SLOs are processing normally.",
          "created_at": "2026-08-10T13:35:25.653-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-08-10T13:35:25.653-07:00",
          "id": "zzj2dq1p60tz",
          "incident_id": "fqv171nyvhyf",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-08-10T13:35:25.653-07:00",
          "wants_twitter_update": true
        },
        {
          "affected_components": [
            {
              "code": "27rh766x05ds",
              "name": "ui.honeycomb.io - US1 Querying",
              "new_status": "degraded_performance",
              "old_status": "operational"
            }
          ],
          "body": "From 11:36UTC to 12:43UTC, queries in US1 that read older data returned errors or timed out. Queries over recent data were un-affected, and no telemetry is being lost.\n\nThe issue has been identified, and a fix is in place. We are currently monitoring the situation to confirm resolution.",
          "created_at": "2026-08-10T06:01:43.048-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-08-10T06:01:43.048-07:00",
          "id": "0wyc3686g2vz",
          "incident_id": "fqv171nyvhyf",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-08-10T06:01:43.048-07:00",
          "wants_twitter_update": true
        }
      ],
      "metadata": {},
      "monitoring_at": "2026-08-10T06:01:43.048-07:00",
      "name": "US1 cold query outage",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-08-10T13:35:25.653-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/n44hjhjkv2jz",
      "started_at": "2026-08-10T06:01:42.941-07:00",
      "status": "resolved",
      "updated_at": "2026-08-10T13:35:25.667-07:00"
    },
    {
      "components": [],
      "created_at": "2026-07-31T07:30:00.000-07:00",
      "id": "by8tbzzgcsmr",
      "impact": "minor",
      "impact_override": "minor",
      "incident_updates": [
        {
          "affected_components": null,
          "body": "Between 10:30 and 11:00 AM EDT, we experienced elevated querying failures and slowness. The impact has subsided, and we are currently investigating and monitoring the situation.",
          "created_at": "2026-07-31T08:42:55.775-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-07-31T07:30:00.000-07:00",
          "id": "yhkh6lsczqpt",
          "incident_id": "by8tbzzgcsmr",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-31T08:42:55.775-07:00",
          "wants_twitter_update": false
        }
      ],
      "metadata": {},
      "monitoring_at": null,
      "name": "Querying issues",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": "Starting at 9:50 AM Central Time on July 31st we received alerts around querying health and latency. The cause was eventually tracked down to a clash of two different factors. First, we have been introducing Lambda Managed Instances \\(LMI\\) into our query architecture, which behave similar to classic lambdas in many ways and had been introduced as part of work around querying performance improvements with the goal of a better and faster user experience. Unfortunately, a difference between classic lambdas and LMIs we discovered is the 15 minute hard stop of classic lambdas does not exist on LMIs, which means that certain assumptions built into our querying system no longer held. At the time of the incident, our newer Canvas feature was running batches of queries for a customer which should have a self imposed one minute time out. This timeout was not properly communicated back to the lambda, and while with classic lambdas the task would automatically cut off at 15 minutes, the new LMIs were not enforcing this sort of safety mechanism. As a result, some large queries were running for up to 30 minutes, driving up query latency across the board and resulting in errors for users trying to run fresh queries. The system recovered by 10:10 AM CT, 20 minutes later, when the large queries cleared out, but we continued to investigate the cause of the incident.\n\nOnce we identified this weakness in our querying architecture we began work to fix it and prevent the same incident from occurring again. We have shipped all associated incident follow ups.",
      "postmortem_body_last_updated_at": "2026-08-21T06:19:24.663-07:00",
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": "2026-08-21T06:19:24.663-07:00",
      "reminder_intervals": null,
      "resolved_at": "2026-07-31T07:30:00.000-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/tkb6gwrml2zz",
      "started_at": "2026-07-31T08:42:55.714-07:00",
      "status": "resolved",
      "updated_at": "2026-08-21T06:19:24.686-07:00"
    },
    {
      "components": [
        {
          "created_at": "2024-01-24T08:02:34.673-08:00",
          "description": "Describes the ability to run new queries on existing data, independent of data ingest.",
          "group_id": null,
          "id": "pnss6nhdwd65",
          "name": "ui.eu1.honeycomb.io - EU1 Querying",
          "page_id": "9xjvdr2lr7rv",
          "position": 9,
          "showcase": true,
          "start_date": "2023-10-01T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-07-28T11:30:16.271-07:00"
        },
        {
          "created_at": "2024-01-24T08:03:13.237-08:00",
          "description": null,
          "group_id": null,
          "id": "sdnpgrc81h4v",
          "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
          "page_id": "9xjvdr2lr7rv",
          "position": 10,
          "showcase": true,
          "start_date": "2023-10-01T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-07-28T11:30:16.289-07:00"
        }
      ],
      "created_at": "2026-07-27T06:13:36.757-07:00",
      "id": "h61nm0nz802x",
      "impact": "minor",
      "impact_override": null,
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "pnss6nhdwd65",
              "name": "ui.eu1.honeycomb.io - EU1 Querying",
              "new_status": "operational",
              "old_status": "degraded_performance"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "degraded_performance"
            }
          ],
          "body": "The degradation in querying and triggers has been resolved and we're back to full functionality.",
          "created_at": "2026-07-28T11:30:16.319-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-28T11:30:16.319-07:00",
          "id": "z5hxg84pq7v1",
          "incident_id": "h61nm0nz802x",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-28T11:30:16.319-07:00",
          "wants_twitter_update": true
        },
        {
          "affected_components": [
            {
              "code": "pnss6nhdwd65",
              "name": "ui.eu1.honeycomb.io - EU1 Querying",
              "new_status": "degraded_performance",
              "old_status": "degraded_performance"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "degraded_performance"
            }
          ],
          "body": "We are continuing to monitor for any further issues.",
          "created_at": "2026-07-28T11:21:11.006-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-28T11:21:11.006-07:00",
          "id": "v2j4zc2c6qlg",
          "incident_id": "h61nm0nz802x",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-28T11:21:11.006-07:00",
          "wants_twitter_update": true
        },
        {
          "affected_components": [
            {
              "code": "pnss6nhdwd65",
              "name": "ui.eu1.honeycomb.io - EU1 Querying",
              "new_status": "degraded_performance",
              "old_status": "degraded_performance"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "degraded_performance"
            }
          ],
          "body": "The partial degradation has been resolved for all customers except for those specifically contacted. We are working on mitigation measures so that we can fully restore trigger functionality for everyone.",
          "created_at": "2026-07-27T15:51:04.427-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-27T15:51:04.427-07:00",
          "id": "ldnz1fsxylkf",
          "incident_id": "h61nm0nz802x",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-27T15:51:04.427-07:00",
          "wants_twitter_update": true
        },
        {
          "affected_components": [
            {
              "code": "pnss6nhdwd65",
              "name": "ui.eu1.honeycomb.io - EU1 Querying",
              "new_status": "degraded_performance",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "operational"
            }
          ],
          "body": "We are seeing a regression in performance with partial degradation of both triggers and querying. We are working to remediate.",
          "created_at": "2026-07-27T12:46:27.140-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-27T12:46:27.000-07:00",
          "id": "twny8w9n4v6g",
          "incident_id": "h61nm0nz802x",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-27T12:46:51.638-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "pnss6nhdwd65",
              "name": "ui.eu1.honeycomb.io - EU1 Querying",
              "new_status": "operational",
              "old_status": "degraded_performance"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "degraded_performance"
            }
          ],
          "body": "We have identified the source of the issue, applied a temporary fix, and are working on a permanent fix. We continue to monitor the situation.",
          "created_at": "2026-07-27T09:01:10.448-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-27T09:01:10.448-07:00",
          "id": "tqgbn5frwk3j",
          "incident_id": "h61nm0nz802x",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-27T09:01:10.448-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "pnss6nhdwd65",
              "name": "ui.eu1.honeycomb.io - EU1 Querying",
              "new_status": "degraded_performance",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "degraded_performance"
            }
          ],
          "body": "We are continuing to see issues with triggers and are also noticing issues with querying. We are investigating for the cause of both.",
          "created_at": "2026-07-27T06:52:50.212-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-27T06:52:50.212-07:00",
          "id": "2hdjb2pf77z2",
          "incident_id": "h61nm0nz802x",
          "status": "investigating",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-27T06:52:50.212-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "operational"
            }
          ],
          "body": "We are seeing triggers intermittently failing in the EU region. We are actively investigating the cause.",
          "created_at": "2026-07-27T06:13:36.845-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-27T06:13:36.845-07:00",
          "id": "6g42p5c5tnyw",
          "incident_id": "h61nm0nz802x",
          "status": "investigating",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-27T06:13:36.845-07:00",
          "wants_twitter_update": false
        }
      ],
      "metadata": {},
      "monitoring_at": "2026-07-27T09:01:10.448-07:00",
      "name": "Triggers intermittently failing in EU region",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-07-28T11:30:16.319-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/0x8zyh2jyvgz",
      "started_at": "2026-07-27T06:13:36.749-07:00",
      "status": "resolved",
      "updated_at": "2026-07-28T11:30:16.336-07:00"
    },
    {
      "components": [
        {
          "created_at": "2020-06-29T08:40:36.995-07:00",
          "description": "Marketing, sales, and documentation.",
          "group_id": null,
          "id": "1hk92bykzr99",
          "name": "www.honeycomb.io",
          "page_id": "9xjvdr2lr7rv",
          "position": 13,
          "showcase": false,
          "start_date": null,
          "status": "operational",
          "updated_at": "2026-07-02T11:13:29.892-07:00"
        }
      ],
      "created_at": "2026-07-02T11:10:57.661-07:00",
      "id": "y19shm1scfbt",
      "impact": "major",
      "impact_override": "major",
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "1hk92bykzr99",
              "name": "www.honeycomb.io",
              "new_status": "operational",
              "old_status": "operational"
            }
          ],
          "body": "This incident has been resolved.",
          "created_at": "2026-07-02T17:31:47.636-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-07-02T17:31:47.636-07:00",
          "id": "889vfq6gwmy3",
          "incident_id": "y19shm1scfbt",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-02T17:31:47.636-07:00",
          "wants_twitter_update": true
        },
        {
          "affected_components": [
            {
              "code": "1hk92bykzr99",
              "name": "www.honeycomb.io",
              "new_status": "operational",
              "old_status": "major_outage"
            }
          ],
          "body": "Clearing the CDN cache appears to have solved the issue.",
          "created_at": "2026-07-02T11:13:29.921-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-07-02T11:13:29.921-07:00",
          "id": "n7nvq7k46hsp",
          "incident_id": "y19shm1scfbt",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-02T11:13:29.921-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "1hk92bykzr99",
              "name": "www.honeycomb.io",
              "new_status": "major_outage",
              "old_status": "operational"
            }
          ],
          "body": "We are currently investigating this issue.",
          "created_at": "2026-07-02T11:10:57.768-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-07-02T11:10:57.768-07:00",
          "id": "krmbnh22ldn0",
          "incident_id": "y19shm1scfbt",
          "status": "investigating",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-07-02T11:10:57.768-07:00",
          "wants_twitter_update": false
        }
      ],
      "metadata": {},
      "monitoring_at": "2026-07-02T11:13:29.921-07:00",
      "name": "honeycomb.io marketing website not working",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-07-02T17:31:47.636-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/vcwr4fk6vh6p",
      "started_at": "2026-07-02T11:10:57.651-07:00",
      "status": "resolved",
      "updated_at": "2026-07-02T17:31:47.650-07:00"
    },
    {
      "components": [
        {
          "created_at": "2024-04-10T04:46:50.485-07:00",
          "description": null,
          "group_id": null,
          "id": "ryhsgbnrzg2l",
          "name": "ui.honeycomb.io - US1 Activity Log",
          "page_id": "9xjvdr2lr7rv",
          "position": 5,
          "showcase": true,
          "start_date": "2024-03-27T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-06-29T17:47:15.291-07:00"
        }
      ],
      "created_at": "2026-06-29T15:07:36.986-07:00",
      "id": "8st0182rtr59",
      "impact": "minor",
      "impact_override": null,
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "ryhsgbnrzg2l",
              "name": "ui.honeycomb.io - US1 Activity Log",
              "new_status": "operational",
              "old_status": "degraded_performance"
            }
          ],
          "body": "Backfill of the Activity Log events has completed. All events during the affected period should be present.",
          "created_at": "2026-06-29T17:47:15.319-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-06-29T17:47:15.319-07:00",
          "id": "yr36mw7kxvlj",
          "incident_id": "8st0182rtr59",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-29T17:47:15.319-07:00",
          "wants_twitter_update": true
        },
        {
          "affected_components": [
            {
              "code": "ryhsgbnrzg2l",
              "name": "ui.honeycomb.io - US1 Activity Log",
              "new_status": "degraded_performance",
              "old_status": "operational"
            }
          ],
          "body": "Starting at 14:24 PT, there was an issue with the Activity Log causing events to stop being processed. You may notice a gap starting at that time. We are slowly backfilling these events and do not expect any data loss to occur. We expect the full data log to be restored around 17:30 PT.",
          "created_at": "2026-06-29T15:07:37.071-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-06-29T15:07:37.071-07:00",
          "id": "1bfzwtrsvrwx",
          "incident_id": "8st0182rtr59",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-29T15:07:37.071-07:00",
          "wants_twitter_update": true
        }
      ],
      "metadata": {},
      "monitoring_at": "2026-06-29T15:07:37.071-07:00",
      "name": "Activity Log delayed in US",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-06-29T17:47:15.319-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/q1fwpz556hft",
      "started_at": "2026-06-29T15:07:36.978-07:00",
      "status": "resolved",
      "updated_at": "2026-06-29T17:47:15.333-07:00"
    },
    {
      "components": [
        {
          "created_at": "2024-01-24T08:00:37.417-08:00",
          "description": "POST endpoint to submit events",
          "group_id": null,
          "id": "r7lr006x58vg",
          "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
          "page_id": "9xjvdr2lr7rv",
          "position": 7,
          "showcase": true,
          "start_date": "2023-10-01T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-06-25T11:31:29.039-07:00"
        },
        {
          "created_at": "2024-01-24T08:03:13.237-08:00",
          "description": null,
          "group_id": null,
          "id": "sdnpgrc81h4v",
          "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
          "page_id": "9xjvdr2lr7rv",
          "position": 10,
          "showcase": true,
          "start_date": "2023-10-01T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-07-28T11:30:16.289-07:00"
        }
      ],
      "created_at": "2026-06-25T11:10:03.918-07:00",
      "id": "zctrrynfyh8t",
      "impact": "minor",
      "impact_override": null,
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "r7lr006x58vg",
              "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "operational"
            }
          ],
          "body": "This incident has been resolved.",
          "created_at": "2026-06-25T13:41:40.549-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-06-25T13:41:40.549-07:00",
          "id": "1xj6zvhbxfv8",
          "incident_id": "zctrrynfyh8t",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-25T13:41:40.549-07:00",
          "wants_twitter_update": true
        },
        {
          "affected_components": [
            {
              "code": "r7lr006x58vg",
              "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
              "new_status": "operational",
              "old_status": "degraded_performance"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "degraded_performance"
            }
          ],
          "body": "We are monitoring the situation. Events may have been delayed between 17:53 and 18:04 UTC.",
          "created_at": "2026-06-25T11:31:29.088-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-06-25T11:31:29.088-07:00",
          "id": "89dn0y3gkl2g",
          "incident_id": "zctrrynfyh8t",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-25T11:31:29.088-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "r7lr006x58vg",
              "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
              "new_status": "degraded_performance",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "operational"
            }
          ],
          "body": "We are investigating an issue with delayed ingest in the EU region. SLOs and Trigger evaluations may also be delayed.",
          "created_at": "2026-06-25T11:10:04.022-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-06-25T11:10:04.022-07:00",
          "id": "xhgjn0sb5k8j",
          "incident_id": "zctrrynfyh8t",
          "status": "investigating",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-25T11:10:04.022-07:00",
          "wants_twitter_update": true
        }
      ],
      "metadata": {},
      "monitoring_at": "2026-06-25T11:31:29.088-07:00",
      "name": "Ingest outage in EU",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-06-25T13:41:40.549-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/d4pj7wczmdxt",
      "started_at": "2026-06-25T11:10:03.909-07:00",
      "status": "resolved",
      "updated_at": "2026-06-25T13:41:40.566-07:00"
    },
    {
      "components": [
        {
          "created_at": "2017-10-13T17:30:35.373-07:00",
          "description": "POST endpoint to submit events",
          "group_id": null,
          "id": "m8wyq69d24y8",
          "name": "api.honeycomb.io - US1 Event Ingest",
          "page_id": "9xjvdr2lr7rv",
          "position": 1,
          "showcase": true,
          "start_date": null,
          "status": "operational",
          "updated_at": "2026-06-24T12:06:10.569-07:00"
        },
        {
          "created_at": "2021-11-23T12:51:23.891-08:00",
          "description": null,
          "group_id": null,
          "id": "p5q31vg0lzrn",
          "name": "ui.honeycomb.io - US1 Trigger \u0026 SLO Alerting",
          "page_id": "9xjvdr2lr7rv",
          "position": 4,
          "showcase": true,
          "start_date": "2021-11-23T00:00:00.000-08:00",
          "status": "operational",
          "updated_at": "2026-06-24T12:06:53.203-07:00"
        },
        {
          "created_at": "2024-01-24T08:00:37.417-08:00",
          "description": "POST endpoint to submit events",
          "group_id": null,
          "id": "r7lr006x58vg",
          "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
          "page_id": "9xjvdr2lr7rv",
          "position": 7,
          "showcase": true,
          "start_date": "2023-10-01T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-06-25T11:31:29.039-07:00"
        },
        {
          "created_at": "2024-01-24T08:03:13.237-08:00",
          "description": null,
          "group_id": null,
          "id": "sdnpgrc81h4v",
          "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
          "page_id": "9xjvdr2lr7rv",
          "position": 10,
          "showcase": true,
          "start_date": "2023-10-01T00:00:00.000-07:00",
          "status": "operational",
          "updated_at": "2026-07-28T11:30:16.289-07:00"
        }
      ],
      "created_at": "2026-06-24T11:14:02.425-07:00",
      "id": "rqp7q8pmm54y",
      "impact": "major",
      "impact_override": "major",
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "m8wyq69d24y8",
              "name": "api.honeycomb.io - US1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "p5q31vg0lzrn",
              "name": "ui.honeycomb.io - US1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "r7lr006x58vg",
              "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "operational"
            }
          ],
          "body": "All services have been fully recovered.\nImpact window: 17:55–18:17 UTC\nRegions affected: Primary impact in US. The EU region was affected to a much lesser degree.\nDuring this period, the following effects may have occurred:\nIngest: Some data was not ingested, resulting in complete or partial data loss for events sent during the window.\nTriggers: Triggers may have failed to fire within the impact window.\nSLOs: SLI values across the affected window are skewed by the missing data and may show artificial dips or accelerated budget burn.\nService Maps: Maps covering the outage window are incomplete. Services and dependencies may be under-counted, as their traces were not ingested.\nWe apologize for the disruption. Please reach out if you have any questions about how this may have affected your data.",
          "created_at": "2026-06-24T12:04:49.421-07:00",
          "custom_tweet": null,
          "deliver_notifications": true,
          "display_at": "2026-06-24T12:04:49.421-07:00",
          "id": "5px8t3rqpgq2",
          "incident_id": "rqp7q8pmm54y",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-24T12:04:49.421-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "m8wyq69d24y8",
              "name": "api.honeycomb.io - US1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "p5q31vg0lzrn",
              "name": "ui.honeycomb.io - US1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "degraded_performance"
            },
            {
              "code": "r7lr006x58vg",
              "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "degraded_performance"
            }
          ],
          "body": "Triggers, SLOs and Service Maps are recovered. We are continuing to monitoring the Ingest Service",
          "created_at": "2026-06-24T11:31:37.653-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-06-24T11:31:37.653-07:00",
          "id": "f594bh4ykfcc",
          "incident_id": "rqp7q8pmm54y",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-24T11:31:37.653-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "m8wyq69d24y8",
              "name": "api.honeycomb.io - US1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "p5q31vg0lzrn",
              "name": "ui.honeycomb.io - US1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "operational"
            },
            {
              "code": "r7lr006x58vg",
              "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "degraded_performance",
              "old_status": "operational"
            }
          ],
          "body": "We have rolled back the deploy and ingest service has recovered. There has been an ingest outage from 17:55 - 18:14 UTC. We have also observed an delay for Service Maps.",
          "created_at": "2026-06-24T11:22:08.208-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-06-24T11:22:08.208-07:00",
          "id": "r2k30sf9kk7w",
          "incident_id": "rqp7q8pmm54y",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-24T11:22:08.208-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": [
            {
              "code": "m8wyq69d24y8",
              "name": "api.honeycomb.io - US1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "p5q31vg0lzrn",
              "name": "ui.honeycomb.io - US1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "r7lr006x58vg",
              "name": "api.eu1.honeycomb.io - EU1 Event Ingest",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "sdnpgrc81h4v",
              "name": "ui.eu1.honeycomb.io - EU1 Trigger \u0026 SLO Alerting",
              "new_status": "operational",
              "old_status": "operational"
            }
          ],
          "body": "We are investigating an issue with delayed ingest in the US and EU region. Our engineers are rolling back a recent deploy. SLOs and Trigger evaluations may also be delayed.",
          "created_at": "2026-06-24T11:14:02.639-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-06-24T11:14:02.639-07:00",
          "id": "ztz0kq11xrfy",
          "incident_id": "rqp7q8pmm54y",
          "status": "investigating",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-24T11:14:02.639-07:00",
          "wants_twitter_update": false
        }
      ],
      "metadata": {
        "jeli": {
          "updatedAt": "1782327888.9863877"
        }
      },
      "monitoring_at": "2026-06-24T11:22:08.208-07:00",
      "name": "Ingest Outage in US and EU",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": "On June 24, we experienced approximately 30 minutes of severe data ingestion degradation in Honeycomb’s US and EU instances. During this degradation, Honeycomb rejected a significant percentage of inbound telemetry, and customers would have seen up to a half-hour gap in ingested telemetry. Additionally, during this same time window, customers would have experienced data processing degradation and service disruption within Anomaly Detection and Service Maps.\n\nA deployment containing a change to how services retrieve dataset schemas into local caches caused a cascading set of failures, resulting in failed remote cache retrievals, and subsequently, a MySQL stampede due to several services falling back to retrieving schemas from the database. This stampede resulted in significant memory utilization growth, causing some services to crash loop with out-of-memory exceptions while retrieving schemas. Customers would have seen 5xx errors from our ingestion APIs when this occurred, because ingestion services were among those that were crash looping. This also included crash looping from services responsible for Anomaly Detection and Service Maps.\n\nRight after these service crash loops began, alerts fired for the crashing services, and procedures were run to pin all services back to the last known good build before the breaking change landed. Once services were running the previous deploy’s build, the 5xx error responses from ingestion APIs returned to nominal baseline levels, and services that were crash looping recovered and resumed normal operation.\n\nThe underlying issue centered on how dataset schema cache payloads and metadata were serialized to memcached for remote cache retrieval, and how memcached clients deserialize that data before writing it to a local cache. The code change contained a feature flag to toggle the serialization behavior, and backwards-compatibility safeguards for clients deserializing different schema versions from memcached. However, services’ schema deserialization prior to the feature flag flip on did not behave as intended, treating each backwards-compatible schema read as a hard error rather than as a fallback, which subsequently caused every remote schema cache read to fall through to the database. Subsequent analysis and review of the code identified the issue and we have re-deployed this change with a fix along with additional tests. Services now correctly and successfully deserialize schemas from memcached in a safe, backwards-compatible manner.",
      "postmortem_body_last_updated_at": "2026-07-02T13:15:51.770-07:00",
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": true,
      "postmortem_notified_twitter": true,
      "postmortem_published_at": "2026-07-02T13:15:51.770-07:00",
      "reminder_intervals": null,
      "resolved_at": "2026-06-24T12:04:49.421-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/4zlcbvz7ky7r",
      "started_at": "2026-06-24T11:14:02.402-07:00",
      "status": "resolved",
      "updated_at": "2026-07-02T13:15:51.785-07:00"
    },
    {
      "components": [],
      "created_at": "2026-06-19T09:39:32.755-07:00",
      "id": "qkgzs5msfltc",
      "impact": "minor",
      "impact_override": "minor",
      "incident_updates": [
        {
          "affected_components": null,
          "body": "MCP Write Tools have been restored. MCP users may need to reconnect to see all available tools.",
          "created_at": "2026-06-19T10:21:14.000-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-06-19T10:21:14.000-07:00",
          "id": "hxv9rgghfvzd",
          "incident_id": "qkgzs5msfltc",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-19T10:21:14.000-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": null,
          "body": "We are aware of an issue affecting Honeycomb MCP. Write tool functionality may be degraded. Read tools are unaffected.",
          "created_at": "2026-06-19T10:18:03.059-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-06-19T10:18:03.059-07:00",
          "id": "85k14gwshzl0",
          "incident_id": "qkgzs5msfltc",
          "status": "monitoring",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-19T10:18:03.059-07:00",
          "wants_twitter_update": false
        },
        {
          "affected_components": null,
          "body": "We are aware of an issue affecting Honeycomb MCP. Write tool functionality may be degraded. Read tools are unaffected.",
          "created_at": "2026-06-19T09:39:32.924-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-06-19T09:39:32.924-07:00",
          "id": "24s3btznk2m5",
          "incident_id": "qkgzs5msfltc",
          "status": "investigating",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-19T09:39:32.924-07:00",
          "wants_twitter_update": false
        }
      ],
      "metadata": {
        "jeli": {
          "updatedAt": 1781889673.675694
        }
      },
      "monitoring_at": "2026-06-19T10:18:03.059-07:00",
      "name": "MCP tool access degraded",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-06-19T10:21:14.000-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/7pdqs6m58kj2",
      "started_at": "2026-06-19T09:39:32.742-07:00",
      "status": "resolved",
      "updated_at": "2026-06-19T10:21:14.030-07:00"
    },
    {
      "components": [
        {
          "created_at": "2026-06-18T11:17:57.239-07:00",
          "description": null,
          "group_id": null,
          "id": "plh6yjykc52q",
          "name": "ui.honeycomb.io - US1 Enhance",
          "page_id": "9xjvdr2lr7rv",
          "position": 6,
          "showcase": true,
          "start_date": "2026-01-01T00:00:00.000-08:00",
          "status": "operational",
          "updated_at": "2026-06-18T12:15:29.365-07:00"
        },
        {
          "created_at": "2026-06-18T11:18:32.483-07:00",
          "description": null,
          "group_id": null,
          "id": "6zprzdf74q84",
          "name": "ui.eu1.honeycomb.io - EU1 Enhance",
          "page_id": "9xjvdr2lr7rv",
          "position": 12,
          "showcase": true,
          "start_date": "2026-01-01T00:00:00.000-08:00",
          "status": "operational",
          "updated_at": "2026-06-18T12:16:04.270-07:00"
        }
      ],
      "created_at": "2026-06-18T12:10:04.990-07:00",
      "id": "kyljjc0hqmx7",
      "impact": "major",
      "impact_override": "major",
      "incident_updates": [
        {
          "affected_components": [
            {
              "code": "plh6yjykc52q",
              "name": "ui.honeycomb.io - US1 Enhance",
              "new_status": "operational",
              "old_status": "operational"
            },
            {
              "code": "6zprzdf74q84",
              "name": "ui.eu1.honeycomb.io - EU1 Enhance",
              "new_status": "operational",
              "old_status": "operational"
            }
          ],
          "body": "Beginning June 17th, enhance-related requests were failing to complete. We've since identified and deployed a fix, and enhance is now fully operational. Thank you for your patience.",
          "created_at": "2026-06-18T12:10:05.125-07:00",
          "custom_tweet": null,
          "deliver_notifications": false,
          "display_at": "2026-06-18T12:10:05.125-07:00",
          "id": "tznyxklydb04",
          "incident_id": "kyljjc0hqmx7",
          "status": "resolved",
          "tweet_id": null,
          "twitter_updated_at": null,
          "updated_at": "2026-06-18T12:10:05.125-07:00",
          "wants_twitter_update": false
        }
      ],
      "metadata": {
        "jeli": {
          "updatedAt": "1781809804.2150087"
        }
      },
      "monitoring_at": null,
      "name": "Enhance Issues",
      "page_id": "9xjvdr2lr7rv",
      "postmortem_body": null,
      "postmortem_body_last_updated_at": null,
      "postmortem_ignored": false,
      "postmortem_notified_subscribers": false,
      "postmortem_notified_twitter": false,
      "postmortem_published_at": null,
      "reminder_intervals": null,
      "resolved_at": "2026-06-18T12:10:05.125-07:00",
      "scheduled_auto_completed": false,
      "scheduled_auto_in_progress": false,
      "scheduled_for": null,
      "scheduled_remind_prior": false,
      "scheduled_reminded_at": null,
      "scheduled_until": null,
      "shortlink": "https://stspg.io/zws9bkwntjkn",
      "started_at": "2026-06-18T12:10:04.973-07:00",
      "status": "resolved",
      "updated_at": "2026-06-18T12:10:05.154-07:00"
    }
  ],
  "page": {
    "id": "9xjvdr2lr7rv",
    "name": "Honeycomb",
    "url": "https://status.honeycomb.io"
  },
  "status": {
    "description": "All Systems Operational",
    "indicator": "none"
  }
}