Snapshot 225367
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"45yb5qck0sw5",
"hsv3glmmrs22",
"wp2dc3v8nykv"
],
"created_at": "2025-04-02T13:04:45.201-07:00",
"description": "Mixpanel user interface and query API",
"group_id": null,
"id": "v5rct9xjfslj",
"name": "Application Availability",
"page_id": "x4m91ldrf511",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-04-02T13:19:59.343-07:00"
},
{
"components": [
"cq5ttq9czyjh",
"47wk10r6f8mm",
"fyfmjh67mrgs",
"v1kc88rj0qmw"
],
"created_at": "2025-04-02T13:08:23.686-07:00",
"description": "SDK events, Ingestion APIs, Warehouse Connectors",
"group_id": null,
"id": "jwswcg5s8vp8",
"name": "Data Ingestion",
"page_id": "x4m91ldrf511",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-04-02T13:49:44.518-07:00"
},
{
"created_at": "2016-03-29T13:59:34.952-07:00",
"description": "Data Pipelines, Cohort Sync, Export API",
"group_id": null,
"id": "tjt23lh5r46g",
"name": "Data Export",
"page_id": "x4m91ldrf511",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-23T00:45:00.865-07:00"
},
{
"created_at": "2016-03-29T13:59:53.252-07:00",
"description": "JavaScript tracking library hosting at cdn.mxpnl.com",
"group_id": null,
"id": "7cdmvbkr0dyq",
"name": "JavaScript Library CDN",
"page_id": "x4m91ldrf511",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-06-12T11:07:13.309-07:00"
}
],
"incidents": [
{
"components": [],
"created_at": "2026-11-04T21:00:00.000-08:00",
"id": "szs4zzvftwyz",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "A technical issue with one of our payment systems caused charges to automatically rerun on a subset of customer accounts, so some customers were charged more than once for the same invoice.\n\nInvoice amounts were correct. Only the payment charges were in error. All amounts charged above each affected customer’s invoice balance have been refunded to the original payment method. Refunds may take 5-10 business days to appear, depending on the customer’s financial institution.\n\nAffected customers have been contacted directly. No action is required. If you have questions about a charge on your account, please contact support@mixpanel.com.",
"created_at": "2026-11-05T22:05:38.211-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-04T21:00:00.000-08:00",
"id": "lq8bn3nylfhn",
"incident_id": "szs4zzvftwyz",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-05T22:05:38.211-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Duplicate charges on a subset of customer accounts",
"page_id": "x4m91ldrf511",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-04T21:00:00.000-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/6d4884lmkf28",
"started_at": "2026-11-05T22:05:38.143-08:00",
"status": "resolved",
"updated_at": "2026-11-05T22:05:38.224-08:00"
},
{
"components": [],
"created_at": "2026-10-21T11:00:43.000-07:00",
"id": "xgznm95n8t2b",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2026-10-21T18:34:02.013-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-21T18:34:02.013-07:00",
"id": "89dq747dms54",
"incident_id": "xgznm95n8t2b",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T18:34:02.013-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We've received reports that requests to decide.mixpanel.com are failing.\n\nAny traffic sent through decide.mixpanel.com may be affected, including feature flags and event tracking.\n\nTo restore service now, point requests to api.mixpanel.com (US), api-eu.mixpanel.com (EU), or api-in.mixpanel.com (IN), depending on your project's Data Residency.\n\nWe've rolled out a mitigation and are monitoring the results. We're seeing traffic restored. \n\nWe will post progress updates on our status page.\n\nIf you have any questions, please \u003ca href=\"https://mixpanel.com/get-support\"\u003econtact support.\u003c/a\u003e",
"created_at": "2026-10-21T15:28:43.980-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-21T11:00:43.000-07:00",
"id": "crn6ck50mm1g",
"incident_id": "xgznm95n8t2b",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-21T15:31:23.327-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-21T11:00:43.000-07:00",
"name": "Degraded Service for Requests to decide.mixpanel.com",
"page_id": "x4m91ldrf511",
"postmortem_body": "# **Mixpanel RCA: Requests to** [**decide.mixpanel.com**](http://decide.mixpanel.com) **Failed, October 15–21, 2026**\n\n### **Summary**\n\nFrom Oct 15, 2026 to Oct 21, 2026, requests sent to `decide.mixpanel.com` and `api-decide.mixpanel.com` failed.\n\nThis affected customers who sent event tracking, profile updates, or feature flag requests to this hostname, usually through a proxy.\n\nEvents sent to this domain during this window were not received by Mixpanel and this data cannot be restored by Mixpanel.\n\n**Traffic sent to** `api.mixpanel.com`**,** `api-js.mixpanel.com,` `api-eu.mixpanel.com`**, and** `api-in.mixpanel.com` **were not affected.**\n\nService on this hostname was unintentionally impacted while we were removing the `/decide` endpoint infrastructure. The hostnames were treated as part of the endpoint due to an incorrect understanding of system dependencies. To prevent this class of issue, we are adding traffic monitoring for the `decide.mixpanel.com` and `api-decide.mixpanel.com` hostnames. We will also require confirmed zero traffic at the hostname level before any hostname is removed.\n\n**We have since restored service on** `decide.mixpanel.com` .\n\n### **What happened**\n\nOlder Mixpanel SDKs called the `/decide` endpoint to fetch configuration for [**Legacy Automatically Tracked Mobile Events**](https://docs.mixpanel.com/docs/tracking-methods/sdks/ios#legacy-automatically-tracked-events). We [announced the endpoint’s retirement](https://docs.mixpanel.com/changelogs#2026-01-26), and for some time `/decide` has returned a `410` response.\n\nSeparately, `decide.mixpanel.com` and `api-decide.mixpanel.com` acted as aliases for our standard API hostnames. For example, `decide.mixpanel.com/track` ingested events the same way `api-js.mixpanel.com/track` does. Some customers had set up proxies or SDK configurations to send their traffic to these hostnames.\n\nOn October 15 at 7:40 PM CDT, during the final cleanup of the `/decide` endpoint, we removed the DNS records for `decide.mixpanel.com` and `api-decide.mixpanel.com` and removed both hostnames from our load balancer.\n\nBefore making the change, we confirmed that traffic to the `/decide` endpoint had dropped to zero. However, our monitoring measured traffic by endpoint rather than by hostname. It therefore did not show that other requests, such as `/track` and `/flags`, were still reaching Mixpanel through these hostnames.\n\nFrom then on, all requests to this hostname failed, including endpoints such as `/track`, `/engage`, `/groups`, and `/flags`.\n\n* **Events and Profiles:** Events, Profiles, and any data intended for [Ingestion API endpoints](https://docs.mixpanel.com/reference/ingestion-api) to these hostnames during this window never reached Mixpanel, so we do not have a copy to restore. If you retained this data elsewhere, such as in proxy logs or your data warehouse, you can re-send it to Ingestion API’s [/import](https://docs.mixpanel.com/reference/import-events), [/engage,](https://docs.mixpanel.com/reference/profile-set) and [/groups](https://docs.mixpanel.com/reference/group-set-property).\n* **Feature flags:** Feature flag requests sent to these hostnames failed. SDKs served the last stored variant if one existed, or otherwise the fallback value configured in your code.\n\n### Timeline \\(Central Time, October 2026\\)\n\n* **October 15, 7:40 PM CDT:** The hostname removal is applied. Requests to `decide.mixpanel.com` and `api-decide.mixpanel.com` begin failing.\n* **October 16, 5:22 AM CDT:** We receive the first customer report. It is initially understood to be related to the planned `/decide` retirement.\n* **October 21, 11:42 AM CDT:** Following an increase in customer reports about tracking failing for proxies, we identify the hostname removal as the cause.\n* **October 21, 12:26 PM CDT:** We restore the hostnames. `/decide` remains retired.\n* **October 21, 1:51 PM CDT:** We confirm traffic to the restored hostnames has recovered.\n* **October 21, 8:34 PM CDT:** The incident is resolved.\n\n### **Root cause**\n\n1. During the `/decide` retirement, `decide.mixpanel.com` and `api-decide.mixpanel.com` were unintentionally removed.\n2. These hostnames had been configured to accept all API traffic, not only `/decide` requests, and this behavior was undocumented.\n3. Our monitoring tracked traffic to the `/decide` endpoint but not to these hostnames, so pre-change checks did not show that other traffic was still active.\n4. Because the hostnames and the retired endpoint share a name, early customer reports were initially interpreted as expected effects of the announced `/decide` retirement. This delayed escalation.\n\n### **What we’re changing**\n\n* **Monitoring and alerting:** We are adding traffic and error-rate monitoring and alerting for `decide.mixpanel.com` and `api-decide.mixpanel.com`, so failures are detected automatically rather than through customer reports.\n* **Hostname audit:** We are auditing all public API hostnames and routing aliases for similar undocumented entry points, and we will review new hostnames and aliases before they go live. We are also tracking traffic that arrives on any unexpected hostname.\n* **Safer removals:** Before any future change to a public hostname, we will confirm zero traffic at the hostname level, and we will identify and contact any projects still using it.\n* **Customer communication:** We will give customers advance notice of material infrastructure changes that affect public hostnames, including publishing them on a timeline in our changelog.\n\n**Already deployed:**\n\n* Restored `decide.mixpanel.com` and `api-decide.mixpanel.com`\n* `/decide` remains retired\n\n#### **Common questions**\n\n* **Was my data affected?** Only if your SDK or proxy sent requests to `decide.mixpanel.com` or `api-decide.mixpanel.com` between October 15 and October 21. Data sent there during that window were not received and cannot be restored by Mixpanel. If you retained a copy of that data, you can re-send it to Mixpanel using our Ingestion API endpoints such as `/import` for events.\n* **Do I need to change anything?** No, the domains work again. We recommend using `api.mixpanel.com` \\(or `api-eu.mixpanel.com` / `api-in.mixpanel.com` for EU and India data residency\\) as your API domain.\n* **Is** `/decide` coming back? No. It remains retired as announced, and it is separate from the `decide.mixpanel.com` hostname.\n\nWe’re sorry for the disruption. If you have questions about your project, please contact your account team or Mixpanel Support.",
"postmortem_body_last_updated_at": "2026-10-23T19:01:10.262-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-23T19:01:10.262-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-21T18:34:02.013-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/40d37f52s2qx",
"started_at": "2026-10-21T11:00:43.000-07:00",
"status": "resolved",
"updated_at": "2026-10-23T19:01:10.378-07:00"
},
{
"components": [],
"created_at": "2026-10-08T09:32:31.501-07:00",
"id": "xxsxtjhxk86l",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved. The upstream issue with our cloud storage provider has been fixed, and our ingestion pipeline has been operating normally with no active delays. No events were lost throughout this incident, all affected data was fully reprocessed.\n\nThank you for your patience throughout this incident.",
"created_at": "2026-10-08T17:22:41.067-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T17:22:41.067-07:00",
"id": "hw58007r866d",
"incident_id": "xxsxtjhxk86l",
"status": "resolved",
"tweet_id": 2108352414122459600,
"twitter_updated_at": "2026-10-08T17:22:41.646-07:00",
"updated_at": "2026-10-08T17:22:41.649-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "Systems have remained stable with no recurrence of the ingestion delay. We’re keeping a close eye on things while we wait for final confirmation from our upstream provider.",
"created_at": "2026-10-08T16:30:13.141-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T16:30:13.141-07:00",
"id": "k703rgn7r4vd",
"incident_id": "xxsxtjhxk86l",
"status": "monitoring",
"tweet_id": 2108339210877043000,
"twitter_updated_at": "2026-10-08T16:30:13.745-07:00",
"updated_at": "2026-10-08T16:30:13.748-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "We are continuing to monitor for any further issues.",
"created_at": "2026-10-08T15:30:30.145-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T15:30:30.145-07:00",
"id": "4f57vdk9n9z3",
"incident_id": "xxsxtjhxk86l",
"status": "monitoring",
"tweet_id": 2108324182245450000,
"twitter_updated_at": "2026-10-08T15:30:30.623-07:00",
"updated_at": "2026-10-08T15:30:30.626-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "No change since our last update, we continue to observe no active ingestion delay on our end, and all systems remain healthy. We’re continuing to monitor closely.",
"created_at": "2026-10-08T14:30:11.409-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T14:30:11.409-07:00",
"id": "81qncchqfgkx",
"incident_id": "xxsxtjhxk86l",
"status": "monitoring",
"tweet_id": 2108309026098593800,
"twitter_updated_at": "2026-10-08T14:30:17.388-07:00",
"updated_at": "2026-10-08T14:30:17.391-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "We are continuing to monitor for any further issues.",
"created_at": "2026-10-08T13:30:56.264-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T13:30:56.264-07:00",
"id": "l1665m5t2cjl",
"incident_id": "xxsxtjhxk86l",
"status": "monitoring",
"tweet_id": 2108294093004763600,
"twitter_updated_at": "2026-10-08T13:30:56.785-07:00",
"updated_at": "2026-10-08T13:30:56.788-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "Our metrics show ingestion is caught up and no longer experiencing delay. Google has not yet confirmed resolution of the underlying upstream issue on their end, so we’re continuing to monitor closely in case of recurrence.",
"created_at": "2026-10-08T12:30:31.887-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T12:30:31.887-07:00",
"id": "9rs45v11gj7l",
"incident_id": "xxsxtjhxk86l",
"status": "monitoring",
"tweet_id": 2108278891370656000,
"twitter_updated_at": "2026-10-08T12:30:32.483-07:00",
"updated_at": "2026-10-08T12:30:32.487-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "We've identified the cause of the ingestion delay as an upstream issue with our cloud storage provider, Google Cloud, impacting a subset of US projects. Google has confirmed this as a high-priority issue on their end but hasn't provided an ETA yet. We've deployed mitigations to ensure a smooth recovery once it clears.\n\nNo events are being lost. This is strictly an ingestion delay, all affected data will continue to be reprocessed once the issue is resolved.",
"created_at": "2026-10-08T11:31:12.023-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T11:31:12.023-07:00",
"id": "zlnb1rs0mwwz",
"incident_id": "xxsxtjhxk86l",
"status": "identified",
"tweet_id": 2108263962559365600,
"twitter_updated_at": "2026-10-08T11:31:13.212-07:00",
"updated_at": "2026-10-08T11:31:13.215-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-08T10:32:23.876-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T10:32:23.876-07:00",
"id": "7r85jl6d0tjs",
"incident_id": "xxsxtjhxk86l",
"status": "identified",
"tweet_id": 2108249161858781700,
"twitter_updated_at": "2026-10-08T10:32:24.362-07:00",
"updated_at": "2026-10-08T10:32:24.365-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "We are currently experiencing delays in our data ingestion pipeline, which is affecting real-time data for projects enrolled in the US projects, resulting in delays for a subset of projects. While no data is being lost, our engineering team is actively investigating the matter and working to restore real-time functionality as quickly as possible. We appreciate your patience during this time.\n\nIf you have any questions, please \u003ca href=\"https://mixpanel.com/get-support\"\u003econtact support.\u003c/a\u003e",
"created_at": "2026-10-08T09:32:31.548-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-08T09:32:31.548-07:00",
"id": "9twfd857zvyl",
"incident_id": "xxsxtjhxk86l",
"status": "investigating",
"tweet_id": 2108234094450786800,
"twitter_updated_at": "2026-10-08T09:32:32.026-07:00",
"updated_at": "2026-10-08T09:32:32.028-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-10-08T12:30:31.887-07:00",
"name": "Temporary Data Ingestion Delay for US Projects",
"page_id": "x4m91ldrf511",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-08T17:22:41.067-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/6t6yl9q9vzjv",
"started_at": "2026-10-08T09:32:31.494-07:00",
"status": "resolved",
"updated_at": "2026-10-08T17:22:41.656-07:00"
},
{
"components": [
{
"created_at": "2016-03-22T13:22:02.024-07:00",
"description": "User interface and query API at mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "45yb5qck0sw5",
"name": "Application Availability (US)",
"page_id": "x4m91ldrf511",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-23T20:46:43.713-07:00"
}
],
"created_at": "2026-08-26T14:39:13.441-07:00",
"id": "t0fcrbxrg8jk",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-08-26T15:45:49.876-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-26T15:45:49.876-07:00",
"id": "x79pbj680j7h",
"incident_id": "t0fcrbxrg8jk",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-26T15:45:49.876-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "A fix has been implemented, and we're seeing improved success rates for the Query API. We will continue monitoring to ensure stability.",
"created_at": "2026-08-26T15:28:00.739-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-26T15:28:00.739-07:00",
"id": "dgchrptk9mfh",
"incident_id": "t0fcrbxrg8jk",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-26T15:28:00.739-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-08-26T15:11:48.765-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-26T15:11:48.765-07:00",
"id": "n698n6bd2qj4",
"incident_id": "t0fcrbxrg8jk",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-26T15:11:48.765-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "Mixpanel is experiencing degraded Query API performance, resulting in HTTP 500 responses when querying reports for projects with US Data Residency. Projects with EU and IN Data Residency remain unaffected. \nWe appreciate your patience while our engineers work to restore normal functionality. We will post progress updates on our status page. \nIf you have any questions, please \u003ca href=\"https://mixpanel.com/get-support\"\u003econtact support.\u003c/a\u003e",
"created_at": "2026-08-26T14:39:13.540-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-26T14:39:13.540-07:00",
"id": "v5rczkh1ckdh",
"incident_id": "t0fcrbxrg8jk",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-26T14:39:13.540-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-08-26T15:28:00.739-07:00",
"name": "Degraded Query API Performance impacting US Projects",
"page_id": "x4m91ldrf511",
"postmortem_body": "# Mixpanel RCA: Temporary Query Service Disruption for US Projects August 26, 2026\n\n# Summary\n\nBetween approximately **2:02 PM and 3:19 PM PT on August 26, 2026**, projects in the US region experienced failures loading reports and running queries through the Mixpanel UI and Query API. During this window, some queries in the region failed or returned errors. **No customer data was lost, and data ingestion was not affected.** All events continued to be collected and stored normally throughout the incident; once query service was restored, all reports reflected complete, accurate data with no customer action required.\n\nThe cause was identified as a recently deployed internal tool for diagnostic query replay, a capability our engineers use to re-run copies of past queries to debug performance, which unexpectedly wrote large amounts of data to the disks of our query servers, consuming storage capacity the servers need to operate. When those disks filled, the affected servers took themselves out of service. Service was restored, the impacted servers were brought back online, and the internal tooling was disabled. The remediations below add storage safeguards to remove internal tooling's ability to consume resources on query servers, and are designed to prevent this class of failure from happening again in the future.\n\n# What happened\n\nMixpanel's query engine runs on a fleet of servers that each use a set of local storage volumes to cache the data needed to answer queries quickly. Separately, our engineers use diagnostic query replay tooling, a capability our engineers use to reproduce and debug query performance.\n\nWhile applying the diagnostic query replay tooling to a large, complex query, a software bug caused the query replay captures to fan out across the entire fleet instead of remaining confined to a single server. Additionally, it caused far more data than intended to be saved without timely eviction on a single server. Two factors then widened the impact of the issue:\n\n* A single full storage volume took a server fully out of service. Each server treats its cache as unhealthy if any one of its storage volumes crosses a usage threshold, even when all other volumes are healthy. The replay data was written to one specific volume on every server, so servers across the region failed their health checks nearly simultaneously.\n* Cleanup limits did not account for data size. The safeguard limiting replay data on disk counted items at the application level rather than bytes at the filesystem level, so a small number of unexpectedly large captures passed the check while consuming most of the volume’s capacity.\n\nTogether, these allowed a single debugging workflow that normally has a negligible footprint to interrupt production query serving across the US region.\n\n# Timeline \\(Pacific Time, August 26, 2026\\)\n\n* **2:00 PM** — First oversized diagnostic replay capture was written; storage volumes began to reach their capacity and query success rate began to drop shortly after.\n* **2:17 PM** — Automated alerting paged the on-call engineer; investigation began immediately and additional engineers were engaged.\n* **2:39 PM** — Status page incident posted; in-app banner displayed at 2:40 PM.\n* **2:53 PM** — Root cause identified; recovery efforts began on the first affected server group.\n* **3:19 PM** — Query service restored for the vast majority of traffic; the final server group fully recovered at 3:27 PM.\n* **3:45 PM** — Incident resolved after a stable observation period. The diagnostic replay tooling that triggered the issue was fully disabled the same evening.\n\n# Root cause\n\n1. **A software bug in our query replay tooling caused unbounded writes to production storage.** A recently deployed capability for reproducing queries mishandled a particular class of complex query, causing captures to spread to every query server in the region and to write far more data to the affected volumes than the design assumed.\n2. **Replay files consumed disk capacity that query serving depends on.** The replay tooling wrote its files to the query servers' local disks, so runaway replay data exhausted storage capacity the servers need to answer queries.\n3. **Safeguards only partially accounted for the behavior.** The cleanup policy for diagnostic replay data limited the number of items on disk, but not their total size, so it did not engage. Alerting on storage volumes flagged the growth but was not escalated as critical on a per-server basis, which delayed detection until query failures began.\n\n# What we're changing\n\nThe end state we are building toward: internal diagnostic query replay data stored in dedicated object storage, placing no load on production query servers.\n\nAlready deployed:\n\n* **Disabled the internal tooling** that caused the incident, and remediated the underlying issue so diagnostic captures are confined to a single server and the specific query class is handled correctly.\n* **Documented the targeted recovery procedure** used during the incident, i.e., clearing only the affected storage volume rather than restarting full server groups, in our operational runbooks, shortening recovery time if any volume’s capacity runs out in the future.\n\nIn progress:\n\n* **Filesystem-level, size-based limits on diagnostic replay data**, capping total bytes on disk rather than item counts, so oversized captures are rejected or evicted before they can affect the volume’s capacity.\n* **Stricter storage alerting**, escalating per-server volume saturation as critical before it can affect query health checks.\n* **Moving query replay data to dedicated object storage,** so it consumes no resources on production query servers.\n\n# Common questions\n\n* **Was any data lost?** No. Data ingestion was unaffected throughout the incident: events continued to be collected, queued, and stored normally. Only the ability to query was interrupted. Once service was restored, all reports reflected complete data.\n* **Were saved reports, dashboards, or project settings affected?** No. The incident affected query execution only. Nothing stored in your project changed.\n* **Why did it affect multiple US projects at once?** The oversized diagnostic replay data written to every query server's disks at nearly the same time, and each server removes itself from service when any one volume fills. Preventing internal tooling from consuming query-server storage is a core part of our remediation work.\n* **How is this prevented going forward?** The tooling issue is resolved and the tooling remains disabled until size-based limits are in place. Storage alerting is being tightened so saturation is caught before it affects query serving. Structurally, we are moving diagnostic data off production query servers entirely, so internal debugging data will consume no storage on the servers responsible for queries processing.\n\nWe apologize for the disruption and for the time reports were unavailable. Please reach out through your account team or support with any questions.",
"postmortem_body_last_updated_at": "2026-09-08T11:28:15.731-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-08T11:28:15.731-07:00",
"reminder_intervals": null,
"resolved_at": "2026-08-26T15:45:49.876-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/p7k1qngjlzn1",
"started_at": "2026-08-26T14:39:13.427-07:00",
"status": "resolved",
"updated_at": "2026-09-08T11:28:15.747-07:00"
},
{
"components": [
{
"created_at": "2016-03-22T13:22:02.024-07:00",
"description": "User interface and query API at mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "45yb5qck0sw5",
"name": "Application Availability (US)",
"page_id": "x4m91ldrf511",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-23T20:46:43.713-07:00"
},
{
"created_at": "2025-04-02T13:06:27.643-07:00",
"description": "User interface and query API at eu.mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "hsv3glmmrs22",
"name": "Application Availability (EU)",
"page_id": "x4m91ldrf511",
"position": 2,
"showcase": true,
"start_date": "2025-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-08-19T20:05:17.036-07:00"
},
{
"created_at": "2025-04-02T13:06:46.569-07:00",
"description": "User interface and query API at in.mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "wp2dc3v8nykv",
"name": "Application Availability (IN)",
"page_id": "x4m91ldrf511",
"position": 3,
"showcase": true,
"start_date": "2025-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-08-19T20:05:17.050-07:00"
}
],
"created_at": "2026-08-19T16:20:44.618-07:00",
"id": "ldsr5xfq69j5",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-08-19T20:05:17.077-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-19T20:05:17.077-07:00",
"id": "nn944lgf6qfm",
"incident_id": "ldsr5xfq69j5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-19T20:05:17.077-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-08-19T19:52:31.247-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-19T19:52:31.247-07:00",
"id": "4pybh19p3vhl",
"incident_id": "ldsr5xfq69j5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-19T19:52:31.247-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-08-19T18:04:53.316-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-19T18:04:53.316-07:00",
"id": "5qgvh9tl4njw",
"incident_id": "ldsr5xfq69j5",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-19T18:04:53.316-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "partial_outage",
"old_status": "degraded_performance"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "partial_outage",
"old_status": "degraded_performance"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "partial_outage",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-08-19T16:23:08.693-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-19T16:23:08.693-07:00",
"id": "9dsg63b476x2",
"incident_id": "ldsr5xfq69j5",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-19T16:23:08.693-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently experiencing issues with populating event properties in dropdown menus. We appreciate your patience while our engineers work to restore functionality. If you have any questions, please contact support@mixpanel.com",
"created_at": "2026-08-19T16:20:44.754-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-19T16:20:44.754-07:00",
"id": "ptjbskx7jfhr",
"incident_id": "ldsr5xfq69j5",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-19T16:20:44.754-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-08-19T19:52:31.247-07:00",
"name": "Issues Populating Event Properties in dropdown menus",
"page_id": "x4m91ldrf511",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-08-19T20:05:17.077-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/f5cgy867j1tj",
"started_at": "2026-08-19T16:20:44.603-07:00",
"status": "resolved",
"updated_at": "2026-08-19T20:05:17.093-07:00"
},
{
"components": [
{
"created_at": "2016-03-22T13:22:02.024-07:00",
"description": "User interface and query API at mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "45yb5qck0sw5",
"name": "Application Availability (US)",
"page_id": "x4m91ldrf511",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-23T20:46:43.713-07:00"
},
{
"created_at": "2025-04-02T13:06:27.643-07:00",
"description": "User interface and query API at eu.mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "hsv3glmmrs22",
"name": "Application Availability (EU)",
"page_id": "x4m91ldrf511",
"position": 2,
"showcase": true,
"start_date": "2025-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-08-19T20:05:17.036-07:00"
},
{
"created_at": "2025-04-02T13:06:46.569-07:00",
"description": "User interface and query API at in.mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "wp2dc3v8nykv",
"name": "Application Availability (IN)",
"page_id": "x4m91ldrf511",
"position": 3,
"showcase": true,
"start_date": "2025-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-08-19T20:05:17.050-07:00"
}
],
"created_at": "2026-08-06T00:19:23.956-07:00",
"id": "2l5pvqg50g09",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-08-06T06:32:35.151-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-08-06T06:32:35.151-07:00",
"id": "9xfp9jtbjbn5",
"incident_id": "2l5pvqg50g09",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-06T06:32:35.151-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been deployed and we're now monitoring the results. Report queries in the Mixpanel MCP server should be functioning normally. We'll continue to watch closely and provide a final update once we've confirmed full resolution.",
"created_at": "2026-08-06T02:12:46.443-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-08-06T02:12:46.443-07:00",
"id": "1z8rwr0vpz1l",
"incident_id": "2l5pvqg50g09",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-06T02:12:46.443-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and fix is being implemented.",
"created_at": "2026-08-06T01:39:20.483-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-08-06T01:39:20.483-07:00",
"id": "47tybgwl5dgw",
"incident_id": "2l5pvqg50g09",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-06T01:39:20.483-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are aware that the Get-Report tool in the Mixpanel MCP server is currently failing when called with skip_results: false. Report metadata is unaffected. We are investigating the issue and will provide updates as we have them.",
"created_at": "2026-08-06T00:19:24.092-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-08-06T00:19:24.092-07:00",
"id": "4274mtjh8jl0",
"incident_id": "2l5pvqg50g09",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-08-06T00:19:24.092-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-08-06T02:12:46.443-07:00",
"name": "Mixpanel MCP: Get-Report degraded",
"page_id": "x4m91ldrf511",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-08-06T06:32:35.151-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/cqmxbxgp67r6",
"started_at": "2026-08-06T00:19:23.942-07:00",
"status": "resolved",
"updated_at": "2026-08-06T06:32:35.174-07:00"
},
{
"components": [],
"created_at": "2026-07-22T01:03:32.780-07:00",
"id": "0tpz1vpqb07b",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2026-07-22T07:47:22.947-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-22T07:47:22.947-07:00",
"id": "hh54r2s6pmbc",
"incident_id": "0tpz1vpqb07b",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-22T07:47:22.947-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-07-22T03:55:08.130-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-22T03:55:08.130-07:00",
"id": "q5fp1d9ll2dp",
"incident_id": "0tpz1vpqb07b",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-22T03:55:08.130-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "the team continues to investigate the issue and next steps. Thanks for your patience.",
"created_at": "2026-07-22T02:52:45.810-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-22T02:52:45.810-07:00",
"id": "bmpspghpnyyl",
"incident_id": "0tpz1vpqb07b",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-22T02:52:45.810-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are currently experiencing issues with the Mixpanel agent within the webapp. Our engineering team has been alerted and is investigating the issue and working to restore its functionality. We appreciate your patience as we work to resolve this matter.\n\nIf you have any questions, please \u003ca href=\"https://mixpanel.com/get-support\"\u003econtact support.\u003c/a\u003e",
"created_at": "2026-07-22T01:03:32.823-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-22T01:03:32.823-07:00",
"id": "sbd14nggnq8b",
"incident_id": "0tpz1vpqb07b",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-22T01:03:32.823-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-22T03:55:08.130-07:00",
"name": "Issues serving responses by the Mixpanel agent",
"page_id": "x4m91ldrf511",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-07-22T07:47:22.947-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/7y5gqnzctpr0",
"started_at": "2026-07-22T01:03:32.774-07:00",
"status": "resolved",
"updated_at": "2026-07-22T07:47:22.966-07:00"
},
{
"components": [
{
"created_at": "2025-04-02T13:08:23.715-07:00",
"description": "/track, /import, and /engage on api.mixpanel.com",
"group_id": "jwswcg5s8vp8",
"id": "cq5ttq9czyjh",
"name": "Ingestion API Availability (US)",
"page_id": "x4m91ldrf511",
"position": 1,
"showcase": true,
"start_date": "2025-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-07-11T07:25:18.472-07:00"
}
],
"created_at": "2026-07-11T00:53:32.250-07:00",
"id": "b1h9cbypnch5",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "cq5ttq9czyjh",
"name": "Data Ingestion - Ingestion API Availability (US)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-07-11T08:54:09.168-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-11T08:54:09.168-07:00",
"id": "6t53t064rdwl",
"incident_id": "b1h9cbypnch5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-11T08:54:09.168-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cq5ttq9czyjh",
"name": "Data Ingestion - Ingestion API Availability (US)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-07-11T07:25:18.510-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-11T07:25:18.510-07:00",
"id": "h98tjsc8ns3x",
"incident_id": "b1h9cbypnch5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-11T07:25:18.510-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cq5ttq9czyjh",
"name": "Data Ingestion - Ingestion API Availability (US)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-07-11T05:54:04.216-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-11T05:54:04.216-07:00",
"id": "6k47hqk03zct",
"incident_id": "b1h9cbypnch5",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-11T05:54:04.216-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cq5ttq9czyjh",
"name": "Data Ingestion - Ingestion API Availability (US)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-07-11T03:15:58.139-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-11T03:15:58.139-07:00",
"id": "s3ljkggfzf5t",
"incident_id": "b1h9cbypnch5",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-11T03:15:58.139-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cq5ttq9czyjh",
"name": "Data Ingestion - Ingestion API Availability (US)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-07-11T02:17:58.170-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-11T02:17:58.170-07:00",
"id": "6vgt9svd4vvl",
"incident_id": "b1h9cbypnch5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-11T02:17:58.170-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cq5ttq9czyjh",
"name": "Data Ingestion - Ingestion API Availability (US)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently experiencing delays in our data ingestion pipeline, which is affecting real-time data for projects enrolled in the US projects, resulting in delays for a subset of projects. While no data is being lost, our engineering team is actively investigating the matter and working to restore real-time functionality as quickly as possible. We appreciate your patience during this time.\n\nIf you have any questions, please \u003ca href=\"https://mixpanel.com/get-support\"\u003econtact support.\u003c/a\u003e",
"created_at": "2026-07-11T00:53:32.336-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-11T00:53:32.336-07:00",
"id": "k00vt15xjcnm",
"incident_id": "b1h9cbypnch5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-11T00:53:32.336-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-11T07:25:18.510-07:00",
"name": "Temporary Data Ingestion Delay for US Projects",
"page_id": "x4m91ldrf511",
"postmortem_body": "# Summary\n\nBetween approximately **11:35 PM PT on July 10 and 7:19 AM PT on July 11, 2026**, data ingestion for Mixpanel projects in the US region ran behind by up to ~2 hours. During this window, reports and dashboards temporarily showed incomplete data — recent time ranges could appear as sharp, artificial drops in metrics such as active users or revenue. **No data was lost.** All events were queued durably and processed in full once the backlog cleared; metrics returned to accurate values on their own, with no customer action required.\n\nThe cause originated in our ingestion controls: for a small number of very high-volume projects, the ingestion rates we allowed had drifted out of alignment with the capacity provisioned for those projects. Large historical imports — entirely legitimate use of the platform — were therefore admitted faster than their infrastructure could absorb, and two properties of our pipeline turned that localized overload into a region-wide delay. The fixes below realign those controls so that any import, of any size, is automatically kept within safe bounds.\n\n# What happened\n\nMixpanel's ingestion pipeline is sharded and multi-tenant: each project's data is distributed across a set of partitions sized for its expected volume, and for throughput, events from many customers are processed together in batches. This design delivers high efficiency, but it depends on one invariant: the rate at which we admit a project's traffic must match the capacity provisioned for it. When that invariant holds, even very large imports are absorbed smoothly.\n\nHere, it did not hold. A small number of very large historical data imports ran on projects whose allowed ingestion rates had, over time, grown well beyond their provisioned partition capacity. The excess volume concentrated onto specific partitions as **hot spots**, saturating the portion of the streaming fleet serving them. Two factors then widened the impact:\n\n* **Batched, multi-tenant processing amplified the hot spots.** Because events from many customers travel together in batches, slowness and failures on the overloaded partitions delayed unrelated customers' events sharing those batches.\n* **Automatic scale-up was ineffective.** Adding capacity cannot dissolve a hot spot of this kind, because the overloaded partitions stay pinned to the same infrastructure, and a capacity limit in a component of our streaming infrastructure prevented the added capacity from taking effect.\n\nTogether, these turned what should have been a brief, self-healing slowdown into a multi-hour delay requiring manual intervention.\n\n# Timeline \\(Pacific Time, July 10–11\\)\n\n* **11:35 PM** — Automated alerting detected the ingestion backlog; on-call engineer engaged immediately.\n* **12:49 AM** — Status page incident posted; impact scoped to the US region only.\n* **1:23 AM** — The largest contributing import was paused.\n* **1:55–6:48 AM** — Progressive mitigations: additional traffic sources throttled, certain backlogged data deferred with the owning customer's agreement, failure isolation enabled in the pipeline, and affected infrastructure nodes replaced.\n* **7:19 AM** — Backlog fully processed; all projects current. Status page moved to monitoring, then resolved after a stable observation period.\n\n# Root cause\n\n1. **Ingestion rate limits misaligned with provisioned capacity.** For the driving projects, the rates our platform admitted had grown out of step with the infrastructure provisioned for them, so legitimate high-volume imports were let in faster than their partitions could absorb. This is the systemic root cause. The imports themselves were a legitimate use of the platform.\n2. **Batched, multi-tenant processing amplified the overload.** Failures on the overloaded partitions delayed unrelated customers' events sharing the same processing batches, spreading a localized problem across the platform.\n3. **Traffic to the overloaded partitions could not be redistributed.** Partition-to-server assignment in the streaming layer was not load-aware, so the hot spots stayed pinned to the same servers regardless of fleet size — total capacity was sufficient, but it could not be brought to bear. A scaling limit in a component of our streaming infrastructure compounded this by preventing scale-up from adding capacity, which delayed diagnosis until engineers intervened manually.\n\n# What we're changing\n\nThe end state we are building toward: **every project's ingestion limits automatically match its provisioned capacity, so that imports of any size, including full historical backfills and data warehouse syncs, can run without advance coordination, and one project's volume is prevented from affecting another's data freshness.**.\n\nAlready deployed — improving our ingestion handling:\n\n* **Improved hot-spot handling.** Traffic distribution in the streaming layer is now load-aware, spreading concentrated load more evenly across the fleet, and individual problem items are now retried separately instead of holding up the rest of their batch — reducing, though not eliminating, the impact a localized overload can have on unrelated traffic.\n* **Re-provisioned the highest-volume workloads** onto appropriately sized infrastructure, prioritized by risk.\n\nAlready deployed — changing how we operate the third-party streaming service:\n\n* **Audited and re-tuned the fleet's capacity profile** so individual servers have substantially more headroom for concentrated load, and worked with the provider to resolve the scaling limitation encountered during the incident.\n\nIn progress:\n\n* **Capacity-aware rate limiting** — closing the gaps between individually granted project rate limits and each project's real provisioned capacity, extending rate limiting to ingestion paths that previously lacked it, and coupling any future limit increase to a capacity increase, which is designed to prevent this class of misalignment from recurring..\n* **Finer-grained volume monitoring and alerting** so capacity misalignment is detected and corrected before it can affect any customer.\n* Evaluating stronger workload isolation/backlog recovery prioritization for bulk/backfill traffic paths, so historical imports have reduced impact on live traffic\n\n# Common questions\n\n* **Was any data lost?** No. Events were durably queued throughout the incident and were fully processed once the backlog cleared. Any metric drops seen during the window were a display artifact of the delay and self-corrected.\n* **Do I need to coordinate large imports or backfills with Mixpanel?** No — Our goal is for the platform to keep any import within safe rates automatically, so that backfills and warehouse syncs can run without scheduling or notice. We are still rolling out the capacity-aware controls that deliver this. In the meantime, if you are planning an unusually large import or backfill, we recommend coordinating with your account team so we can confirm capacity in advance.\n* **How is this prevented going forward?** The systemic fix is tightening the gaps between individually granted project rate limits and each project's real provisioned capacity, and extending rate limiting to the ingestion paths that previously lacked it — so overload of this kind is stopped at admission. In addition, the scaling limitation that prolonged the incident is fixed, the pipeline now retries individual problem items separately so a localized overload has far less impact on unrelated traffic, and bulk traffic is being further isolated from live traffic.\n* **Can individual projects be prioritized during recovery?** This capability did not exist during the incident — all projects recovered at the same rate. We are evaluating prioritization mechanisms for backlog recovery as part of our follow-up work.\n\nWe apologize for the disruption and for the concern the temporarily depressed metrics caused. Please reach out through your account team or support with any questions.",
"postmortem_body_last_updated_at": "2026-07-17T05:11:37.328-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-07-16T16:48:27.355-07:00",
"reminder_intervals": null,
"resolved_at": "2026-07-11T08:54:09.168-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/hdvh21fgtdzd",
"started_at": "2026-07-11T00:53:32.239-07:00",
"status": "resolved",
"updated_at": "2026-07-17T05:11:37.332-07:00"
},
{
"components": [
{
"created_at": "2016-03-22T13:22:02.024-07:00",
"description": "User interface and query API at mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "45yb5qck0sw5",
"name": "Application Availability (US)",
"page_id": "x4m91ldrf511",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-23T20:46:43.713-07:00"
},
{
"created_at": "2025-04-02T13:06:27.643-07:00",
"description": "User interface and query API at eu.mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "hsv3glmmrs22",
"name": "Application Availability (EU)",
"page_id": "x4m91ldrf511",
"position": 2,
"showcase": true,
"start_date": "2025-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-08-19T20:05:17.036-07:00"
},
{
"created_at": "2025-04-02T13:06:46.569-07:00",
"description": "User interface and query API at in.mixpanel.com",
"group_id": "v5rct9xjfslj",
"id": "wp2dc3v8nykv",
"name": "Application Availability (IN)",
"page_id": "x4m91ldrf511",
"position": 3,
"showcase": true,
"start_date": "2025-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-08-19T20:05:17.050-07:00"
}
],
"created_at": "2026-06-29T12:47:13.055-07:00",
"id": "6125yr0c9s4f",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-06-29T20:30:14.022-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-29T20:30:14.022-07:00",
"id": "15th8q58b7d8",
"incident_id": "6125yr0c9s4f",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-29T20:30:14.022-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-06-29T19:15:26.487-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-29T19:15:26.487-07:00",
"id": "z8fd6rpvxn17",
"incident_id": "6125yr0c9s4f",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-29T19:15:26.487-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Query latency has stabilized, though our team continues to investigate the underlying cause of the degraded performance. We will continue to post updates as our investigation progresses. Thank you for your patience.",
"created_at": "2026-06-29T15:42:22.660-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-29T15:42:22.660-07:00",
"id": "wnj7x4lc3jbw",
"incident_id": "6125yr0c9s4f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-29T15:42:22.660-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-06-29T14:04:43.762-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-29T14:04:43.762-07:00",
"id": "nfc0735x261s",
"incident_id": "6125yr0c9s4f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-29T14:04:43.762-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "45yb5qck0sw5",
"name": "Application Availability - Application Availability (US)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "hsv3glmmrs22",
"name": "Application Availability - Application Availability (EU)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "wp2dc3v8nykv",
"name": "Application Availability - Application Availability (IN)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Mixpanel is experiencing degraded performance with our Query API, including increased latency. You may see slow-loading reports or query errors. We appreciate your patience while our engineers work to restore normal functionality. We will post progress updates on our status page. \nIf you have any questions, please \u003ca href=\"https://mixpanel.com/get-support\"\u003econtact support.\u003c/a\u003e",
"created_at": "2026-06-29T12:47:13.181-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-06-29T12:47:13.181-07:00",
"id": "0vwfcd36qlwj",
"incident_id": "6125yr0c9s4f",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-29T12:47:13.181-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2026-06-29T19:15:26.487-07:00",
"name": "Query API Degraded Performance",
"page_id": "x4m91ldrf511",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-06-29T20:30:14.022-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/whg5ydkh3b8m",
"started_at": "2026-06-29T12:47:13.041-07:00",
"status": "resolved",
"updated_at": "2026-06-29T20:30:14.038-07:00"
},
{
"components": [],
"created_at": "2026-06-03T13:05:03.464-07:00",
"id": "gdj9jdwxkx2x",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2026-06-03T14:27:08.222-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-03T14:27:08.222-07:00",
"id": "svzhfm9ckpjl",
"incident_id": "gdj9jdwxkx2x",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-03T14:27:08.222-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-06-03T13:17:47.276-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-03T13:17:47.276-07:00",
"id": "gwy62p1mnmwj",
"incident_id": "gdj9jdwxkx2x",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-03T13:17:47.276-07:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Mixpanel is currently experiencing a disruption in the ability to invite and delete internal users within organizations. We appreciate your patience while our engineers work to restore this functionality. If you have any questions, please contact support@mixpanel.com",
"created_at": "2026-06-03T13:05:03.509-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-03T13:05:03.509-07:00",
"id": "mf45qwl98dts",
"incident_id": "gdj9jdwxkx2x",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-03T13:05:03.509-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Issue with Inviting and Deleting Users",
"page_id": "x4m91ldrf511",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-06-03T14:27:08.222-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/29vwsbz39l6q",
"started_at": "2026-06-03T13:05:03.457-07:00",
"status": "resolved",
"updated_at": "2026-06-03T14:27:08.239-07:00"
}
],
"page": {
"id": "x4m91ldrf511",
"name": "Mixpanel",
"url": "https://www.mixpanelstatus.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}