Snapshot 41332
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2017-05-03T15:03:22.513-07:00",
"description": "API that serves all SDKs",
"group_id": null,
"id": "9gjkrg48y7w4",
"name": "SDK API",
"page_id": "bn51dbtm6djn",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2024-05-15T15:00:28.355-07:00"
},
{
"created_at": "2017-05-03T15:04:02.648-07:00",
"description": "Public and web console API",
"group_id": null,
"id": "s469h0b1pjcs",
"name": "API",
"page_id": "bn51dbtm6djn",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.537-08:00"
},
{
"created_at": "2017-05-03T15:03:54.067-07:00",
"description": null,
"group_id": null,
"id": "pdb48tnrnlz6",
"name": "Web Console",
"page_id": "bn51dbtm6djn",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.553-08:00"
},
{
"created_at": "2017-05-03T14:22:30.812-07:00",
"description": null,
"group_id": null,
"id": "shdlrt53dbxw",
"name": "Data Processing",
"page_id": "bn51dbtm6djn",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-05-03T15:28:35.844-07:00"
},
{
"created_at": "2017-05-03T15:59:31.085-07:00",
"description": null,
"group_id": null,
"id": "kf9hkr38yhc2",
"name": "Integrations",
"page_id": "bn51dbtm6djn",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-29T21:48:39.705-07:00"
},
{
"created_at": "2017-05-03T15:57:02.634-07:00",
"description": null,
"group_id": null,
"id": "x111lx8js3lc",
"name": "CDN",
"page_id": "bn51dbtm6djn",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2021-06-08T07:36:09.970-07:00"
},
{
"created_at": "2017-05-03T14:22:30.792-07:00",
"description": null,
"group_id": null,
"id": "xp7yq7jlqpz9",
"name": "Corporate Site",
"page_id": "bn51dbtm6djn",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2023-10-18T14:04:34.467-07:00"
},
{
"created_at": "2017-05-04T09:55:16.830-07:00",
"description": null,
"group_id": null,
"id": "3mqr5p0pl4q1",
"name": "Documentation",
"page_id": "bn51dbtm6djn",
"position": 8,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2022-08-03T12:41:40.027-07:00"
},
{
"created_at": "2020-05-04T11:17:58.468-07:00",
"description": null,
"group_id": null,
"id": "0n6zlvhp2q8s",
"name": "Streaming Authentication Service",
"page_id": "bn51dbtm6djn",
"position": 9,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T12:53:15.323-07:00"
}
],
"incidents": [
{
"components": [
{
"created_at": "2017-05-03T15:04:02.648-07:00",
"description": "Public and web console API",
"group_id": null,
"id": "s469h0b1pjcs",
"name": "API",
"page_id": "bn51dbtm6djn",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.537-08:00"
},
{
"created_at": "2017-05-03T15:03:54.067-07:00",
"description": null,
"group_id": null,
"id": "pdb48tnrnlz6",
"name": "Web Console",
"page_id": "bn51dbtm6djn",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.553-08:00"
}
],
"created_at": "2025-12-15T14:02:00.891-08:00",
"id": "prfh94snr5w8",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We monitored the system for 16 hours and it remains stable with no errors observed. We will provide a detailed RCA once we receive the analysis from our vendor",
"created_at": "2025-12-16T08:22:33.581-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T08:22:33.581-08:00",
"id": "vcd5dj8d817t",
"incident_id": "prfh94snr5w8",
"status": "resolved",
"tweet_id": 2000964777724797400,
"twitter_updated_at": "2025-12-16T08:22:33.994-08:00",
"updated_at": "2025-12-16T08:22:33.996-08:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to monitor for any further issues.",
"created_at": "2025-12-15T15:04:24.142-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T15:04:24.142-08:00",
"id": "7qslr22kv4qj",
"incident_id": "prfh94snr5w8",
"status": "monitoring",
"tweet_id": 2000703516613239000,
"twitter_updated_at": "2025-12-15T15:04:24.414-08:00",
"updated_at": "2025-12-15T15:04:24.416-08:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "we were facing an intermittent latency issue impacting some of our flows. The system is now stable, and we’re investigating the root cause with our cloud provider",
"created_at": "2025-12-15T14:31:07.177-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T14:31:07.000-08:00",
"id": "lj7k2xfk0lpl",
"incident_id": "prfh94snr5w8",
"status": "monitoring",
"tweet_id": 2000695140915318800,
"twitter_updated_at": "2025-12-15T14:31:07.492-08:00",
"updated_at": "2025-12-15T15:03:13.716-08:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2025-12-15T14:10:09.531-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T14:10:09.531-08:00",
"id": "8zkqqwjtsmss",
"incident_id": "prfh94snr5w8",
"status": "investigating",
"tweet_id": 2000689865860768000,
"twitter_updated_at": "2025-12-15T14:10:09.824-08:00",
"updated_at": "2025-12-15T14:10:09.826-08:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "502 error",
"created_at": "2025-12-15T14:02:00.974-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T14:02:00.974-08:00",
"id": "3sjt6j8xz7ds",
"incident_id": "prfh94snr5w8",
"status": "investigating",
"tweet_id": 2000687816871960600,
"twitter_updated_at": "2025-12-15T14:02:01.307-08:00",
"updated_at": "2025-12-15T14:02:01.309-08:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2025-12-15T14:31:07.000-08:00",
"name": "Intermittent login flow errors on FME",
"page_id": "bn51dbtm6djn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-12-16T08:22:33.581-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/7zvd5yndd4l0",
"started_at": "2025-12-15T14:02:00.886-08:00",
"status": "resolved",
"updated_at": "2025-12-16T08:22:34.002-08:00"
},
{
"components": [],
"created_at": "2025-12-04T15:06:10.993-08:00",
"id": "4j5zh65hjpkj",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2025-12-04T15:06:11.035-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-04T21:30:00.000-08:00",
"id": "44p8s5qmhvpl",
"incident_id": "4j5zh65hjpkj",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-04T15:14:24.538-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Users cannot create experiments",
"page_id": "bn51dbtm6djn",
"postmortem_body": "# RCA - Users cannot create experiments\n\n## **Summary**\n\nBetween at least 04:50 PM UTC and 09:35 PM UTC on Thursday, Dec 4, 2025, FME customers with more than 50 Feature flags in their project may have experienced issues creating Experiments from the UI. No data loss or any other issues were encountered.\n\n## **Root Cause**\n\nAn oversight in our testing coverage revealed that the Create Experiment form would fail if the user attempted to select a feature flag that was not in the first 50 results. This was a state issue that caused the UI component to not locate the correct feature flag, causing the form to silently fail. After identifying the issue, we deployed a fix to fully resolve the issue.\n\n## **Impact**\n\n* Customers were possibly unable to create an Experiment if their project had more than 50 feature flags.\n* All customers using Experiments in projects which had fewer than 50 feature flags were not impacted.\n* Warehouse Native Experiments were not impacted by this issue.\n\n## **Timeline**\n\n| **Time \\(UTC\\)** | **Event** |\n| --- | --- |\n\n| **Time \\(UTC\\)** | **Event** |\n| --- | --- |\n| 04:50 PM Dec 4, 2025 | Customers reported issue with creating experiments. |\n| 09:35 PMNov 17, 2025 | Full restoration after fixing the UI component. |\n\n## **Remediation**\n\nA code fix was deployed to fix the issue.\n\n## **Action Items**\n\n1. **Fix the state issue in the component.** The UI component should not be relying on a separate query to retrieve feature flags. It should be using the selected feature flag’s data instead.\n2. **Improve testing coverage.** For this particular component \\(and for other components that rely on paginated data\\), we should have proper integration testing in place to ensure that pagination does not break interactions like this.\n3. **Additional metrics and alerting.** There were no logs or alerts on this issue. Debugging was very challenging and it wasn’t clear where the error was coming from. We should make sure that we have proper logging and alerts in place to catch these issues sooner.",
"postmortem_body_last_updated_at": "2025-12-04T15:13:19.732-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2025-12-04T15:13:19.732-08:00",
"reminder_intervals": null,
"resolved_at": "2025-12-04T21:30:00.000-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/mp4l2f34l04n",
"started_at": "2025-12-04T17:00:00.000-08:00",
"status": "resolved",
"updated_at": "2025-12-04T15:14:24.553-08:00"
},
{
"components": [],
"created_at": "2025-11-17T07:30:00.000-08:00",
"id": "5fnzh8jdjckd",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2025-11-17T15:33:10.850-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-11-17T22:30:00.000-08:00",
"id": "t81rv7bz17qg",
"incident_id": "5fnzh8jdjckd",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-11-17T15:40:20.590-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Intermittent Connectivity Issues",
"page_id": "bn51dbtm6djn",
"postmortem_body": "### Impact\n\nFrom 07:12 UTC, some customers experienced intermittent errors when accessing the UI, calling the Admin API, or sending events and impressions. Targeting and other SDK functionality remained fully operational. Depending on client retry and retention settings, some customers may have lost impressions or events due to repeated export failures.\n\n### Timeline\n\n* 07:12 UTC — Internal service-to-service communication began failing intermittently, causing customer-facing errors.\n* 07:48 UTC — We mitigated the issue by removing a problematic networking component from the service mesh, reducing the error rate.\n* 19:44 UTC — Full service restoration after safely reintroducing the networking component.\n\n### Root Cause \n\nA networking component within our service mesh malfunctioned, intermittently disrupting internal service communication responsible for UI access, API calls, and event/impression ingestion.\n\n**Mitigation** \nWe isolated and removed the faulty component from the mesh to stabilize traffic flow. After validating its behavior and functionality, we reinstated it during the evening maintenance window.\n\n**Next Steps** \nWe will publish a detailed RCA, including contributing factors and long-term prevention measures.\n\n\n\n# **RCA**\n\n## **Summary**\n\nBetween **07:12 AM UTC** and **07:44 PM UTC on Monday, Nov 17, 2025** , FME customers may have experienced intermittent errors when accessing the UI, calling the Admin API, or sending events and impressions from SDKs. Targeting and other SDK functionality remained fully operational. \n\nDepending on client retry and retention settings, some customers may have lost impressions or events due to repeated export failures.\n\n## **Root Cause**\n\n_A networking component within our service mesh was pushed invalid certificates, preventing communication between limited replicas of specific services. We were alerted to the elevated error rate, allowing us to isolate and remove the component to stabilize traffic flow which mitigated the the vast majority of errors. After validating the certificates, it was redeployed during the evening maintenance window to fully resolve the issue._\n\n## **Impact**\n\n* During the initial window from 07:12-07:48 AM, an elevated error rate prevented some events and impressions from reaching the platform and caused intermittent errors when accessing the UI or calling the Admin API\n* From 07:48 AM - 07:44 PM, the error rate recovered to baseline, however a small fraction of internal service calls faced minor connectivity issues leading to retries and a small number of timeouts that could cause an API call made by a user or the UI to fail.\n* Throughout, SDK feature flag operations remained unimpacted.\n\n## **Timeline**\n\n## **Remediation**\n\nThe component was removed to mitigate the issue promptly, while the underlying issue could be investigated, tested and resolved. The updated component was deployed in the next maintenance window.\n\n## **Action Items**\n\n1. **Permanently remove the problematic networking component.** This component is in the process of being phased out to simplify the platform and increase stability.\n2. **Additional metrics and alerting.** Although our existing alerts notified the initial issue, the low error rate after removing the component was flagged by our team monitoring the recovery.\n3. **Continue to build out network hardening.** Additional logic around connection handling and retries allowed certain services to remain resilient throughout the incident.",
"postmortem_body_last_updated_at": "2025-11-25T11:15:34.972-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2025-11-17T15:40:44.278-08:00",
"reminder_intervals": null,
"resolved_at": "2025-11-17T22:30:00.000-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8rzv20447s4b",
"started_at": "2025-11-17T22:30:00.000-08:00",
"status": "resolved",
"updated_at": "2025-11-25T11:15:34.976-08:00"
},
{
"components": [
{
"created_at": "2017-05-03T15:59:31.085-07:00",
"description": null,
"group_id": null,
"id": "kf9hkr38yhc2",
"name": "Integrations",
"page_id": "bn51dbtm6djn",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-29T21:48:39.705-07:00"
}
],
"created_at": "2025-10-29T11:49:02.000-07:00",
"id": "lkfrbq8lxxd9",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2025-10-29T21:48:39.739-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-29T21:48:39.739-07:00",
"id": "77vfswgfw2mw",
"incident_id": "lkfrbq8lxxd9",
"status": "resolved",
"tweet_id": 1983757922942513200,
"twitter_updated_at": "2025-10-29T21:48:40.052-07:00",
"updated_at": "2025-10-29T21:48:40.053-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The Azure incident has been resolved.\nFollowing the resolution, we’re seeing a significantly faster reduction in lag built up in delivering impression events via webhook.\nWe’re continuing to monitor system performance to ensure full recovery and stability.",
"created_at": "2025-10-29T19:15:22.897-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-29T19:15:22.897-07:00",
"id": "n6dvns70tx0d",
"incident_id": "lkfrbq8lxxd9",
"status": "monitoring",
"tweet_id": 1983719348612296700,
"twitter_updated_at": "2025-10-29T19:15:23.252-07:00",
"updated_at": "2025-10-29T19:15:23.253-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We continue to monitor the situation closely..\nWe’re seeing signs of improvement in delivery delays for customers relying on impression webhook events\nOur team will keep tracking progress and provide further updates as recovery continues.",
"created_at": "2025-10-29T15:25:23.289-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-29T15:25:23.289-07:00",
"id": "pfq9pfrkyd0c",
"incident_id": "lkfrbq8lxxd9",
"status": "monitoring",
"tweet_id": 1983661468827709400,
"twitter_updated_at": "2025-10-29T15:25:23.625-07:00",
"updated_at": "2025-10-29T15:25:23.626-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "There's no impact to core Harness FME SaaS capabilities.\n\nHowever, due to an ongoing Azure outage, customers who rely on impression webhook events may experience longer than usual delays in delivery.\n\nOur team is monitoring the situation closely and will continue to post updates here.",
"created_at": "2025-10-29T11:49:02.231-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-29T11:49:02.000-07:00",
"id": "8z7033zj54nr",
"incident_id": "lkfrbq8lxxd9",
"status": "identified",
"tweet_id": 1983607022450045400,
"twitter_updated_at": "2025-10-29T11:49:02.589-07:00",
"updated_at": "2025-10-29T12:43:18.126-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2025-10-29T15:25:23.289-07:00",
"name": "Delayed impressions webhook delivery",
"page_id": "bn51dbtm6djn",
"postmortem_body": "## **Summary**\n\nBetween 29 October 2025, 15:45 UTC and 30 October 2025, 03:45 UTC 2025, customers using the **Outgoing Webhook \\(Impressions\\)** integration experienced **delays in receiving impression data**. \nA small subset of customers hosted on **Microsoft Azure** would have stopped receiving impression events due to persistent failures on their side to receive these events. Still, those customers can export data from our platform.\n\nAll other services, including APIs and SDKs, remained fully operational.\n\n## **Root Cause**\n\nDuring the **Microsoft Azure outage**, customers hosted on Azure experienced persistent delivery failures for their impression events. These repeated retries resulted in **longer processing times** in our webhook delivery system, which in turn caused **delays for other customers** whose destinations were not impacted.\n\nWe initially chose to wait before stopping deliveries to the affected Azure customers. Once we confirmed the issue’s scope, we **temporarily paused deliveries** to the failing destinations, allowing delivery for all other customers to recover quickly.\n\nCustomers whose deliveries were paused can still **retrieve their impression events** for the impact window by exporting them directly from our platform. We also resumed sending events to all of our customers right after the Azure outage was mitigated.\n\n## **Timeline**\n\n| Time \\(UTC\\) | Event Description |\n| --- | --- |\n| **29/10/2025 15:45:00 UTC** | Azure goes down; internal monitoring alerts delays in delivering webhook events to customers. |\n| **29/10/2025 16:15:00 UTC** | On-call paged. |\n| **29/10/2025 16:55:00 UTC** | Root cause identified as an ongoing Azure outage. |\n| **29/10/2025 19:42:00 UTC** | Signs of recovery in Azure system; delay starts to decrease, and response times improve. |\n| **29/10/2025 22:40:00 UTC** | Blocked 100% of webhook traffic to one failing customer to speed recovery. |\n| **30/10/2025 01:15:00 UTC** | Azure incident declared fully remediated. |\n| **30/10/2025 03:45:00 UTC** | Resumed delivering webhook impression events to all customers; internal monitoring confirmed restoration of real-time delivery. Incident closed. |\n\n\n\n## **Remediation**\n\nWe initially waited for the affected Azure-hosted customers to recover as the Azure outage progressed. However, to maintain timely delivery for all other customers, we **temporarily paused event deliveries** to a small number of endpoints that continued to experience failures. Once Azure services stabilized, we **resumed normal delivery** to all customers before closing the incident.\n\n## **Action Items**\n\n* We are implementing automatic noisy-neighbor protections in the Impressions Webhook integration to isolate failing destinations and prevent cross-customer impact.\n* We are reviewing similar delivery patterns across our platform to identify and proactively mitigate any comparable risks.\n* We have updated our operational runbooks to enable faster, more decisive action when customer endpoints cause widespread delays, helping ensure continued reliability for all customers.",
"postmortem_body_last_updated_at": "2025-11-05T10:17:04.428-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2025-11-04T17:48:25.030-08:00",
"reminder_intervals": null,
"resolved_at": "2025-10-29T21:48:39.739-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8wkgjy5x4ywj",
"started_at": "2025-10-29T11:49:02.000-07:00",
"status": "resolved",
"updated_at": "2025-11-05T10:17:04.433-08:00"
},
{
"components": [],
"created_at": "2025-10-29T20:30:00.000-07:00",
"id": "wct15ylj5dry",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2025-11-05T10:30:31.735-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-10-29T20:30:00.000-07:00",
"id": "4gz16cgd6qr0",
"incident_id": "wct15ylj5dry",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-11-05T11:04:42.696-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "ChangeRequest API 500 Error",
"page_id": "bn51dbtm6djn",
"postmortem_body": "Summary \nBetween 08:41 PM UTC on Wednesday, October 29, 2025 and 03:27 AM UTC on Thursday, October 30, 2025, FME customers and integrations using the update change request endpoint from [api.split.io](http://api.split.io) with token-based authentication encountered “There was an error processing your request” responses with a 500 status when attempting to update existing change requests.\n\nAll other endpoints and functionality remained fully operational, including the ability to approve or reject change requests through the UI.\n\nRoot Cause \nA recent change introduced a regression that impacted change request updates through the API. \nThis caused the endpoint to return 500 errors for affected requests. Automated monitoring did not trigger alerts because the volume of reported errors was below our detection threshold. Once the issue was identified, remediation efforts were delayed due to an incident impacting third-party services using Azure. Throughout this incident window, all the services, including updates to change requests via our UI, remained operational.\n\nImpact \nAPI calls to \nPUT [https://api.split.io/internal/api/v2/changeRequests/\\{\\{ChangeRequestId\\}\\}](https://api.split.io/internal/api/v2/changeRequests/%7B%7BChangeRequestId%7D%7D) \nfailed with a 500 error.\n\nAll other API endpoints, SDK operations, and UI functionality remained unaffected.\n\nTimeline \\(UTC\\) \n08:41 PM Oct 29, 2025 - Authentication updates promoted to production \n\n08:57 PM Oct 29, 2025 - Increased 5xx error rate on the change request endpoint, but it remained below our detection threshold\n\n11:55 PM Oct 29, 2025 - Root cause identified after being reported as an issue by customers. Fix was blocked from being deployed due to the ongoing Azure outage\n\n03:21 AM Oct 30, 2025 - Issues were remedied after the fix was rolled out to all instances\n\nRemediation \nA code fix was deployed to address the issue \n\nAction Items \nExtend monitoring and alerting: Extend endpoint-specific alerting to detect errors proactively. Additionally, we will extend our internal pre-release testing to this scenario and other wider consumption patterns of this API.\n\nAccelerate deployment resilience efforts: Continue work to reduce third-party dependencies and improve deployment fault tolerance, minimizing the impact of external service instability.",
"postmortem_body_last_updated_at": "2025-11-05T12:24:31.778-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2025-11-05T12:24:31.778-08:00",
"reminder_intervals": null,
"resolved_at": "2025-10-29T20:30:00.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/y9vpk29wgxz9",
"started_at": "2025-10-29T20:30:00.000-07:00",
"status": "resolved",
"updated_at": "2025-11-05T12:30:21.500-08:00"
},
{
"components": [
{
"created_at": "2017-05-03T15:04:02.648-07:00",
"description": "Public and web console API",
"group_id": null,
"id": "s469h0b1pjcs",
"name": "API",
"page_id": "bn51dbtm6djn",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.537-08:00"
},
{
"created_at": "2017-05-03T15:03:54.067-07:00",
"description": null,
"group_id": null,
"id": "pdb48tnrnlz6",
"name": "Web Console",
"page_id": "bn51dbtm6djn",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.553-08:00"
},
{
"created_at": "2017-05-03T15:59:31.085-07:00",
"description": null,
"group_id": null,
"id": "kf9hkr38yhc2",
"name": "Integrations",
"page_id": "bn51dbtm6djn",
"position": 5,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-29T21:48:39.705-07:00"
},
{
"created_at": "2020-05-04T11:17:58.468-07:00",
"description": null,
"group_id": null,
"id": "0n6zlvhp2q8s",
"name": "Streaming Authentication Service",
"page_id": "bn51dbtm6djn",
"position": 9,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T12:53:15.323-07:00"
}
],
"created_at": "2025-10-20T07:44:39.000-07:00",
"id": "9wzkprp5xwlf",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0n6zlvhp2q8s",
"name": "Streaming Authentication Service",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2025-10-20T17:27:16.500-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-20T17:27:16.500-07:00",
"id": "hs6z6ckt481w",
"incident_id": "9wzkprp5xwlf",
"status": "resolved",
"tweet_id": 1980430651754242600,
"twitter_updated_at": "2025-10-20T17:27:16.783-07:00",
"updated_at": "2025-10-20T17:27:16.784-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "0n6zlvhp2q8s",
"name": "Streaming Authentication Service",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "All issues affecting the web application, public API, and streaming authentication service have been fully mitigated, and these systems are now operating normally.",
"created_at": "2025-10-20T14:44:58.862-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-20T14:44:58.862-07:00",
"id": "chpt6vstrk7n",
"incident_id": "9wzkprp5xwlf",
"status": "monitoring",
"tweet_id": 1980389816224350500,
"twitter_updated_at": "2025-10-20T14:45:01.036-07:00",
"updated_at": "2025-10-20T14:45:01.149-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "degraded_performance",
"old_status": "partial_outage"
},
{
"code": "0n6zlvhp2q8s",
"name": "Streaming Authentication Service",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "All issues affecting the web application, public API and streaming authentication service have been mitigated and these systems are now fully operational.\n\nData pipelines involved in our outbound S3 integration are showing signs of recovery as well.\n\nOur team will continue to monitor and provide updates.",
"created_at": "2025-10-20T12:53:15.354-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-20T12:53:15.354-07:00",
"id": "hy70l3jxbr1p",
"incident_id": "9wzkprp5xwlf",
"status": "monitoring",
"tweet_id": 1980361692996641300,
"twitter_updated_at": "2025-10-20T12:53:15.780-07:00",
"updated_at": "2025-10-20T12:53:15.792-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "partial_outage"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "degraded_performance",
"old_status": "partial_outage"
},
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "0n6zlvhp2q8s",
"name": "Streaming Authentication Service",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Our team has mitigated the issues affecting our web application and public API, with both being operational at this time. We continue to monitor in case the underlying AWS outage may impact further.",
"created_at": "2025-10-20T10:30:01.623-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-20T10:30:01.623-07:00",
"id": "h0v2ggyk2lj5",
"incident_id": "9wzkprp5xwlf",
"status": "monitoring",
"tweet_id": 1980325648154656800,
"twitter_updated_at": "2025-10-20T10:30:01.998-07:00",
"updated_at": "2025-10-20T10:30:01.999-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "s469h0b1pjcs",
"name": "API",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "kf9hkr38yhc2",
"name": "Integrations",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "0n6zlvhp2q8s",
"name": "Streaming Authentication Service",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "Customer client event consupmtion remains healthy, and impressions are being processed normally. However, web application is impacted with non-migrated (only login through app.split.io) customers experiencing intermittent login failures and configuration operation of feature flags and experiments.\n\nSDKs continue to receive flag updates, and customer applications won't see any disruptions. Customers may see timeouts for auth.split.io, with the SDK falling back to polling for updates resulting in slightly higher propagation times.\n\nCustomer may see errors and intermittent timeouts using the public API, which may prevent them from updating flags.\n\nWe have confirmed the first disruption occured from 12:44 AM to 1:31 AM PST, with the second being at 7:32 AM PST and is ongoing.\n\nOur teams are actively monitoring the situation and working with AWS to restores the functionality.",
"created_at": "2025-10-20T10:18:27.982-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-20T10:18:27.982-07:00",
"id": "l037h746g4j3",
"incident_id": "9wzkprp5xwlf",
"status": "monitoring",
"tweet_id": 1980322738897396000,
"twitter_updated_at": "2025-10-20T10:18:28.373-07:00",
"updated_at": "2025-10-20T10:18:28.375-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "partial_outage",
"old_status": "degraded_performance"
}
],
"body": "The ongoing AWS incident is causing timeouts for some of our internal services and providers, which can result in intermittent login issues.\n\nOur SDKs and data processing are unaffected and continue to operate without issues.",
"created_at": "2025-10-20T07:57:28.937-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-20T07:57:28.937-07:00",
"id": "98jjncsy62y3",
"incident_id": "9wzkprp5xwlf",
"status": "monitoring",
"tweet_id": 1980287258793320700,
"twitter_updated_at": "2025-10-20T07:57:29.261-07:00",
"updated_at": "2025-10-20T07:57:29.262-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "An issue affecting AWS is resulting in increased latencies for our web console and preventing the Live Tail functionality from working properly. We will continue to monitor the situation and provide updates here.\n\nMore details can be found on AWS status page: https://health.aws.amazon.com/health/status",
"created_at": "2025-10-20T07:44:39.612-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-10-20T07:44:39.000-07:00",
"id": "gp77jjknshww",
"incident_id": "9wzkprp5xwlf",
"status": "monitoring",
"tweet_id": 1980284032173884000,
"twitter_updated_at": "2025-10-20T07:44:39.976-07:00",
"updated_at": "2025-10-20T09:08:05.197-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2025-10-20T07:44:39.000-07:00",
"name": "Increased web console latencies and Live Tail issues",
"page_id": "bn51dbtm6djn",
"postmortem_body": "## **Summary**\n\nOn **October 20, 2025, 07:52 UTC**, a widespread outage in the **AWS us-east-1** region caused intermittent **login failures**, **API authentication errors**, and **temporary interruptions** when accessing the Live Tail service. These login issues did not impact customers who had already migrated to the **Harness platform**.\n\nAt **16:40 UTC**, we mitigated the login and API authentication issues by **redistributing traffic internally**, and full recovery was achieved at **21:43 UTC** once AWS confirmed complete remediation of the underlying outage.\n\nDuring the incident, some customers may have experienced intermittent delays or timeouts when sending impressions or other SDK events. In some instances, this could have resulted in event loss if SDK retries were exhausted due to failures in reaching our services. However, **feature flag evaluations and targeting systems remained fully operational** throughout, and no customer action was required.\n\n## **Root Cause**\n\nThe AWS **us-east-1 outage** prevented our autoscaling from provisioning new infrastructure to handle a **sudden traffic spike** caused by multiple customer applications restarting during this time. Although a backup cluster was available to restore services, we decided **not to initiate DR failover** due to the continued stability of our core feature management services and the high risk of disruption from affected third-party dependencies during the AWS outage.\n\n## Timeline\n\n| Time \\(UTC\\) | Event Description |\n| --- | --- |\n| **19 Oct 2025, 07:52 UTC** | Critical service degradation alerts received. Investigation began, initially focused on the affected service. |\n| **19 Oct 2025, 14:14 UTC** | AWS confirmed a widespread us-east-1 outage. |\n| **19 Oct 2025, 14:31 UTC** | Decision made not to fail over to the backup cluster to avoid risk to stable core systems. |\n| **19 Oct 2025, 16:40 UTC** | Traffic rerouted internally to alleviate login failures and API authentication errors. Monitoring confirmed significant improvement. |\n| **19 Oct 2025, 20:38 UTC** | Systems operational with minor residual degradation. |\n| **20 Oct 2025, 21:43 UTC** | Full recovery confirmed; incident moved to monitoring. |\n| **20 Oct 2025, 22:53 UTC** | AWS declared full restoration. |\n| **20 Oct 2025, 23:12 UTC** | Incident closed. |\n\n## **Remediation**\n\nWe mitigated login and authentication failures by redistributing **internal traffic**. **Feature flag evaluation and delivery** remained resilient and unaffected due to robust architectural design.\n\n## **Action Items**\n\n**We are auditing and reducing third-party dependencies,** particularly in areas that limit our ability to fail over confidently in the event of external outages. We are also **evaluating** the active cluster for relocation to a more resilient region \\(e.g., **us-west**\\) to reduce exposure to the fragility of the us-east region.",
"postmortem_body_last_updated_at": "2025-11-05T10:20:26.964-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2025-11-04T17:51:10.304-08:00",
"reminder_intervals": null,
"resolved_at": "2025-10-20T17:27:16.500-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/b1nnwvvfjzrt",
"started_at": "2025-10-20T07:44:39.000-07:00",
"status": "resolved",
"updated_at": "2025-11-05T10:20:26.966-08:00"
},
{
"components": [
{
"created_at": "2017-05-03T15:03:54.067-07:00",
"description": null,
"group_id": null,
"id": "pdb48tnrnlz6",
"name": "Web Console",
"page_id": "bn51dbtm6djn",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.553-08:00"
}
],
"created_at": "2025-09-10T07:42:36.000-07:00",
"id": "1sj60s37f4rd",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2025-09-10T11:46:21.157-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-09-10T11:46:21.157-07:00",
"id": "tvsy40h4zh3z",
"incident_id": "1sj60s37f4rd",
"status": "resolved",
"tweet_id": 1965849341366776000,
"twitter_updated_at": "2025-09-10T11:46:21.527-07:00",
"updated_at": "2025-09-10T11:46:21.528-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2025-09-10T11:05:34.478-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-09-10T11:05:34.478-07:00",
"id": "mgwxjvdmnqgq",
"incident_id": "1sj60s37f4rd",
"status": "monitoring",
"tweet_id": 1965839079431709000,
"twitter_updated_at": "2025-09-10T11:05:34.859-07:00",
"updated_at": "2025-09-10T11:05:34.861-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2025-09-10T10:23:02.022-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-09-10T10:23:02.022-07:00",
"id": "l2kb8t5z7t15",
"incident_id": "1sj60s37f4rd",
"status": "identified",
"tweet_id": 1965828373500101000,
"twitter_updated_at": "2025-09-10T10:23:02.372-07:00",
"updated_at": "2025-09-10T10:23:02.373-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating an issue causing intermittent problems loading the FME module on app.harness.io for migrated accounts.",
"created_at": "2025-09-10T07:42:36.678-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-09-10T07:42:36.000-07:00",
"id": "br7y9lcfszrm",
"incident_id": "1sj60s37f4rd",
"status": "investigating",
"tweet_id": 1965788001902224000,
"twitter_updated_at": "2025-09-10T07:42:37.079-07:00",
"updated_at": "2025-09-10T07:45:32.889-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2025-09-10T11:05:34.478-07:00",
"name": "Unable to start FME functionality",
"page_id": "bn51dbtm6djn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-09-10T11:46:21.157-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/3s6f7sxx31yj",
"started_at": "2025-09-10T07:42:36.000-07:00",
"status": "resolved",
"updated_at": "2025-09-10T11:46:21.534-07:00"
},
{
"components": [
{
"created_at": "2017-05-03T15:03:54.067-07:00",
"description": null,
"group_id": null,
"id": "pdb48tnrnlz6",
"name": "Web Console",
"page_id": "bn51dbtm6djn",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.553-08:00"
}
],
"created_at": "2025-07-22T11:50:03.815-07:00",
"id": "n3m8rc0vh6z4",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2025-07-22T13:46:46.443-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-07-22T13:46:46.443-07:00",
"id": "xbzylkw8z5yf",
"incident_id": "n3m8rc0vh6z4",
"status": "resolved",
"tweet_id": 1947760253006975200,
"twitter_updated_at": "2025-07-22T13:46:46.730-07:00",
"updated_at": "2025-07-22T13:46:46.732-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2025-07-22T12:46:02.682-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-07-22T12:46:02.682-07:00",
"id": "bvs6jbzmcv99",
"incident_id": "n3m8rc0vh6z4",
"status": "monitoring",
"tweet_id": 1947744970250895600,
"twitter_updated_at": "2025-07-22T12:46:03.036-07:00",
"updated_at": "2025-07-22T12:46:03.038-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2025-07-22T12:37:02.837-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-07-22T12:37:02.837-07:00",
"id": "3wvln6z06f0t",
"incident_id": "n3m8rc0vh6z4",
"status": "identified",
"tweet_id": 1947742705720955100,
"twitter_updated_at": "2025-07-22T12:37:03.145-07:00",
"updated_at": "2025-07-22T12:37:03.146-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating an issue causing intermittent failures during SSO for some of our customers.\n\nThis only affects web console SSO. Login using email and password via https://app.split.io/login is available for organizations configured to allow non-SSO logins.\n\nFeature flag evaluations using our SDKs are unaffected, as are all integrations and data processing.",
"created_at": "2025-07-22T11:50:03.909-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-07-22T11:50:03.909-07:00",
"id": "690vstcnbpwc",
"incident_id": "n3m8rc0vh6z4",
"status": "investigating",
"tweet_id": 1947730882741342500,
"twitter_updated_at": "2025-07-22T11:50:04.362-07:00",
"updated_at": "2025-07-22T11:50:04.363-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2025-07-22T12:46:02.682-07:00",
"name": "SAML issues",
"page_id": "bn51dbtm6djn",
"postmortem_body": "# Summary\n\nAn upgrade on the third party authentication platform used by Split, FusionAuth, caused initial issues related with older SAML cert algorithms that prevented a number of customer’s users from logging in with SSO to the Split web console.\n\nA second issue was caused by a Harness/Split deployment that included code changes impacting the authN flow. These changes were masked by the first issue. This second issue was affecting a number of customers who have SCIM enabled, whose users were unable to login via SSO to the Split web console.\n\n# Issue\n\nMany Split customers are using SSO configurations that have signing certificates based on older less secure algorithms, in this case `#rsa-sha1.`\n\nThe FusionAuth upgrade also upgraded the version of Java used within the FusionAuth cloud service from v17 to v21. Java 21 raised the security baseline by disabling several legacy and weak cryptographic algorithms by default, including `#rsa-sha1`.\n\nAfter the upgrade, some customer’s users were unable to login to the Split web console when using SSO.\n\nThere was a further complication caused by a Harness/Split deployment with changes that were incorrectly handling the authentication token sent from resulting in `404` responses.\n\nWhen a SCIM org user logs into Split via FusionAuth and using SSO, a `user-update` event is triggered because of minor changes to the user record \\(such as last login timestamp\\) and this `user-event` is sent as a request to a web-hook handled by a Split service. As part of validating this request, a request is made to fetch the organization memberships of the user included in the event, and it was this request that was not being handled correctly and sending back a `404` response which triggered a `500` response back to FusionAuth, failing the web-hook request and subsequently the SSO login flow.\n\n# Timeline\n\n| **Time \\(IST\\)** | **Event** |\n| --- | --- |\n| 7/22/2025 00:00 UTC | FusionAuth production cloud instance upgraded |\n| 7/22/2025 15:00 UTC | First customer reported issue with SSO logins |\n| 7/22/2025 15:20 UTC | Slack thread opened with FusionAuth support |\n| 7/22/2025 16:00 UTC | Deployment to production |\n| 7/22/2025 18:05 UTC | FusionAuth applied a fix to address the SAML cert issue |\n| 7/22/2025 18:32 UTC | FME Incident opened |\n| 7/22/2025 19:14 UTC | Previous deployment rolled back |\n| 7/23/2025 15:00 UTC | AuthZ fix deployed |\n| 7/23/2025 16:00 UTC | Incident marked as resolved |\n\n## RCA\n\nThe issue was caused by a FusionAuth upgrade which incorrectly applied stricter SAML certificate enforcement rules. The issue was compounded by a deployment which was incorrectly managing the authentication on some endpoints and affecting SAML login flows.\n\n# Action Items\n\n### Improvements\n\n* Improve logging/monitoring on the webhooks used in the authentication and validation service.\n* Update alerts for SAML and FusionAuth client errors to generate more effective notifications.\n* Collaborate with the FusionAuth team on upgrades to better analyze the potential impact before applying any further upgrades.",
"postmortem_body_last_updated_at": "2025-08-27T19:01:43.696-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": true,
"postmortem_published_at": "2025-08-27T18:04:03.169-07:00",
"reminder_intervals": null,
"resolved_at": "2025-07-22T13:46:46.443-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/3nzhwlg5fgkj",
"started_at": "2025-07-22T11:50:03.810-07:00",
"status": "resolved",
"updated_at": "2025-08-27T19:01:43.699-07:00"
},
{
"components": [],
"created_at": "2025-05-14T10:34:33.922-07:00",
"id": "jm11ljhd3mgv",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "The scheduled maintenance has been completed.",
"created_at": "2025-05-14T12:35:21.413-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-05-14T12:35:21.413-07:00",
"id": "65q3l0yn1wq8",
"incident_id": "jm11ljhd3mgv",
"status": "completed",
"tweet_id": 1922737517608174000,
"twitter_updated_at": "2025-05-14T12:35:21.750-07:00",
"updated_at": "2025-05-14T12:35:21.751-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2025-05-14T10:35:21.404-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-05-14T10:35:21.404-07:00",
"id": "tczdywk8d808",
"incident_id": "jm11ljhd3mgv",
"status": "in_progress",
"tweet_id": 1922707318552265000,
"twitter_updated_at": "2025-05-14T10:35:21.697-07:00",
"updated_at": "2025-05-14T10:35:21.698-07:00",
"wants_twitter_update": true
},
{
"affected_components": null,
"body": "To ensure the reliability of Split, we are conducting an application failover test on Wednesday, May 14th 10:30AM through 12:30PM PDT.",
"created_at": "2025-05-14T10:34:33.967-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-05-14T10:34:33.967-07:00",
"id": "pj4lyw63dxky",
"incident_id": "jm11ljhd3mgv",
"status": "scheduled",
"tweet_id": 1922707119641592000,
"twitter_updated_at": "2025-05-14T10:34:34.286-07:00",
"updated_at": "2025-05-14T10:34:34.287-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": null,
"name": "Quarterly DR Testing Maintenance Notice",
"page_id": "bn51dbtm6djn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-05-14T12:35:21.413-07:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2025-05-14T10:35:00.000-07:00",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2025-05-14T12:35:00.000-07:00",
"shortlink": "https://stspg.io/0np0mwnj6ngh",
"started_at": "2025-05-14T10:34:33.918-07:00",
"status": "completed",
"updated_at": "2025-05-14T12:35:21.756-07:00"
},
{
"components": [
{
"created_at": "2017-05-03T15:03:54.067-07:00",
"description": null,
"group_id": null,
"id": "pdb48tnrnlz6",
"name": "Web Console",
"page_id": "bn51dbtm6djn",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-16T08:22:33.553-08:00"
}
],
"created_at": "2025-05-07T11:17:29.804-07:00",
"id": "jhb04tdzvl51",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The fix resolved the issue. This incident is closed.",
"created_at": "2025-05-07T17:30:47.192-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-05-07T17:30:47.192-07:00",
"id": "rjg106n63wvc",
"incident_id": "jhb04tdzvl51",
"status": "resolved",
"tweet_id": 1920275149770928400,
"twitter_updated_at": "2025-05-07T17:30:47.475-07:00",
"updated_at": "2025-05-07T17:30:47.476-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2025-05-07T11:34:10.016-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-05-07T11:34:10.016-07:00",
"id": "k8q0tcvl37kn",
"incident_id": "jhb04tdzvl51",
"status": "monitoring",
"tweet_id": 1920185403707175400,
"twitter_updated_at": "2025-05-07T11:34:10.344-07:00",
"updated_at": "2025-05-07T11:34:10.346-07:00",
"wants_twitter_update": true
},
{
"affected_components": [
{
"code": "pdb48tnrnlz6",
"name": "Web Console",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Our team is currently investigating an issue affecting user authentication into the Split console for some users.",
"created_at": "2025-05-07T11:17:29.880-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-05-07T11:17:29.880-07:00",
"id": "brgjsz95fdtt",
"incident_id": "jhb04tdzvl51",
"status": "investigating",
"tweet_id": 1920181208782418400,
"twitter_updated_at": "2025-05-07T11:17:30.200-07:00",
"updated_at": "2025-05-07T11:17:30.201-07:00",
"wants_twitter_update": true
}
],
"metadata": {},
"monitoring_at": "2025-05-07T11:34:10.016-07:00",
"name": "Console authentication errors: \"Unable to start the FME functionality\"",
"page_id": "bn51dbtm6djn",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-05-07T17:30:47.192-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/7r9f60nmrj05",
"started_at": "2025-05-07T11:17:29.799-07:00",
"status": "resolved",
"updated_at": "2025-05-09T08:01:53.664-07:00"
}
],
"page": {
"id": "bn51dbtm6djn",
"name": "Split by Harness",
"url": "http://status.split.io"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}