Snapshot 316708
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"gt2fnpw91w5n",
"tksr3gcljnfh",
"pxxsfygldcfl",
"76wdvy8bc1s3",
"dyklptn1198n",
"pnyc8gx7wnz4",
"m0pjjfy20gk0",
"xz527893vv6l",
"9khchh1b07l7",
"4s0wtkq8l0z7",
"181q92gf4bp5",
"zvc6lrqg56kt",
"m2c1m6jzlgn7",
"qpyhcxy2qcgm",
"lf2tg93w80nn",
"pkmms5szc0h4",
"99t8zwnh19hm",
"gj7q3m9jbdsz",
"v8pc51wfhj9l",
"kl04y4xmnf86",
"90g1srlfnzcl"
],
"created_at": "2021-08-05T15:55:59.817-04:00",
"description": null,
"group_id": null,
"id": "w4n3wydsjxyn",
"name": "Socure ID+ Platform",
"page_id": "n4fswhz8sbxm",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2022-05-27T12:48:29.133-04:00"
},
{
"components": [
"qds43pwhq7d4",
"yrst3l8blyyb",
"vmjlz8c3p1rl",
"t1pf26cwpw0h"
],
"created_at": "2022-05-27T12:56:03.211-04:00",
"description": null,
"group_id": null,
"id": "607lfgjr8t56",
"name": "Global Watchlist",
"page_id": "n4fswhz8sbxm",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2022-06-08T12:40:02.424-04:00"
},
{
"components": [
"78kl1svyhzxg",
"ln63jswy1h8y",
"s8w1wr9813vj",
"4bznvvb8b79f",
"r6xx1b4wtr5c",
"c47j3knnfvkq"
],
"created_at": "2022-05-27T13:30:59.353-04:00",
"description": null,
"group_id": null,
"id": "xq1q5phvgbg2",
"name": "Socure SDKs",
"page_id": "n4fswhz8sbxm",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2022-06-08T12:40:02.439-04:00"
},
{
"created_at": "2020-10-19T18:24:32.788-04:00",
"description": "A centralized place to monitor verifications, view reports, and configure your service settings.",
"group_id": null,
"id": "kw3mhkvgbt7l",
"name": "Admin Dashboard",
"page_id": "n4fswhz8sbxm",
"position": 4,
"showcase": false,
"start_date": "2020-10-18T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-20T16:22:43.392-05:00"
},
{
"created_at": "2021-05-17T22:15:42.430-04:00",
"description": "Stay up-to-date with API references, release notes, blog posts, videos, and best practices for identity solutions.",
"group_id": null,
"id": "djgdt5qv5qn7",
"name": "Developer Hub",
"page_id": "n4fswhz8sbxm",
"position": 5,
"showcase": false,
"start_date": "2021-05-01T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2024-12-24T20:48:14.848-05:00"
},
{
"created_at": "2025-02-13T17:42:35.748-05:00",
"description": null,
"group_id": null,
"id": "m9s64thydsxz",
"name": "RiskOS",
"page_id": "n4fswhz8sbxm",
"position": 7,
"showcase": true,
"start_date": "2025-02-13T00:00:00.000-05:00",
"status": "operational",
"updated_at": "2026-10-16T17:18:17.552-04:00"
},
{
"components": [
"dy33c14qf88q",
"t26q478t11wy",
"6w9519m0d6w1",
"nd4l7cc8sl89",
"927msr7sgbzp",
"t2x1393c7wrc",
"lg3clp01llc6",
"9h39vxjbch9t",
"4qzh7ccgq8mt",
"4vhkgllrrgdq",
"dvd0z7bcz36s",
"kpqsl52l55yb",
"h4gbyv4kng2h",
"7z12p791gxxg",
"11v8jxfscv83",
"rllzkvrdft2p",
"kfn2mq2159xn",
"4fcs0yrmxzh8",
"knb164lmdkg9",
"p7bz63c8gtm3",
"twvlgbhlhkfy"
],
"created_at": "2025-07-30T17:41:48.055-04:00",
"description": null,
"group_id": null,
"id": "zhmnh3qjfvqh",
"name": "RiskOS Platform",
"page_id": "n4fswhz8sbxm",
"position": 8,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-07-30T17:41:48.055-04:00"
}
],
"incidents": [
{
"components": [
{
"created_at": "2023-05-24T07:42:49.083-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "w4n3wydsjxyn",
"id": "zvc6lrqg56kt",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2021-08-10T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.236-05:00"
},
{
"created_at": "2025-07-30T17:52:58.457-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "zhmnh3qjfvqh",
"id": "kpqsl52l55yb",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.295-05:00"
}
],
"created_at": "2026-11-20T17:03:15.945-05:00",
"id": "0msbvr9k9y59",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "During Saturday, November 21, 2026, 11:00PM (ET) through Saturday, November 21, 2026, 11:30PM (ET), SSA will be utilizing a portion of the standard maintenance window to complete multiple activities and anticipate an outage for the eCBSV service during this timeframe.",
"created_at": "2026-11-20T17:03:16.035-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T17:03:16.035-05:00",
"id": "j5z6vjfw6568",
"incident_id": "0msbvr9k9y59",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T17:03:16.035-05:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "eCBSV - Maintenance Notification",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": null,
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-11-21T23:00:00.000-05:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-11-21T23:30:00.000-05:00",
"shortlink": "https://stspg.io/979wqxxq88v5",
"started_at": "2026-11-20T17:03:15.932-05:00",
"status": "scheduled",
"updated_at": "2026-11-20T17:03:16.038-05:00"
},
{
"components": [
{
"created_at": "2020-10-19T18:24:32.788-04:00",
"description": "A centralized place to monitor verifications, view reports, and configure your service settings.",
"group_id": null,
"id": "kw3mhkvgbt7l",
"name": "Admin Dashboard",
"page_id": "n4fswhz8sbxm",
"position": 4,
"showcase": false,
"start_date": "2020-10-18T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-20T16:22:43.392-05:00"
}
],
"created_at": "2026-11-20T15:37:15.935-05:00",
"id": "39cqsrb484f4",
"impact": "critical",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "kw3mhkvgbt7l",
"name": "Admin Dashboard",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "We apologize for the service disruption and thank you for your patience. The service has been resolved, and we’ll share the RCA once it’s complete.",
"created_at": "2026-11-20T16:22:43.435-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T16:22:43.435-05:00",
"id": "lbd0krzb9q16",
"incident_id": "39cqsrb484f4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T16:22:43.435-05:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kw3mhkvgbt7l",
"name": "Admin Dashboard",
"new_status": "major_outage",
"old_status": "operational"
}
],
"body": "We have identified an issue that is causing the transaction details to fail to load on the Admin Dashboard. We are actively working on a mitigation plan.",
"created_at": "2026-11-20T15:37:16.048-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-20T15:37:16.048-05:00",
"id": "hwycm5sw4js7",
"incident_id": "39cqsrb484f4",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-20T15:37:16.048-05:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Admin Dashboard - Issues with Loading Transactions",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-20T16:22:43.435-05:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/y1s34102dzw4",
"started_at": "2026-11-20T15:37:15.927-05:00",
"status": "resolved",
"updated_at": "2026-11-20T16:22:43.460-05:00"
},
{
"components": [
{
"created_at": "2023-05-24T07:42:49.083-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "w4n3wydsjxyn",
"id": "zvc6lrqg56kt",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2021-08-10T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.236-05:00"
},
{
"created_at": "2025-07-30T17:52:58.457-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "zhmnh3qjfvqh",
"id": "kpqsl52l55yb",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.295-05:00"
}
],
"created_at": "2026-11-09T21:50:39.454-05:00",
"id": "2llmj71x2lh3",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-11-14T23:30:07.377-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-14T23:30:07.377-05:00",
"id": "py41sxqs2b2v",
"incident_id": "2llmj71x2lh3",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-14T23:30:07.377-05:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-11-14T23:00:09.229-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-14T23:00:09.229-05:00",
"id": "j1n36xt5y5b1",
"incident_id": "2llmj71x2lh3",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-14T23:00:09.229-05:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Maintenance will begin as scheduled in 60 minutes.",
"created_at": "2026-11-14T22:00:22.485-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-14T22:00:22.485-05:00",
"id": "71nz03pyl38n",
"incident_id": "2llmj71x2lh3",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-14T22:00:22.485-05:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "During Saturday,November 14, 2026, 11:00PM (ET) until 11:30PM (ET), SSA will be utilizing a portion of the standard maintenance window to complete multiple activities and anticipate an outage for the eCBSV service during this timeframe.",
"created_at": "2026-11-09T21:50:39.548-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-09T21:50:39.548-05:00",
"id": "r1583wxx3q6g",
"incident_id": "2llmj71x2lh3",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-09T21:50:39.548-05:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "eCBSV - Maintenance Notification",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-14T23:30:07.377-05:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-11-14T23:00:00.000-05:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": "2026-11-14T22:00:22.855-05:00",
"scheduled_until": "2026-11-14T23:30:00.000-05:00",
"shortlink": "https://stspg.io/r68yhds2s24n",
"started_at": "2026-11-09T21:50:39.438-05:00",
"status": "completed",
"updated_at": "2026-11-14T23:30:07.426-05:00"
},
{
"components": [
{
"created_at": "2020-12-29T12:27:26.096-05:00",
"description": "Global identity document verification and anti-fraud.",
"group_id": "w4n3wydsjxyn",
"id": "pxxsfygldcfl",
"name": "Predictive DocV",
"page_id": "n4fswhz8sbxm",
"position": 3,
"showcase": true,
"start_date": "2020-12-28T00:00:00.000-05:00",
"status": "operational",
"updated_at": "2026-11-12T00:34:35.566-05:00"
},
{
"created_at": "2025-07-30T17:44:37.972-04:00",
"description": "Global identity document verification and anti-fraud.",
"group_id": "zhmnh3qjfvqh",
"id": "6w9519m0d6w1",
"name": "Predictive DocV",
"page_id": "n4fswhz8sbxm",
"position": 3,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-12T00:34:35.593-05:00"
}
],
"created_at": "2026-11-11T23:49:02.951-05:00",
"id": "l1qcyt0ygpxt",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "pxxsfygldcfl",
"name": "Socure ID+ Platform - Predictive DocV",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "6w9519m0d6w1",
"name": "RiskOS Platform - Predictive DocV",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The issue affecting Document Verification decisioning has been fully resolved as of 12:26am Eastern Time. We apologize for any inconveniences this has caused and will share an RCA with a full analysis of the impact.",
"created_at": "2026-11-12T00:34:35.634-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-12T00:34:35.634-05:00",
"id": "v7f0b9qm8jjb",
"incident_id": "l1qcyt0ygpxt",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-12T00:34:35.634-05:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "pxxsfygldcfl",
"name": "Socure ID+ Platform - Predictive DocV",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "6w9519m0d6w1",
"name": "RiskOS Platform - Predictive DocV",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We have implemented a fix to mitigate the impact on Document Verification decisioning. We are currently monitoring the situation and will share another update once we can confirm this issue is resolved.",
"created_at": "2026-11-12T00:05:46.856-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-12T00:05:46.856-05:00",
"id": "kgtqgf3nkybh",
"incident_id": "l1qcyt0ygpxt",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-12T00:05:46.856-05:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "pxxsfygldcfl",
"name": "Socure ID+ Platform - Predictive DocV",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "6w9519m0d6w1",
"name": "RiskOS Platform - Predictive DocV",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating an active incident that is impacting decisioning on Document Verification. We will provide updates as soon possible as we are working to mitigate the issue.",
"created_at": "2026-11-11T23:49:03.101-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-11T23:49:03.101-05:00",
"id": "sjl2rgkyrdr7",
"incident_id": "l1qcyt0ygpxt",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T23:49:03.101-05:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-12T00:05:46.856-05:00",
"name": "[SEV1] Document Verification Decisioning Impacted",
"page_id": "n4fswhz8sbxm",
"postmortem_body": "**Date Range: 2026/11/10 20:15 PM ET - 2026/11/12 00:24 AM ET** \n**Impact:** ~7.7% DocV Transactions Impact \n**Status:** Resolved\n\n### 1. Summary\n\n**What happened?** Between November 10, 2026, at 20:15 PM ET and November 12, 2026, at 00:24 AM ET, a decision break occurred in the DocV service during data matching evaluation. The issue caused certain transaction decisions with specific reason codes to fail to process properly, resulting in ~7.7% of the DocV transactions subject to incorrect final decisions across affected customers.\n\n**Who/what was affected?** Approximately 7.7% document verification transactions were impacted. Affected accounts experienced decision overrides where transactions that should have been rejected or sent to review were processed with unintended outcomes. Actual customer impact is lower than the total count: some customers make decisions using DocV’s reason codes rather than its final decision, and those customers were less exposed. To reiterate the reason codes were firing normally and the only impact was on the DocV Decision being overridden.\n\n\n\n**Duration:** 28 hours 9 minutes \\(approximately 27 hours undetected; 64 minutes from detection to fix\\)\n\n### **2. Timeline**\n\n**Date / Time \\(ET\\)** \n**2026/11/10 20:15** - A feature flag was enabled for one internal test account to prepare for an upcoming change. A bug applied it to all customer accounts. Impact begins.\n\n**2026/11/11 23:20** - The team reported incorrect DocV decisions on transactions with specific reason codes. Investigation began.\n\n**2026/11/11 23:28** - Incident declared. The team rolled back recent code changes to contain impact.\n\n**2026/11/12 00:08** - Testing confirmed the rollback did not resolve the issue. Investigation shifted to configuration changes.\n\n**2026/11/12 00:24** - Feature flag disabled. New transactions correctly completed with the decision based on reason codes.\n\n### **3. Root Cause**\n\n* **Primary Root Cause:** A defect in DocV’s data-matching logic prevented reason codes from mapping to rejection or review outcomes. Enabling a feature flag activated the defect, and a separate bug applied that flag to all customers instead of one internal test account.\n* **Contributing Factors:**\n\n * The initial deployment rollback failed to resolve the issue because the breaking change originated from a feature flag modification rather than a standard code deployment.\n * Limited auditability and change tracking within the feature flag service hindered rapid identification of the root cause.\n * Active flag changes were not documented or communicated across teams, delaying diagnosis.\n * Validation gaps occurred during feature flag testing: while the designated target test account was verified, unintended side effects on non-target accounts were not evaluated prior to activation.\n * No alerting on abnormal shifts in rejection or review rates, allowing the impact to go undetected for approximately 27 hours.\n \n* **Non-contributing Factors:**\n\n * Standard code deployments were confirmed to be non-contributing factors.\n \n\n### **4. Resolution**\n\nAt 00:24 ET on November 12, we disabled the feature flag, restoring correct mapping of reason codes to rejection and review outcomes. Testing confirmed that new transactions with the affected reason codes now reject correctly.\n\n\n\n### **5. Corrective and Preventive Actions**\n\n**Fix flag scoping bug:** Fix the bug that applied an account-scoped feature flag to all customers. \\(WIP - ETA is set to 11/17.\\)\n\n**Strengthen feature flag testing:** Test the impact of flag changes on both the intended target accounts and other accounts before enabling them in production. This will be enforced in the DocV testing strategy for future releases. \\(Ongoing\\)\n\n**Improve feature flag change tracking:** Log who changes a flag, what changed, and when, so flag-related changes are identified quickly during incidents. \\(Ongoing\\)\n\n**Review impacted transactions:** Identify affected transactions and coordinate any needed remediation with customers, accounting for the fact that some customers make decisions using reason codes independently. \\(WIP - We will support individual customers based on the remediation needs.\\)\n\n**Audit similar feature flags:** Review active and recently changed feature flags for similar data-matching or reason-code evaluation risks. Confirm each flag’s owner, purpose, scope, and current status, and disable or remediate any problematic configurations found. \\(WIP - We’ve done the audit for high risk ones and found none. ETA to finish all is set to 11/20.\\)\n\n**Improve monitoring and alerting:** Add monitoring for unusual changes in reason-code outcomes, rejection and review rates, and decision overrides. Alert the on-call team when metrics move outside expected thresholds. \\(Planned - We will work with our Data Science team and build smarter alerts on the outcome instead of static alerts. ETA for the initial design is by the end of Q4.\\)\n\n### **6. Lessons Learned**\n\n* **Feature flag governance and testing:** Feature flag changes can affect production decisioning as much as code deployments. They need equivalent review, testing, and change controls. Validation should cover target and non-target accounts, as well as relevant workflows, to catch unintended effects before activation.\n* **Auditability and incident response:** Complete, timely records of who changed a flag, what changed, and when are essential for diagnosing incidents. Response procedures should also account for configuration changes, since a code rollback may not reverse their effects.\n* **Customer-specific impact:** Customers may rely on DocV decisions or use reason codes in their own logic. Impact assessments, remediation, and communications should account for these different integration models.\n* **Monitoring:** Without reason-code and override monitoring, this incident went undetected for approximately 27 hours. Outcome-level alerting would have caught it within minutes.\n\n### **7. Next Steps \u0026 Ongoing Commitment**\n\nWe have restored the expected decisioning behavior by disabling the feature flag and verified that new transactions with the affected reason codes are receiving the correct outcomes. We are reviewing potentially similar feature flags and assessing the impact on affected transactions so we can coordinate any necessary follow-up with customers.\n\nWe will strengthen feature flag auditability, testing, rollout controls, and monitoring for reason-code outcomes and decision overrides. We will track these actions through completion and share updates as the review and remediation progress. We are committed to continuously improving our processes and building more resilient software so we can better prevent, detect, and respond to similar issues in the future.",
"postmortem_body_last_updated_at": "2026-11-13T17:03:05.254-05:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-13T14:00:46.707-05:00",
"reminder_intervals": null,
"resolved_at": "2026-11-12T00:34:35.634-05:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/c5g52tg3k85r",
"started_at": "2026-11-11T23:49:02.936-05:00",
"status": "resolved",
"updated_at": "2026-11-13T17:03:05.257-05:00"
},
{
"components": [
{
"created_at": "2023-05-24T07:42:49.083-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "w4n3wydsjxyn",
"id": "zvc6lrqg56kt",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2021-08-10T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.236-05:00"
},
{
"created_at": "2025-07-30T17:52:58.457-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "zhmnh3qjfvqh",
"id": "kpqsl52l55yb",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.295-05:00"
}
],
"created_at": "2026-10-29T11:11:10.096-04:00",
"id": "5tcb1cqmfz95",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-11-01T02:00:09.322-05:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-01T02:00:09.322-05:00",
"id": "dfftwy0kh1qx",
"incident_id": "5tcb1cqmfz95",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-01T02:00:09.322-05:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-31T23:30:08.621-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T23:30:08.621-04:00",
"id": "0mypqm9zhs97",
"incident_id": "5tcb1cqmfz95",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T23:30:08.621-04:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Maintenance will begin as scheduled in 60 minutes.",
"created_at": "2026-10-31T22:30:42.681-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T22:30:42.681-04:00",
"id": "vqjzg6cp654c",
"incident_id": "5tcb1cqmfz95",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T22:30:42.681-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "During Saturday, October 31, 2026, 11:30PM (ET) until Sunday, November 1, 2026, 3:00AM (ET), SSA will be utilizing a portion of the standard maintenance window to complete multiple activities and anticipate an outage for the eCBSV service during this timeframe",
"created_at": "2026-10-29T11:11:10.209-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-29T11:11:10.209-04:00",
"id": "km7mmz4hq9y0",
"incident_id": "5tcb1cqmfz95",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-29T11:11:10.209-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "eCBSV - Maintenance Notification",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-01T02:00:09.322-05:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-31T23:30:00.000-04:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": "2026-10-31T22:30:45.085-04:00",
"scheduled_until": "2026-11-01T02:00:00.000-05:00",
"shortlink": "https://stspg.io/j45dgbq45j5k",
"started_at": "2026-10-29T11:11:10.080-04:00",
"status": "completed",
"updated_at": "2026-11-01T02:00:10.218-05:00"
},
{
"components": [],
"created_at": "2026-10-27T12:00:00.000-04:00",
"id": "w5983cfyv5ns",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "On October 27, 2026, some customers were unable to receive RiskOS Sandbox webhooks. The issue was caused by invalid webhook configurations that caused the Sandbox notification sender to stop processing events. The malformed configurations have since been corrected. All processing has recovered and webhook delivery has successfully resumed",
"created_at": "2026-10-27T13:08:00.612-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-27T12:00:00.000-04:00",
"id": "v60dt6lmj6v7",
"incident_id": "w5983cfyv5ns",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T13:08:00.612-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "RETROACTIVE: [SEV3] - Sandbox RiskOS Webhooks Not Firing",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-27T12:00:00.000-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/63vvvjscmbqk",
"started_at": "2026-10-27T13:08:00.543-04:00",
"status": "resolved",
"updated_at": "2026-10-27T13:08:00.624-04:00"
},
{
"components": [
{
"created_at": "2023-05-24T07:42:49.083-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "w4n3wydsjxyn",
"id": "zvc6lrqg56kt",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2021-08-10T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.236-05:00"
},
{
"created_at": "2025-07-30T17:52:58.457-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "zhmnh3qjfvqh",
"id": "kpqsl52l55yb",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.295-05:00"
}
],
"created_at": "2026-10-23T17:00:25.378-04:00",
"id": "x274xnglptkf",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "At approximately 4:56 (ET), service has been restored and eCBSV is operating normally at this time. We apologize for any inconveniences this may have caused.",
"created_at": "2026-10-23T17:12:09.878-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-23T17:12:09.878-04:00",
"id": "vtcn6zbt456g",
"incident_id": "x274xnglptkf",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T17:12:09.878-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "SSA is currently experiencing a service-impacting event affecting multiple applications and services, including the Electronic Consent Based Social Security Number Verification (eCBSV) service. A small number of transactions are affected as the disruption was first noted at approximately 4:47 PM (ET) on October 23, 2026. We will share another update once we receive notice that service has been restored, and eCBSV is operating normally again.",
"created_at": "2026-10-23T17:00:25.503-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T17:00:25.503-04:00",
"id": "7lf853bsrnz4",
"incident_id": "x274xnglptkf",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T17:00:25.503-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-23T17:00:25.503-04:00",
"name": "[SEV3] eCBSV API Outage Resulting in 5xx Errors",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-23T17:12:09.878-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/gt4g490btl5s",
"started_at": "2026-10-23T17:00:25.364-04:00",
"status": "resolved",
"updated_at": "2026-10-23T17:12:09.902-04:00"
},
{
"components": [
{
"created_at": "2020-12-29T12:27:26.096-05:00",
"description": "Global identity document verification and anti-fraud.",
"group_id": "w4n3wydsjxyn",
"id": "pxxsfygldcfl",
"name": "Predictive DocV",
"page_id": "n4fswhz8sbxm",
"position": 3,
"showcase": true,
"start_date": "2020-12-28T00:00:00.000-05:00",
"status": "operational",
"updated_at": "2026-11-12T00:34:35.566-05:00"
},
{
"created_at": "2025-07-30T17:44:37.972-04:00",
"description": "Global identity document verification and anti-fraud.",
"group_id": "zhmnh3qjfvqh",
"id": "6w9519m0d6w1",
"name": "Predictive DocV",
"page_id": "n4fswhz8sbxm",
"position": 3,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-12T00:34:35.593-05:00"
}
],
"created_at": "2026-10-19T15:04:11.271-04:00",
"id": "16mjtdmm64j5",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "pxxsfygldcfl",
"name": "Socure ID+ Platform - Predictive DocV",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6w9519m0d6w1",
"name": "RiskOS Platform - Predictive DocV",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This issue has been resolved, and we’re monitoring performance to ensure the services remain stable. We apologize for any inconvenience this may have caused.",
"created_at": "2026-10-19T21:57:46.080-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T21:57:46.080-04:00",
"id": "24j9jk496hdm",
"incident_id": "16mjtdmm64j5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T21:57:46.080-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "pxxsfygldcfl",
"name": "Socure ID+ Platform - Predictive DocV",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6w9519m0d6w1",
"name": "RiskOS Platform - Predictive DocV",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Our team has taken steps to mitigate the latency experienced for DocV image display and downloads. There continues to be no impact to DocV evaluations at this time and the team will continue to monitor the situation. We will provide another update once all impact has been resolved.",
"created_at": "2026-10-19T17:15:31.744-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-19T17:15:31.744-04:00",
"id": "2l5n5hsbd6gx",
"incident_id": "16mjtdmm64j5",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T17:15:31.744-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "pxxsfygldcfl",
"name": "Socure ID+ Platform - Predictive DocV",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6w9519m0d6w1",
"name": "RiskOS Platform - Predictive DocV",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are currently investigating an issue that is causing intermittent latency with displaying and downloading DocV Images for a small percentage of completed transactions. There is no impact to DocV nor evaluations at this time but images may take additional time to appear.",
"created_at": "2026-10-19T15:04:11.367-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T15:04:11.367-04:00",
"id": "xnwyg7xhhc7c",
"incident_id": "16mjtdmm64j5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T15:04:11.367-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-19T17:15:31.744-04:00",
"name": "[SEV3] Intermittent Latency in Dashboard For DocV Image Display and Download",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-19T21:57:46.080-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/0s56nl1t6nc1",
"started_at": "2026-10-19T15:04:11.237-04:00",
"status": "resolved",
"updated_at": "2026-10-19T21:57:50.461-04:00"
},
{
"components": [
{
"created_at": "2023-05-24T07:42:49.083-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "w4n3wydsjxyn",
"id": "zvc6lrqg56kt",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2021-08-10T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.236-05:00"
},
{
"created_at": "2025-07-30T17:52:58.457-04:00",
"description": "Securely verify your customers’ personally identifiable information through a direct connection with the Social Security Administration (SSA).",
"group_id": "zhmnh3qjfvqh",
"id": "kpqsl52l55yb",
"name": "eCBSV",
"page_id": "n4fswhz8sbxm",
"position": 12,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-11-14T23:30:07.295-05:00"
}
],
"created_at": "2026-10-18T15:33:58.165-04:00",
"id": "1jn0qhkbr242",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "zvc6lrqg56kt",
"name": "Socure ID+ Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kpqsl52l55yb",
"name": "RiskOS Platform - eCBSV",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "SSA experienced a service-impacting event that affected multiple applications and services, including the Electronic Consent Based Social Security Number Verification (eCBSV) service. A small number of transactions were affected as the disruption began at approximately 2:50PM (ET) on October 18, 2026, and was resolved at approximately 3:10PM (ET). Service has been restored, and eCBSV is operating normally at this time.",
"created_at": "2026-10-18T15:33:58.237-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-18T15:33:58.237-04:00",
"id": "s0pyj861cptf",
"incident_id": "1jn0qhkbr242",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-18T15:33:58.237-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "RETROACTIVE: [SEV3] - eCBSV API Outage Resulting in 5xx Errors",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-18T15:33:58.237-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/hztck0m1xcsr",
"started_at": "2026-10-18T15:33:58.154-04:00",
"status": "resolved",
"updated_at": "2026-10-18T15:33:58.247-04:00"
},
{
"components": [
{
"created_at": "2025-02-13T17:42:35.748-05:00",
"description": null,
"group_id": null,
"id": "m9s64thydsxz",
"name": "RiskOS",
"page_id": "n4fswhz8sbxm",
"position": 7,
"showcase": true,
"start_date": "2025-02-13T00:00:00.000-05:00",
"status": "operational",
"updated_at": "2026-10-16T17:18:17.552-04:00"
}
],
"created_at": "2026-10-16T16:20:09.007-04:00",
"id": "lzv3z7d4zkws",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "m9s64thydsxz",
"name": "RiskOS",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The incident has been resolved, and we are no longer seeing errors. Services are operating normally, and we’ll continue to monitor performance. We appreciate your patience while we worked through this.",
"created_at": "2026-10-16T22:10:39.768-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T22:10:39.768-04:00",
"id": "7268nmbbb81p",
"incident_id": "lzv3z7d4zkws",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T22:10:39.768-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "m9s64thydsxz",
"name": "RiskOS",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We appreciate your patience as our team is still monitoring the situation closely. Error rates have subsided as we continue to work towards a full resolution.",
"created_at": "2026-10-16T18:59:58.548-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-16T18:59:58.548-04:00",
"id": "gg27bj7tbgvs",
"incident_id": "lzv3z7d4zkws",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T18:59:58.548-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "m9s64thydsxz",
"name": "RiskOS",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We’ve taken an interim measure, and the elevated error rate has subsided. We haven’t observed related errors in the past 15 minutes. We’re moving the incident to monitoring while we continue work on a full resolution. We’ll share another update when the situation changes.",
"created_at": "2026-10-16T17:18:17.591-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-16T17:18:17.591-04:00",
"id": "xx4z22jjj0dv",
"incident_id": "lzv3z7d4zkws",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T17:18:17.591-04:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "m9s64thydsxz",
"name": "RiskOS",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Starting at 3:00 EDT, some customers may experience intermittent errors or failed requests when using the RiskOS Evaluate APIs. We’ve identified a database issue and are actively working to resolve it. We apologize for the disruption and appreciate your patience while we investigate.",
"created_at": "2026-10-16T16:20:09.096-04:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-16T16:20:09.096-04:00",
"id": "dkrl0j3rn4vd",
"incident_id": "lzv3z7d4zkws",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T16:20:09.096-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-16T17:18:17.591-04:00",
"name": "[Sev3] Elevated error rate for RiskOS Evaluation APIs",
"page_id": "n4fswhz8sbxm",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-16T22:10:39.768-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/11w26pyfmff0",
"started_at": "2026-10-16T16:20:09.001-04:00",
"status": "resolved",
"updated_at": "2026-10-16T22:10:39.789-04:00"
},
{
"components": [
{
"created_at": "2021-06-13T17:59:20.362-04:00",
"description": "Screen and monitor customers in real-time against Anti-Money Laundering (AML) watchlists.",
"group_id": "607lfgjr8t56",
"id": "qds43pwhq7d4",
"name": "Global Watchlist Screening with Monitoring",
"page_id": "n4fswhz8sbxm",
"position": 1,
"showcase": false,
"start_date": "2021-05-01T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-10-15T19:04:27.885-04:00"
},
{
"created_at": "2025-07-30T17:41:48.088-04:00",
"description": null,
"group_id": "zhmnh3qjfvqh",
"id": "dy33c14qf88q",
"name": "Sigma Identity Fraud",
"page_id": "n4fswhz8sbxm",
"position": 1,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-09-16T06:07:39.386-04:00"
},
{
"created_at": "2020-12-29T12:29:20.966-05:00",
"description": "Screen customers in real-time against Anti-Money Laundering (AML) watchlists.",
"group_id": "607lfgjr8t56",
"id": "yrst3l8blyyb",
"name": "Watchlist Standard",
"page_id": "n4fswhz8sbxm",
"position": 2,
"showcase": false,
"start_date": "2020-12-28T00:00:00.000-05:00",
"status": "operational",
"updated_at": "2026-10-15T19:04:27.907-04:00"
},
{
"created_at": "2025-07-30T17:43:32.406-04:00",
"description": "Reduce synthetic identity fraud.",
"group_id": "zhmnh3qjfvqh",
"id": "t26q478t11wy",
"name": "Sigma Synthetic Fraud",
"page_id": "n4fswhz8sbxm",
"position": 2,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-09-16T06:07:39.418-04:00"
},
{
"created_at": "2022-05-27T12:41:14.895-04:00",
"description": "Screen customers in real-time against Anti-Money Laundering (AML) watchlists.",
"group_id": "607lfgjr8t56",
"id": "vmjlz8c3p1rl",
"name": "Watchlist Plus",
"page_id": "n4fswhz8sbxm",
"position": 3,
"showcase": false,
"start_date": "2022-05-27T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-10-15T19:04:27.949-04:00"
},
{
"created_at": "2022-05-27T13:25:09.206-04:00",
"description": "Screen customers in real-time against Anti-Money Laundering (AML) watchlists.",
"group_id": "607lfgjr8t56",
"id": "t1pf26cwpw0h",
"name": "Watchlist Premier",
"page_id": "n4fswhz8sbxm",
"position": 4,
"showcase": false,
"start_date": "2022-05-27T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-10-15T19:04:27.993-04:00"
},
{
"created_at": "2025-07-30T17:44:52.568-04:00",
"description": "Global KYC and identity verification data.",
"group_id": "zhmnh3qjfvqh",
"id": "nd4l7cc8sl89",
"name": "KYC",
"page_id": "n4fswhz8sbxm",
"position": 4,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-08-03T13:32:06.101-04:00"
},
{
"created_at": "2025-07-30T17:45:13.727-04:00",
"description": "Assess the risk level of an email on a more granular level.",
"group_id": "zhmnh3qjfvqh",
"id": "927msr7sgbzp",
"name": "Email RiskScore",
"page_id": "n4fswhz8sbxm",
"position": 5,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-08-03T13:32:06.130-04:00"
},
{
"created_at": "2025-07-30T17:50:45.048-04:00",
"description": "Assess the risk level of an address on a more granular level.",
"group_id": "zhmnh3qjfvqh",
"id": "t2x1393c7wrc",
"name": "Address RiskScore",
"page_id": "n4fswhz8sbxm",
"position": 6,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-08-03T13:32:06.159-04:00"
},
{
"created_at": "2025-07-30T17:51:52.688-04:00",
"description": "Assess the risk level of an phone on a more granular level.",
"group_id": "zhmnh3qjfvqh",
"id": "9h39vxjbch9t",
"name": "Phone RiskScore",
"page_id": "n4fswhz8sbxm",
"position": 8,
"showcase": true,
"start_date": "2025-07-30T00:00:00.000-04:00",
"status": "operational",
"updated_at": "2026-08-03T13:32:06.216-04:00"
}
],
"created_at": "2026-10-15T22:23:49.908-04:00",
"id": "h3vjhns0pshf",
"impact": "critical",
"impact_override": "critical",
"incident_updates": [
{
"affected_components": [
{
"code": "qds43pwhq7d4",
"name": "Global Watchlist - Global Watchlist Screening with Monitoring",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "yrst3l8blyyb",
"name": "Global Watchlist - Watchlist Standard",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "vmjlz8c3p1rl",
"name": "Global Watchlist - Watchlist Plus",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t1pf26cwpw0h",
"name": "Global Watchlist - Watchlist Premier",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "dy33c14qf88q",
"name": "RiskOS Platform - Sigma Identity Fraud",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t26q478t11wy",
"name": "RiskOS Platform - Sigma Synthetic Fraud",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nd4l7cc8sl89",
"name": "RiskOS Platform - KYC",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "927msr7sgbzp",
"name": "RiskOS Platform - Email RiskScore",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t2x1393c7wrc",
"name": "RiskOS Platform - Address RiskScore",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9h39vxjbch9t",
"name": "RiskOS Platform - Phone RiskScore",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Socure RiskOS and ID+ services have been restored. An error in a rollout caused a temporary spike in error rates from our services between 18:12 and 18:56 Eastern time. The error has been corrected and services are stable now. We understand this outage is unacceptable and a full RCA will be available as soon as full analysis of the issue is complete.",
"created_at": "2026-10-15T22:23:49.994-04:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-15T22:23:49.994-04:00",
"id": "vqwwfdmcpcnc",
"incident_id": "h3vjhns0pshf",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T22:23:49.994-04:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "RETROACTIVE: [SEV 1] Outage Resulting in 5xx Errors",
"page_id": "n4fswhz8sbxm",
"postmortem_body": "## **Root Cause Analysis**\n\n**Incident:** Socure APIs Outage\n\n**Date Range:** 15-Oct-2026\n\n**Impact:** Interruption in service for Socure APIs\n\n**Status:** Resolved\n\n### **1. Summary**\n\nOn 15-Oct-2026, between 18:12 and 19:00 Eastern time Socure experienced an interruption to its RiskOS and ID\\+ APIs, causing errors and timeouts to users. The first phase ran from 18:12 to 18:20, and was a major interruption for virtually every service. The second phase of the outage ran from 18:20 to 19:00, and was a partial outage for services that rely on KYC data, where customers saw elevated error rates and latencies for some services, but traffic was still being served.\n\n### **2. Timeline**\n\n**Date / Time \\(ET\\)** | Event\n\n**18:12** | A routine cleanup change is merged, moving a source repo from one code repository to another. A bug in our review tool hides the scope of the change.\n\n**18:13** | ArgoCD restarts most services in a major Kubernetes cluster all at once.\n\n**18:20** | All service pods are running. Partial API service restored, although KYC services are still seeing an elevated error rate.\n\n**18:21** | The flood of startup traffic overwhelms our KYC Search service. Error rates spike.\n\n**18:42** | SRE diagnoses the cascading failure impacting KYC Search and blocks traffic to it so it can come up.\n\n**18:56** | The KYC Search service is fully up, and SRE returns traffic to it. Downstream services stabilize. \n\n**19:00** | All traffic running as normal, incident declared stable.\n\n### **3. Root Cause**\n\n* **Primary Root Cause:**\n\nAs a part of an ongoing migration, an on-call SRE made a change to ArgoCD, the system we use to manage the kubernetes clusters Socure runs its services on. The change was scoped to a single cluster, was reviewed carefully before pushing, caused no issues in other environments it was tested in, and should have had no visible impact. Unfortunately, the change updated a legacy setting that did not appear in the change review. That legacy setting governed what services ArgoCD managed, and removed many live services from ArgoCD. In consequence, ArgoCD removed those services from production briefly, before the setting was corrected.\n\n* **Contributing Factors:**\n\nThe KYC Search service is a backend piece of Socure infrastructure that provides services to several of our internal products. When that service restarted from scratch at the same time as all of the clients of the service did, a small number of pods in the service crashed under load. This increased the load on the remaining pods, some of which crashed in turn. This cascading failure prevented the service from coming back on its own the way the rest of Socure’s infrastructure did. Diagnosing this separate failure mode took additional time, and remediating it fixed the remaining errors after the initial incident.\n\n### **4. Resolution**\n\nThe ArgoCD change was reverted immediately, causing services to come back to their prior state. The KYC Search service required manual effort to block traffic from reaching it until a majority of pods in the service could come up. After the service was healthy, we could restore traffic to it and resolve the overall incident.\n\n### **5. Corrective and Preventive Actions**\n\n### **6. Lessons Learned**\n\n* Socure’s tooling for viewing the changes it makes to production did not work correctly. We are reviewing these tools to ensure they operate correctly under all use cases and fix any cases like this where the full scope of a change is not clear.\n* Augment the existing change review process with agentic support. While in this case, the specific change was not visible in the plan, having an extra reviewer that is supported by an extensive training model will help ensure issues are caught before submission.\n* Even changes that appear to have no impact, when they are made to the governance system for our production jobs, can be risky, and should be shifted further out of US business hours.\n* Our KYC Search service is vulnerable to a “thundering-herd” problem when it receives full traffic flow when it is not fully online. We need to strengthen our circuit breakers to prevent this issue recurring.\n* We run too many of our core API services in a single kubernetes cluster. Even though that cluster is distributed across multiple availability zones, the shared control plane creates too high a blast radius for changes to that cluster.\n\n### **7. Next Steps \u0026 Ongoing Commitment**\n\nSocure understands our customers rely on us as a key provider in critical business flows, and that this service interruption is not acceptable. We apologize for the impact this issue caused, and commit to ensuring it does not recur. To that end, we will deliver details on our plans to re-architect our application layout by November, as well as improve the resilience of the KYC Search service in Q1 of next year.",
"postmortem_body_last_updated_at": "2026-10-22T17:49:24.180-04:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-16T21:42:34.368-04:00",
"reminder_intervals": null,
"resolved_at": "2026-10-15T22:23:49.994-04:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bkj6g50lfvlq",
"started_at": "2026-10-15T22:23:49.886-04:00",
"status": "resolved",
"updated_at": "2026-10-22T17:49:24.183-04:00"
}
],
"page": {
"id": "n4fswhz8sbxm",
"name": "Socure",
"url": "https://status.socure.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}