Snapshot 71761
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2021-06-24T20:27:00.695Z",
"description": null,
"group_id": null,
"id": "xr048l3j0887",
"name": "Device APIs",
"page_id": "2qlwmy349kf8",
"position": 1,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.088Z"
},
{
"created_at": "2025-06-06T11:54:41.445Z",
"description": null,
"group_id": null,
"id": "1wlhwqqkmzt6",
"name": "Issuing API",
"page_id": "2qlwmy349kf8",
"position": 2,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-27T22:36:52.845Z"
},
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
},
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
},
{
"created_at": "2022-03-30T21:01:15.910Z",
"description": null,
"group_id": null,
"id": "83thpbn47dfm",
"name": "Crypto APIs",
"page_id": "2qlwmy349kf8",
"position": 5,
"showcase": true,
"start_date": "2022-02-28T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-11-29T19:24:23.688Z"
},
{
"created_at": "2024-08-29T00:36:38.738Z",
"description": null,
"group_id": null,
"id": "0pvxdjtcngy8",
"name": "External Provider",
"page_id": "2qlwmy349kf8",
"position": 6,
"showcase": false,
"start_date": "2024-08-29T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-06-03T00:30:07.188Z"
},
{
"created_at": "2026-01-13T11:34:54.767Z",
"description": null,
"group_id": null,
"id": "5qbjzhmsyn96",
"name": "Crypto Web",
"page_id": "2qlwmy349kf8",
"position": 7,
"showcase": true,
"start_date": "2026-01-13T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-01-13T11:40:11.227Z"
},
{
"created_at": "2026-10-01T15:33:39.695Z",
"description": null,
"group_id": null,
"id": "sj976kyh6d71",
"name": "Business API",
"page_id": "2qlwmy349kf8",
"position": 8,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.139Z"
},
{
"created_at": "2026-10-01T15:34:09.693Z",
"description": null,
"group_id": null,
"id": "6xs7z1zzrf8v",
"name": "Feedback API",
"page_id": "2qlwmy349kf8",
"position": 9,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.163Z"
}
],
"incidents": [
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-10-06T13:21:13.161Z",
"id": "bzkmpbzlmt8k",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing scheduled database maintenance on each production region at the times below. \n\nEach window may take up to 1 hour.\n\nIndia: Wed Oct 14, 03:15 IST (Tue Oct 13, 21:45 UTC)\n\nWhat to expect during the maintenance window:\nDashboard: Creating or updating rules, queues, and alerts will be temporarily unavailable.\nRisk rule evaluation (online API): No impact. Evaluations will continue to operate as usual.\nAlerts created via API: These will continue to be created, but you may see short delays.\n\n\nIf you have any questions, please reach out to your account manager or our support team.",
"created_at": "2026-10-06T13:21:13.314Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T13:21:13.314Z",
"id": "vsdd4wllb0vb",
"incident_id": "bzkmpbzlmt8k",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T13:27:49.854Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled database maintenance India region",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": null,
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-13T21:45:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-13T22:45:00.000Z",
"shortlink": "https://stspg.io/l0rgl8lg5902",
"started_at": "2026-10-06T13:21:13.152Z",
"status": "scheduled",
"updated_at": "2026-10-06T13:28:04.821Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-10-06T13:30:02.248Z",
"id": "f3pyczp9czhx",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing scheduled database maintenance on each production region at the times below. \n\nEach window may take up to 1 hour.\n\nAustralia: Thu, Oct 15, 03:15 AEDT (Wed, Oct 14, 16:15 UTC)\n\n\nWhat to expect during the maintenance window:\nDashboard: Creating or updating rules, queues, and alerts will be temporarily unavailable.\nRisk rule evaluation (online API): No impact. Evaluations will continue to operate as usual.\nAlerts created via API: These will continue to be created, but you may see short delays.\n\n\nIf you have any questions, please reach out to your account manager or our support team.",
"created_at": "2026-10-06T13:30:02.336Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T13:30:02.336Z",
"id": "clc8f7kgpvmg",
"incident_id": "f3pyczp9czhx",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T13:30:02.336Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled database maintenance Australia region",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": null,
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-14T16:15:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-14T17:15:00.000Z",
"shortlink": "https://stspg.io/bn30cv9vhprw",
"started_at": "2026-10-06T13:30:02.242Z",
"status": "scheduled",
"updated_at": "2026-10-06T13:30:02.339Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-10-06T13:26:03.575Z",
"id": "nfrwmn68xjkz",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing scheduled database maintenance on each production region at the times below. \n\nEach window may take up to 1 hour.\n\nCanada: Thu Oct 15, 03:15 EDT (07:15 UTC)\n\nWhat to expect during the maintenance window:\nDashboard: Creating or updating rules, queues, and alerts will be temporarily unavailable.\nRisk rule evaluation (online API): No impact. Evaluations will continue to operate as usual.\nAlerts created via API: These will continue to be created, but you may see short delays.\n\n\nIf you have any questions, please reach out to your account manager or our support team.",
"created_at": "2026-10-06T13:26:03.661Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T13:26:03.661Z",
"id": "dr6vsb3xb9b0",
"incident_id": "nfrwmn68xjkz",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T13:26:03.661Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled database maintenance Canada region",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": null,
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-15T07:15:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-15T08:15:00.000Z",
"shortlink": "https://stspg.io/g5bdg2j9r170",
"started_at": "2026-10-06T13:26:03.568Z",
"status": "scheduled",
"updated_at": "2026-10-06T13:26:03.663Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-10-06T13:32:16.774Z",
"id": "0kcb1qgcv0mp",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing scheduled database maintenance on each production region at the times below. \n\nEach window may take up to 1 hour.\n\nEU: Sun, Oct 18, 07:00 CEST (05:00 UTC)\n\n\nWhat to expect during the maintenance window:\nDashboard: Creating or updating rules, queues, and alerts will be temporarily unavailable.\nRisk rule evaluation (online API): No impact. Evaluations will continue to operate as usual.\nAlerts created via API: These will continue to be created, but you may see short delays.\n\n\nIf you have any questions, please reach out to your account manager or our support team.",
"created_at": "2026-10-06T13:32:16.878Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T13:32:16.878Z",
"id": "xxb9plv1qvps",
"incident_id": "0kcb1qgcv0mp",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T13:32:16.878Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled database maintenance EU region",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": null,
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-18T05:00:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-18T06:00:00.000Z",
"shortlink": "https://stspg.io/wmkh51wkfbbw",
"started_at": "2026-10-06T13:32:16.766Z",
"status": "scheduled",
"updated_at": "2026-10-06T13:32:16.882Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-10-06T13:33:23.233Z",
"id": "0qn62w8jcbvl",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing scheduled database maintenance on each production region at the times below. \n\nEach window may take up to 1 hour.\n\nUS: Sat, Oct 24, 05:00 EDT (09:00 UTC)\n\n\nWhat to expect during the maintenance window:\nDashboard: Creating or updating rules, queues, and alerts will be temporarily unavailable.\nRisk rule evaluation (online API): No impact. Evaluations will continue to operate as usual.\nAlerts created via API: These will continue to be created, but you may see short delays.\n\n\nIf you have any questions, please reach out to your account manager or our support team.",
"created_at": "2026-10-06T13:33:23.320Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T13:33:23.320Z",
"id": "tgbh421cmv1t",
"incident_id": "0qn62w8jcbvl",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T13:33:23.320Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled database maintenance US region",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": null,
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-24T09:00:00.000Z",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-24T10:00:00.000Z",
"shortlink": "https://stspg.io/n5v66kqxsclx",
"started_at": "2026-10-06T13:33:23.226Z",
"status": "scheduled",
"updated_at": "2026-10-06T13:33:23.323Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:00.695Z",
"description": null,
"group_id": null,
"id": "xr048l3j0887",
"name": "Device APIs",
"page_id": "2qlwmy349kf8",
"position": 1,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.088Z"
},
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
},
{
"created_at": "2026-10-01T15:33:39.695Z",
"description": null,
"group_id": null,
"id": "sj976kyh6d71",
"name": "Business API",
"page_id": "2qlwmy349kf8",
"position": 8,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.139Z"
},
{
"created_at": "2026-10-01T15:34:09.693Z",
"description": null,
"group_id": null,
"id": "6xs7z1zzrf8v",
"name": "Feedback API",
"page_id": "2qlwmy349kf8",
"position": 9,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.163Z"
}
],
"created_at": "2026-10-01T14:49:34.000Z",
"id": "z55f0sq9phj0",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "xr048l3j0887",
"name": "Device APIs",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sj976kyh6d71",
"name": "Business API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6xs7z1zzrf8v",
"name": "Feedback API",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The elevated latency and increased response times affecting the APIs in PROD-EU have been resolved. Response times have returned to normal levels and have remained stable since the fix was applied. We will continue to monitor the environment closely.\n\nA root cause analysis (RCA) will be shared once it is complete. We apologize for the disruption.",
"created_at": "2026-10-01T21:18:30.046Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-01T19:58:30.000Z",
"id": "1011gz4524bp",
"incident_id": "z55f0sq9phj0",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-01T21:19:10.805Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xr048l3j0887",
"name": "Device APIs",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sj976kyh6d71",
"name": "Business API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6xs7z1zzrf8v",
"name": "Feedback API",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-01T17:03:05.413Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-01T17:03:05.413Z",
"id": "c57xxybs34c9",
"incident_id": "z55f0sq9phj0",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-01T17:03:05.413Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xr048l3j0887",
"name": "Device APIs",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "sj976kyh6d71",
"name": "Business API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "6xs7z1zzrf8v",
"name": "Feedback API",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A mitigation has been applied and latency is improving. We are continuing to monitor and will post another update later.",
"created_at": "2026-10-01T16:39:14.222Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-01T16:39:14.222Z",
"id": "4hzxtbqq72vc",
"incident_id": "z55f0sq9phj0",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-01T16:39:14.222Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xr048l3j0887",
"name": "Device APIs",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "sj976kyh6d71",
"name": "Business API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "6xs7z1zzrf8v",
"name": "Feedback API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-01T15:39:12.795Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-01T15:39:12.795Z",
"id": "lt7bpjt98kkm",
"incident_id": "z55f0sq9phj0",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-01T15:39:12.795Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "sj976kyh6d71",
"name": "Business API",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "6xs7z1zzrf8v",
"name": "Feedback API",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating elevated latency and increased response times affecting Customer API, Feedback API and Business API requests in our PROD-EU environment. Our engineering team is actively investigating, and we will provide an update as soon as we have more information.",
"created_at": "2026-10-01T15:36:34.737Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-01T14:49:34.000Z",
"id": "gsgz01xrw6qg",
"incident_id": "z55f0sq9phj0",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-01T15:40:15.359Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-01T17:03:05.413Z",
"name": "Elevated latency on multiple APIs in PROD-EU",
"page_id": "2qlwmy349kf8",
"postmortem_body": "## Summary\n\nOn 1 October 2026, from approximately 14:20 to 15:45 UTC, our EU environment experienced increased latency and timeouts on the `/v1/customers`, `/v1/issuing/risks`, and `/v1/feedbacks` endpoints. We added service capacity to end the main customer-impact period and subsequently deployed a code change that resolved the underlying cause.\n\n## Customer impact\n\nCustomers using the affected EU endpoints experienced slow responses, errors, and timeouts.\n\nThe verified affected endpoints were `POST /v1/customers`, `POST /v1/issuing/risks`, and `POST /v1/feedbacks`. Other measured endpoints did not show comparable latency increases. The US environment and other environments were not affected.\n\n## What happened and why\n\nUnder peak EU traffic, a service supporting these endpoints allocated memory faster than it reclaimed it. Part of this work involved unnecessary database reads and writes to calculate a crypto-recipient feature even when the request contained no crypto recipient address. The service's memory management was also not configured to account for its operating memory limit.\n\nAs instances reached that limit, they became unhealthy and restarted. Their traffic moved to the remaining instances, increasing the load and causing further restarts. The EU environment had insufficient spare capacity, and our automatic scaling settings did not add capacity quickly enough to contain the failure.\n\nOur monitoring did not give us a clear, sustained signal of the underlying problem. Some latency alerts cleared on their own and were not thoroughly investigated, and we lacked dedicated alerts for this service's memory pressure and repeated restarts. A customer report escalation brought the disruption to our attention. We also mistook a temporary improvement for recovery while requests were still affected.\n\n## Resolution\n\nAt 15:33 UTC, we increased service capacity in the EU environment. Tail latency across all three affected endpoints returned to its usual level at approximately 15:42 UTC, and service restarts had stopped by approximately 15:45 UTC, ending the main customer-impact period. This capacity increase was a mitigation rather than the root-cause fix.\n\nAt 17:59 UTC, we deployed a code change that skips unnecessary crypto-recipient database operations when no recipient address is present. Post-deployment measurements showed a 79% reduction in memory use, a 40% reduction in CPU time, and a 67% reduction in garbage-collection activity. This change eliminated more than 99% of the unnecessary database reads and writes and resolved the underlying memory-allocation cycle.\n\n## Timeline \\(UTC\\)\n\nAll events below occurred on 1 October 2026.\n\n* 14:20: Customer-impact window began.\n* 14:39: Our on-call engineer received and acknowledged a latency alert for this incident.\n* 14:44: A customer reported the disruption to us.\n* 15:03: We declared an incident and began a coordinated response.\n* 15:33: We increased service capacity.\n* 15:42: Tail latency across the affected endpoints returned to its usual level.\n* 15:45: The customer-impact window ended; service restarts had stopped and the expanded capacity was ready.\n* 17:59: We deployed the root-cause code fix to skip unnecessary database operations.\n\n## Prevention and action items\n\n* We have added capacity in the EU and raised the memory available to the service.\n* We have changed the service so it manages memory against its real limit.\n* We are correcting the automatic scaling trigger so it adds capacity before instances fail.\n* We are adding alerts for memory pressure, repeated restarts and out-of-memory terminations, and per-client latency so we can detect this type of issue ourselves and notify customers first.\n* We deployed the root-cause code fix that skips unnecessary database operations when no crypto-recipient address is present.\n* We are continuing work to reduce memory use per request for high-volume business workloads.",
"postmortem_body_last_updated_at": "2026-10-09T14:31:00.264Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-09T14:31:00.264Z",
"reminder_intervals": null,
"resolved_at": "2026-10-01T19:58:30.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/2z3f77k0d46h",
"started_at": "2026-10-01T14:49:34.000Z",
"status": "resolved",
"updated_at": "2026-10-09T14:31:00.291Z"
},
{
"components": [
{
"created_at": "2025-06-06T11:54:41.445Z",
"description": null,
"group_id": null,
"id": "1wlhwqqkmzt6",
"name": "Issuing API",
"page_id": "2qlwmy349kf8",
"position": 2,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-27T22:36:52.845Z"
},
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
}
],
"created_at": "2026-09-27T21:56:17.712Z",
"id": "7b24m3y5rl8w",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "1wlhwqqkmzt6",
"name": "Issuing API",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved. We'll follow up with post mortem.",
"created_at": "2026-09-27T22:36:52.888Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-27T22:36:52.888Z",
"id": "h8878n4bk61c",
"incident_id": "7b24m3y5rl8w",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-27T22:36:52.888Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "1wlhwqqkmzt6",
"name": "Issuing API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-09-27T22:21:56.768Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-27T22:21:56.768Z",
"id": "h2j5ljt78y9f",
"incident_id": "7b24m3y5rl8w",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-27T22:21:56.768Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "1wlhwqqkmzt6",
"name": "Issuing API",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Australia instance (api.au.sardine.ai) is degraded for v1/customers API and v1/issuing/risks API. Team is actively looking into it.",
"created_at": "2026-09-27T21:56:17.793Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-27T21:56:17.793Z",
"id": "ct76cngwhkbr",
"incident_id": "7b24m3y5rl8w",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-27T21:56:17.793Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-27T22:21:56.768Z",
"name": "Prod Australia instance performance degradation (v1/customers API and v1/issuing/risks API)",
"page_id": "2qlwmy349kf8",
"postmortem_body": "## Summary\n\nOn September 27–28, 2026, Sardine experienced two periods of elevated latency and request timeouts in our **Australian production region**. Requests to `/v1/issuing/risks` , `/v1/customers`, and `/v1/feedbacks` were affected from 12:16–22:00 UTC on September 27 and again from 23:20 UTC on September 27 until 00:23 UTC on September 28.\n\nA separate high-volume data-processing workload overloaded a shared regional database. An unbounded historical aggregation initially consumed most of the database capacity. As records accumulated, stale database statistics caused another frequently used transaction lookup to select an inefficient query plan. Query timeouts then recycled database connections, repeatedly reintroducing that plan and amplifying the load.\n\nThe incident affected API availability, transaction persistence, and the risk level evaluation.\n\n## Timeline\n\n_All times UTC._\n\n* **September 27, 12:16** — A high-volume data-processing workload begins. Database CPU reaches capacity and `/v1/issuing/risks` timeouts start.\n* **14:20** — We receive the first customer report.\n* **14:29** — The primary support on-call acknowledges a page, but the issue is not escalated effectively.\n* **18:02** — We send the first substantive customer impact update.\n* **18:05** — The need for engineering escalation is identified.\n* **21:06** — Sardine declares an incident.\n* **21:38** — Active engineering investigation and remediation begin.\n* **22:03** — A database resize and primary switchover restore customer-facing service.\n* **22:12** — The high-volume workload is paused and database utilization returns to normal.\n* **22:40** — The first event is marked resolved.\n* **22:49** — Database CPU monitors are added for all production regions.\n* **23:20** — The workload resumes and triggers a second degradation.\n* **23:41** — The new database monitor pages the on-call team and is acknowledged within 13 seconds.\n* **September 28 00:14** — A second database resize and primary switchover resolve the inefficient transaction lookup.\n* **00:23** — Optional aggregation work is disabled and the remaining customer-facing timeouts end.\n* **00:26 onward** — Database statistics are refreshed and an optimized composite index is deployed.\n\n## Root Cause\n\nThe incident resulted from the interaction of four conditions:\n\n1. **Unbounded aggregation:** A synchronous query scanned the full transaction history for given condition. This consumed most database capacity when the high-volume workload began.\n2. **Inefficient transaction lookup:** As new records accumulated, database statistics did not refresh quickly enough. The query planner selected an index that scanned a customer’s growing transaction history rather than performing a targeted lookup.\n3. **Connection recycling:** Timed-out queries caused database connections to close. Replacement connections repeatedly began with the same inefficient query plan, creating a feedback loop of expensive scans, timeouts, and reconnects.\n\n## Contributing Factors\n\n* Our API monitoring excluded the relevant client-timeout responses, and we did not alert on query cancellations or sustained database-connection churn. Database CPU monitor was only configured for non-pagerduty alert\n* The customer detected the first event before our monitoring did. Delayed engineering escalation extended the response time.\n* Our first recovery addressed the immediate pressure but did not prevent the workload from triggering a second event when it resumed.\n\n## Resolution and Recovery\n\nFor the first event, we resized the database and switched to the new primary, then paused the high-volume workload. For the second event, we again resized and switched the primary, then disabled optional aggregation to remove the remaining load.\n\nAfter service recovery, we:\n\n* Refreshed database statistics.\n* Deployed a composite index that gives both new and established connections an efficient transaction-lookup path.\n* Added database CPU monitors for every production region.\n* Removed optional, unused aggregation from the `/v1/customers` path\n* Kept the Australian production database at a higher baseline capacity while we complete the permanent query fixes and capacity review.\n\n## Corrective Actions\n\nWe are taking the following actions:\n\n### Database and application resilience\n\n* Replace the affected transaction lookup with an atomic, consistently indexed operation.\n* Skip unnecessary historical aggregation and bound any required fallback query.\n* Evaluate a read pool so issuing-risk reads do not compete with customer-event writes on the primary.\n* Review steady-state database capacity after the query fixes are complete.\n\n### Monitoring and response\n\n* Expand monitoring for client timeouts and database metrics\n* Improve incident routing so customer reports, support tickets, and on-call escalation converge quickly.",
"postmortem_body_last_updated_at": "2026-09-29T21:43:25.658Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-29T21:43:25.658Z",
"reminder_intervals": null,
"resolved_at": "2026-09-27T22:36:52.888Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/5dj4wswvfj9h",
"started_at": "2026-09-27T21:56:17.704Z",
"status": "resolved",
"updated_at": "2026-09-29T21:43:25.683Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-09-17T08:12:39.932Z",
"id": "r8twmrs2svc4",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-09-17T11:25:42.600Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-17T11:25:42.600Z",
"id": "6fxwbdlclhhk",
"incident_id": "r8twmrs2svc4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-17T11:25:42.600Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Severity: Minor\n\nStatus: Identified / Investigating\n\nAffected Component: Connection Graph (Production - EU)\n\nImpact:\nUsers may be unable to view Connection Graph data for the past three days. Graphs generated during this window are not currently visible. No other functionality is affected.\n\nDescription:\nWe have identified a delay in the Connection Graph pipeline in our EU production environment, which began approximately three days ago. As a result, graph data from this period is not currently reflected. We are actively investigating the root cause and working on a resolution. We will post updates here as more information becomes available.",
"created_at": "2026-09-17T08:12:40.049Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-17T08:12:40.049Z",
"id": "fl6663p6glqy",
"incident_id": "r8twmrs2svc4",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-17T08:12:40.049Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Connection Graph Pipeline Delay – Prod EU",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-17T11:25:42.600Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/957r59g3rhpf",
"started_at": "2026-09-17T08:12:39.920Z",
"status": "resolved",
"updated_at": "2026-09-17T11:25:42.614Z"
},
{
"components": [
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
},
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-09-16T12:08:14.119Z",
"id": "jkhdy900m48d",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-09-16T15:19:03.084Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T15:19:03.084Z",
"id": "t3cnzz8mhzql",
"incident_id": "jkhdy900m48d",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T15:19:03.084Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "degraded_performance"
},
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-09-16T12:08:55.507Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T12:08:55.507Z",
"id": "7tnf2rrqby9g",
"incident_id": "jkhdy900m48d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T12:08:55.507Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The Sandbox EU environment is experiencing degraded performance. We are working on resolution.",
"created_at": "2026-09-16T12:08:14.199Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-16T12:08:14.199Z",
"id": "vb6c58tbkdf0",
"incident_id": "jkhdy900m48d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-16T12:08:14.199Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Sandbox EU degradation",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-16T15:19:03.084Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/l515v781sgrn",
"started_at": "2026-09-16T12:08:14.111Z",
"status": "resolved",
"updated_at": "2026-09-16T15:19:03.100Z"
},
{
"components": [
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
}
],
"created_at": "2026-09-04T22:19:23.000Z",
"id": "mjk9b3rz4w2h",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "The incident is now resolved. We'll provide a post mortem in the future explaining the details.",
"created_at": "2026-09-05T03:50:18.645Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-05T03:50:18.000Z",
"id": "hcslm5nsz8ck",
"incident_id": "mjk9b3rz4w2h",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-05T17:18:57.760Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The fix is being validated and being rolled out.",
"created_at": "2026-09-05T03:15:58.339Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-05T03:15:58.000Z",
"id": "7lwdm9cr2w42",
"incident_id": "mjk9b3rz4w2h",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-05T17:18:07.281Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Our team is actively looking into it. Issue is partially mitigated, and we're still working on fixing the root cause.",
"created_at": "2026-09-05T01:49:40.554Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-05T01:49:40.000Z",
"id": "sy6fy6g7g4vk",
"incident_id": "mjk9b3rz4w2h",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-05T17:17:39.580Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have identified the root cause of the issue affecting Advanced Aggregations and our engineering team is actively working on a fix. We will provide a further update once the fix has been deployed and verified.",
"created_at": "2026-09-05T00:38:29.224Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-09-05T00:38:29.224Z",
"id": "4xx7wfn3y2kk",
"incident_id": "mjk9b3rz4w2h",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-05T00:38:29.224Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue",
"created_at": "2026-09-04T23:11:48.876Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-09-04T23:11:48.876Z",
"id": "pg4zp1mjsft6",
"incident_id": "mjk9b3rz4w2h",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-04T23:11:48.876Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating an issue affecting Advanced Aggregations, where some customers may experience inaccurate results (missing recent transaction from aggregation)",
"created_at": "2026-09-04T22:19:23.881Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-09-04T22:19:23.000Z",
"id": "gvjsk67n5r6h",
"incident_id": "mjk9b3rz4w2h",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-04T22:27:26.329Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-09-05T03:15:58.000Z",
"name": "Advanced Aggregations — Degraded Performance",
"page_id": "2qlwmy349kf8",
"postmortem_body": "### Summary\n\nOn September 4–5, 2026 \\(UTC\\), we experienced an issue affecting **Advanced Aggregations** **in which some customers saw inaccurate aggregation results because recent card transaction data may have been missing**. We fixed **the issue, and backfilled the data**. Advanced Aggregations results are expected to be correct again after **2026-09-05 03:50 UTC**.\n\n### What happened\n\nA subset of card transaction records failed to be written into one of our tables because the column used to store the payment method id could not support the size of newly generated card IDs. As a result, those specific transactions were missing from the aggregated dataset, which could lead to incorrect results for customers ingesting this type of data. An impact assessment is available upon request.\n\nNo data was permanently lost: failed writes were routed to a durable deadletter queue for later reprocessing. After mitigation was deployed, we replayed the queued events and completed a backfill so that aggregations reflected the correct underlying transaction data. Incident started at 2026-09-04 01:02 UTC and resolved at 2026-09-05 03:50 UTC. \n\n### Why it happened\n\nThe root cause was a **data type mismatch**: the `cards` table `id` values grew beyond the maximum value supported by a **4-byte integer**. `id` field has 8-byte integer so there was no error storing cards. However, `card_id` field in the Advanced Aggregations schema was 4-byte, causing inserts to fail for transactions related to newer cards.\n\nTo remediate safely at scale, we:\n\n* Added new **8-byte integer \\(**`bigint`\\) columns for the affected identifiers and updated write logic to use them.\n* Introduced a **compatibility read path** so reads return the correct identifier whether data is stored in the old or new columns, along with new indexes to preserve query performance.\n\n### What we are doing about it\n\n* We’re reviewing improvements to monitor schema limit risks earlier and prevent similar issues in the future.\n* We’ll enhance monitoring on deadletter queue so we can detect issue like this sooner.",
"postmortem_body_last_updated_at": "2026-09-08T22:48:54.480Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-09-08T22:48:54.480Z",
"reminder_intervals": null,
"resolved_at": "2026-09-05T03:50:18.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/x3jc4n8zgnw0",
"started_at": "2026-09-04T22:19:23.000Z",
"status": "resolved",
"updated_at": "2026-09-08T22:48:54.499Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-07-24T00:42:27.633Z",
"id": "hc7fh8z7hvv6",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-07-24T02:07:50.584Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-24T02:07:50.584Z",
"id": "2rfs0nc9v5zp",
"incident_id": "hc7fh8z7hvv6",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-24T02:07:50.584Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "New sessions are being shown OK, a few older ones might still be missing, they'll be shown shortly.",
"created_at": "2026-07-24T01:41:39.563Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-24T01:41:39.563Z",
"id": "zxs6fk368t0l",
"incident_id": "hc7fh8z7hvv6",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-24T01:41:39.563Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-07-24T01:19:48.433Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-24T01:19:48.433Z",
"id": "jvnnzzrg9hxg",
"incident_id": "hc7fh8z7hvv6",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-24T01:19:48.433Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-07-24T00:58:01.429Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-24T00:58:01.429Z",
"id": "yqy17ygfjbml",
"incident_id": "hc7fh8z7hvv6",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-24T00:58:01.429Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-07-24T00:42:27.719Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-24T00:42:27.719Z",
"id": "15x6dnxz88v9",
"incident_id": "hc7fh8z7hvv6",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-24T00:42:27.719Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-24T01:19:48.433Z",
"name": "Customer Intelligence tab in US Production Dashboard not showing sessions/transactions",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-07-24T02:07:50.584Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/00zbwqkk7l5b",
"started_at": "2026-07-24T00:42:27.626Z",
"status": "resolved",
"updated_at": "2026-07-24T02:07:50.603Z"
},
{
"components": [
{
"created_at": "2021-06-24T20:27:09.652Z",
"description": null,
"group_id": null,
"id": "y8n5vc01jy94",
"name": "Dashboard",
"page_id": "2qlwmy349kf8",
"position": 4,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-17T11:25:42.569Z"
}
],
"created_at": "2026-07-16T12:40:01.380Z",
"id": "kbptphf5hcjn",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "The issue has been resolved.",
"created_at": "2026-07-16T14:25:38.666Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-16T14:25:38.666Z",
"id": "7l16qrzlvj7l",
"incident_id": "kbptphf5hcjn",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T14:25:38.666Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "partial_outage"
}
],
"body": "We are continuing to monitor for any further issues.",
"created_at": "2026-07-16T13:13:42.242Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-16T13:13:42.242Z",
"id": "9vsp8dlkdfzr",
"incident_id": "kbptphf5hcjn",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T13:13:42.242Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A fix has been implemented and results are being displayed in the Customer Intelligence page. We are monitoring the process until full service is restored.",
"created_at": "2026-07-16T13:13:29.129Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-16T13:13:29.129Z",
"id": "n0pgbyn2lb33",
"incident_id": "kbptphf5hcjn",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T13:13:29.129Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-07-16T12:41:52.276Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-16T12:41:52.276Z",
"id": "r5wphhtxtk5k",
"incident_id": "kbptphf5hcjn",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T12:41:52.276Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2026-07-16T12:41:17.192Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-16T12:41:17.192Z",
"id": "8jc455t4pwwj",
"incident_id": "kbptphf5hcjn",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T12:41:17.192Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-07-16T12:40:29.322Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-16T12:40:29.322Z",
"id": "k27rfhqhdr4b",
"incident_id": "kbptphf5hcjn",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T12:40:29.322Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "y8n5vc01jy94",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are currently experiencing degraded performance in retrieving the latest customer sessions in the Customer Intelligence page. Root cause has been determined. We are working on resolution.",
"created_at": "2026-07-16T12:40:01.440Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-16T12:40:01.440Z",
"id": "tyfvf9jvykv0",
"incident_id": "kbptphf5hcjn",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T12:40:01.440Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-07-16T13:13:29.129Z",
"name": "Degraded Performance for EU Dashboard",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": "[]",
"resolved_at": "2026-07-16T14:25:38.666Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/wnrp12glc2cw",
"started_at": "2026-07-16T12:40:01.371Z",
"status": "resolved",
"updated_at": "2026-07-16T14:25:38.686Z"
},
{
"components": [
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
}
],
"created_at": "2026-07-15T14:34:04.349Z",
"id": "3019vlcvh9tk",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved. A Postmortem will be published soon.",
"created_at": "2026-07-16T02:36:08.916Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-15T19:56:08.000Z",
"id": "h20cvj88lssq",
"incident_id": "3019vlcvh9tk",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T02:38:27.332Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-07-15T14:34:04.411Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-15T14:34:04.411Z",
"id": "6172cj7lxqr5",
"incident_id": "3019vlcvh9tk",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-15T14:34:04.411Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Performance degradation on APIs",
"page_id": "2qlwmy349kf8",
"postmortem_body": "**Summary** \nOn 2026-07-15, API requests experienced elevated error rates and increased tail latency \\(p99\\). Median latency \\(p50\\) was not affected and there was little effect on 95th percentile latency \\(p95\\), so most requests performed normally. We began investigating at 14:34 UTC and applied a fix at approximately 19:56 UTC, after which performance returned to normal.\n\n**What happened?** \nThrough the affected period, a portion of API requests returned errors, responded with reason code SITO \\(Sardine internal timeout\\) or timed out rather than completing, with intermittent spikes during which a larger share of requests were affected.\n\n**Why did it happen?** \nA scaling configuration prevented part of our platform from adding capacity as traffic increased. As traffic rose through the day, available capacity was exhausted, causing timeouts and errors for some requests.\n\n**What are we doing about this?** \nWe corrected the configuration at **19:56 UTC**, which immediately restored normal performance. We are also making the affected part of the platform more resilient to sudden traffic increases and adding proactive monitoring so we can detect and respond to this class of issue faster in future. We apologize for the disruption.",
"postmortem_body_last_updated_at": "2026-07-16T02:50:54.695Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-07-16T02:50:54.695Z",
"reminder_intervals": "[]",
"resolved_at": "2026-07-15T19:56:08.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/fwxbjd1bpcfh",
"started_at": "2026-07-15T19:56:08.000Z",
"status": "resolved",
"updated_at": "2026-07-16T02:50:54.711Z"
},
{
"components": [
{
"created_at": "2025-06-06T11:54:41.445Z",
"description": null,
"group_id": null,
"id": "1wlhwqqkmzt6",
"name": "Issuing API",
"page_id": "2qlwmy349kf8",
"position": 2,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-09-27T22:36:52.845Z"
},
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
}
],
"created_at": "2026-06-30T13:58:51.435Z",
"id": "s4y3y57spn60",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "1wlhwqqkmzt6",
"name": "Issuing API",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-06-30T19:41:12.609Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-30T19:41:12.609Z",
"id": "prh66jrscqfs",
"incident_id": "s4y3y57spn60",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-30T19:41:12.609Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "1wlhwqqkmzt6",
"name": "Issuing API",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-06-30T13:58:51.527Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-30T13:58:51.527Z",
"id": "8kdjkr3blytz",
"incident_id": "s4y3y57spn60",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-30T13:58:51.527Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Performance degradation on APIs",
"page_id": "2qlwmy349kf8",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": "[]",
"resolved_at": "2026-06-30T19:41:12.609Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8g676p1d0sx1",
"started_at": "2026-06-30T13:58:51.426Z",
"status": "resolved",
"updated_at": "2026-06-30T19:41:12.626Z"
},
{
"components": [
{
"created_at": "2021-09-20T22:15:17.371Z",
"description": null,
"group_id": null,
"id": "nfgcl24mlv3m",
"name": "Customer APIs",
"page_id": "2qlwmy349kf8",
"position": 3,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-01T16:39:14.115Z"
}
],
"created_at": "2026-06-25T15:20:35.000Z",
"id": "457lqdvz4f4g",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "The incident regarding the latency increased has been resolved. \n\nIt started at 15:20 UTC and lasting until 16:23 UTC. \n\nWe'll include a postmortem into this status page in the future.",
"created_at": "2026-06-25T18:13:43.824Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-25T16:23:43.000Z",
"id": "mwy1bwr2txy3",
"incident_id": "457lqdvz4f4g",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-25T18:14:47.542Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "nfgcl24mlv3m",
"name": "Customer APIs",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-06-25T15:56:35.571Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-06-25T15:20:35.000Z",
"id": "s2w1zkwt9n9y",
"incident_id": "457lqdvz4f4g",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-06-25T18:12:06.297Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Increased latency in PROD US instance API",
"page_id": "2qlwmy349kf8",
"postmortem_body": "### Summary\n\nDue to a database performance issue, API calls to `/v1/customer` , `/v1/issuing/risks` and `/v1/feedbacks` in June 25th faced increased latency.\n\n### What happened\n\nOne of the database we use to load certain customer and transaction data experienced slow latency for queries used in our customer and feedback APIs, causing latency issues on those APIs from 15:20 UTC to 16:23 UTC at June 25th. We've identified the root cause and are working on fixes to mitigate the issue.\n\n### Why it happened\n\nOne of our clients has very high volume and a particular usage pattern that didn't match well with our current database design for this data, causing the entire cluster to degrade in performance.\n\n### What We’re Doing About It\n\nWe are actively working on a solution to this issue, which includes moving the affected customer/transaction data to more performant databases, eliminating the risk altogether. In parallel we are also looking into actions to gracefully handle the usage patterns that cause degradation in the entire cluster.",
"postmortem_body_last_updated_at": "2026-06-30T18:51:31.903Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-06-30T18:51:31.903Z",
"reminder_intervals": "[]",
"resolved_at": "2026-06-25T16:23:43.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/2w49ptds19zr",
"started_at": "2026-06-25T15:20:35.000Z",
"status": "resolved",
"updated_at": "2026-06-30T18:51:31.923Z"
}
],
"page": {
"id": "2qlwmy349kf8",
"name": "Sardine AI",
"url": "https://status.sardine.ai"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}