Snapshot 64890
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"data": {
"attributes": {
"aggregate_state": "operational",
"announcement": null,
"company_name": "Nango",
"company_url": "https://nango.dev",
"contact_url": "https://nango.dev/slack",
"created_at": "2023-11-21T14:45:12.143Z",
"custom_domain": "status.nango.dev",
"logo_remote_url": null,
"logo_url": "https://d1lppblt9t2x15.cloudfront.net/logos/ce52849abb5a99f83f70bd12ff491ed0.png",
"subdomain": "nango",
"timezone": "Paris",
"updated_at": "2026-10-08T13:18:08.769Z"
},
"id": "174740",
"relationships": {
"resources": {
"data": [
{
"id": "8335202",
"type": "status_page_resource"
}
]
},
"sections": {
"data": [
{
"id": "189318",
"type": "status_page_section"
}
]
},
"status_reports": {
"data": [
{
"id": "1080253",
"type": "status_report"
},
{
"id": "1062328",
"type": "status_report"
},
{
"id": "1032965",
"type": "status_report"
},
{
"id": "994460",
"type": "status_report"
}
]
}
},
"type": "status_page"
},
"included": [
{
"attributes": {
"name": "",
"position": 0
},
"id": "189318",
"type": "status_page_section"
},
{
"attributes": {
"availability": 1,
"explanation": "Based on monitoring of server/database latency \u0026 syncs/actions/webhooks execution.",
"position": 0,
"public_name": "Nango Cloud Health",
"resource_id": 1574458,
"resource_type": "Monitor",
"status": "operational",
"status_history": [
{
"day": "2026-07-11",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-12",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-13",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-14",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-15",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-16",
"downtime_duration": 8100,
"maintenance_duration": 0,
"status": "degraded"
},
{
"day": "2026-07-17",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-18",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-19",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-20",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-21",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-22",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-23",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-24",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-25",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-26",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-27",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-28",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-29",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-30",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-07-31",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-01",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-02",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-03",
"downtime_duration": 5160,
"maintenance_duration": 0,
"status": "degraded"
},
{
"day": "2026-08-04",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-05",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-06",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-07",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-08",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-09",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-10",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-11",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-12",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-13",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-14",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-15",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-16",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-17",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-18",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-19",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-20",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-21",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-22",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-23",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-24",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-25",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-26",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-27",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-28",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-29",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-30",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-08-31",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-01",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-02",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-03",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-04",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-05",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-06",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-07",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-08",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-09",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-10",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-11",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-12",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-13",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-14",
"downtime_duration": 12900,
"maintenance_duration": 0,
"status": "degraded"
},
{
"day": "2026-09-15",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-16",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-17",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-18",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-19",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-20",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-21",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-22",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-23",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-24",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-25",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-26",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-27",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-28",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-29",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-30",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-10-01",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-10-02",
"downtime_duration": 1620,
"maintenance_duration": 0,
"status": "degraded"
},
{
"day": "2026-10-03",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-10-04",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-10-05",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-10-06",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-10-07",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-10-08",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
}
],
"status_page_section_id": 189318
},
"id": "8335202",
"type": "status_page_resource"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8335202"
}
],
"message": "We are seeing some recovery in task processing.",
"notify_subscribers": true,
"published_at": "2026-10-02T17:58:00.000Z",
"published_at_timezone": "Paris"
},
"id": "5864535",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"message": "This issue has been mitigated. We are now investigating the root cause and will update here when complete.",
"notify_subscribers": true,
"published_at": "2026-10-02T18:00:00.000Z",
"published_at_timezone": "Paris"
},
"id": "5864547",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8335202"
}
],
"message": "Actions and syncs are failing to execute. We are investigating and will update here as soon as we have more information.",
"notify_subscribers": true,
"published_at": "2026-10-02T17:33:00.000Z",
"published_at_timezone": "Paris"
},
"id": "5864471",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8335202"
}
],
"message": "We are still investigating. All nango tasks (actions, syncs, and webhooks) are currently not running.",
"notify_subscribers": true,
"published_at": "2026-10-02T17:54:00.000Z",
"published_at_timezone": "Paris"
},
"id": "5864527",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"message": "POSTMORTEM:\n\nA recent change caused the execution of all functions (actions, webhooks, syncs) to stop on Friday October 2nd between 17:22 and 17:54 UTC.\n\nWe identified a performance regression introduced by a recent query refactor in our task scheduler. The change caused one database query to scale poorly when the number of fresh execution increased significantly.\nUnder normal load, the query remained fast, which is why the issue was not immediately visible after deployment. 8 hours later, a spike in queued tasks triggered the inefficient execution path, increasing database work and causing the dequeuing query to time out.\n\nWe have since corrected the query and are adding additional safeguards, including additional monitoring to reduce the likelihood of similar regressions in the future.",
"notify_subscribers": true,
"published_at": "2026-10-06T09:07:00.000Z",
"published_at_timezone": "Paris"
},
"id": "5879012",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-10-02T17:33:00.000Z",
"title": "Action and syncs are not currently running"
},
"id": "1080253",
"relationships": {
"status_updates": {
"data": [
{
"id": "5864535",
"type": "status_update"
},
{
"id": "5864547",
"type": "status_update"
},
{
"id": "5864471",
"type": "status_update"
},
{
"id": "5864527",
"type": "status_update"
},
{
"id": "5879012",
"type": "status_update"
}
]
}
},
"type": "status_report"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"message": "We have mitigated the issue and will continue observing the service behavior.",
"notify_subscribers": true,
"published_at": "2026-09-14T02:35:00.000Z",
"published_at_timezone": "Brasilia"
},
"id": "5775733",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8335202"
}
],
"message": "Some syncs may be failing to persist records. We are investigating the root cause.",
"notify_subscribers": true,
"published_at": "2026-09-13T23:00:00.000Z",
"published_at_timezone": "Amsterdam"
},
"id": "5775574",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"message": "This issue has been resolved",
"notify_subscribers": true,
"published_at": "2026-09-14T03:02:00.000Z",
"published_at_timezone": "Paris"
},
"id": "5776571",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"message": "## Degraded record sync performance post-mortem\n\n### Post-incident summary\n\n**Date:** 12–14 September 2026\n\n**Impact:** Slower record syncs, with \u003e1% failing on timeout and retrying. Most syncs completed successfully.\n\n**Status:** Resolved\n\n#### Summary\n\nOur records database experienced progressive performance degradation over a period of roughly two days. Customers experienced slower sync runs, and a subset (\u003e1%) of syncs failed with timeout errors and had to retry.\n\nMost syncs continued to complete successfully throughout. Customers syncing large record volumes were significantly more likely to see failures, and some experienced repeated failures on specific syncs over a period of hours. Other parts of the system were not affected.\n\nNo data was lost.\n\n#### Timeline\n\nIssue began: 12 September, 16:00 UTC\nIssue detected: 13 September, 23:05 UTC\nMitigated: 14 September, 02:30 UTC\nFixed: 17 September, 16:49 UTC\n\n#### Root cause\n\nA background job removes stored records for connections that have gone inactive. Because of a change made earlier this year, it was performing all of its work as a single, long-running database operation rather than in small batches.\n\nWhile it ran, the database could not perform its routine internal cleanup, which reclaims obsolete data and can only run once no active operation still depends on it. Obsolete rows accumulated, so every query had to read progressively more to return the same results. As read costs rose, write operations began exceeding their time limit and failing — and because failed syncs retry, the degradation compounded.\n\n#### Resolution\n\nWe restored the batching behavior to the cleanup job, so it now processes a bounded amount of data at a time and releases the database between batches.\n\nThe database's internal cleanup resumed immediately. Read performance returned to its normal baseline the same day and has remained stable since. Syncs that failed during the incident recovered automatically on their next run; no customer action was required.\n\n#### Next steps\n\n* Monitoring: The condition behind this incident was visible on our health dashboards throughout, but we had no monitor configured to alert on it. We are closing this gap with new monitors.",
"notify_subscribers": true,
"published_at": "2026-09-23T19:48:00.000Z",
"published_at_timezone": "Brasilia"
},
"id": "5823173",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-09-13T23:00:00.000Z",
"title": "Records database is experiencing degraded performance"
},
"id": "1062328",
"relationships": {
"status_updates": {
"data": [
{
"id": "5775733",
"type": "status_update"
},
{
"id": "5775574",
"type": "status_update"
},
{
"id": "5776571",
"type": "status_update"
},
{
"id": "5823173",
"type": "status_update"
}
]
}
},
"type": "status_report"
},
{
"attributes": {
"affected_resources": [],
"message": "We will be performing a database upgrade. During this maintenance window, actions, syncs, and webhook task processing may see brief delays or disruptions. We don't expect any availability issues and the maintenance should be done within a few minutes.",
"notify_subscribers": true,
"published_at": "2026-08-28T08:30:34.000Z",
"published_at_timezone": null
},
"id": "5653507",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [],
"message": "Maintenance ended",
"notify_subscribers": true,
"published_at": "2026-08-28T08:45:34.000Z",
"published_at_timezone": null
},
"id": "5667557",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [],
"aggregate_state": "resolved",
"ends_at": "2026-08-28T08:45:34.000Z",
"report_type": "maintenance",
"starts_at": "2026-08-28T08:30:34.000Z",
"title": "Scheduled Maintenance - Database Upgrade"
},
"id": "1032965",
"relationships": {
"status_updates": {
"data": [
{
"id": "5653507",
"type": "status_update"
},
{
"id": "5667557",
"type": "status_update"
}
]
}
},
"type": "status_report"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"message": "# 2026-08-03 Missing logs\n\n## Summary\n\nOn August 3, 2026, Nango's log ingestion pipeline began rejecting writes because the index for that day had already transitioned to a read-only state.\n\nOur investigation indicates that several Connect UI events were received from a browser whose system clock was approximately six days ahead. We accepted the client-provided timestamp without validation and used it to select the storage index, causing the August 3 index to be created prematurely. After 48 hours, our lifecycle policy moved the index to the warm tier, shrank it, and made it read-only—before August 3 had arrived.\n\nCustomer-facing function executions were not affected. We mitigated the incident by re-enabling writes to the affected index.\n\n## Customer impact\n\n- **What:** Nango logs could not be written while the affected index was read-only. Function executions and other customer workloads were not affected.\n- **When:** August 3, 2026, from approximately 00:00 UTC to 01:26 UTC.\n- **Data integrity:** All log events received during this period were lost.\n\n## What happened\n\nLog indices are selected using the `createdAt` timestamp attached to each event.\n\nOn July 28, a Connect UI session from a Windows/Chrome client emitted several events. The client’s clock appears to have been approximately six days ahead, so these events carried August 3 timestamps. The telemetry endpoint accepted the client-supplied timestamp and used it directly as `createdAt`. The first event therefore created the August 3 index on July 28.\n\nOur index lifecycle policy moves indices to the warm tier after 48 hours, where they are shrunk and made read-only. Around July 30, the policy applied this transition based on the index’s actual age.\n\nWhen normal August 3 traffic began writing to the same index, those writes were rejected because the index had already been shrunk and locked.\n\nThe lifecycle policy behaved as configured. The underlying issue was that an unvalidated client timestamp could control index selection and create a future-dated index.\n\n## Resolution\n\nWe restored log ingestion by re-enabling writes to the shrunk index. \n\n## What we’re doing to prevent recurrence\n\n- **Validate clock skew:** We will normalize client-provided timestamps that fall unreasonably far in the past or future.\n\nWe apologize for the disruption. If you have questions about how this affected you, please reach out.\n",
"notify_subscribers": false,
"published_at": "2026-08-03T01:26:00.000Z",
"published_at_timezone": "UTC"
},
"id": "5521527",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8335202"
}
],
"message": "Nango logs are not being ingested",
"notify_subscribers": false,
"published_at": "2026-08-03T00:00:00.000Z",
"published_at_timezone": "UTC"
},
"id": "5521516",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8335202"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-08-03T00:00:00.000Z",
"title": "Missing logs"
},
"id": "994460",
"relationships": {
"status_updates": {
"data": [
{
"id": "5521527",
"type": "status_update"
},
{
"id": "5521516",
"type": "status_update"
}
]
}
},
"type": "status_report"
}
]
}