Snapshot 174505
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"created_at": "2015-08-18T18:37:03.144Z",
"description": null,
"group_id": null,
"id": "430q802wdbyz",
"name": "Upload API",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-19T17:31:49.875Z"
},
{
"created_at": "2021-06-01T07:34:33.078Z",
"description": "The service sends real-time updates about uploads.",
"group_id": null,
"id": "kf5n2d80ls0j",
"name": "Webhooks",
"page_id": "ltnt1pzqrlmg",
"position": 2,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-12-11T11:00:21.570Z"
},
{
"created_at": "2015-08-18T18:37:03.158Z",
"description": null,
"group_id": null,
"id": "73mzhqs4mzy4",
"name": "REST API",
"page_id": "ltnt1pzqrlmg",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-18T08:53:10.218Z"
},
{
"created_at": "2016-08-30T14:15:44.762Z",
"description": null,
"group_id": null,
"id": "sv5vvph3q2jz",
"name": "CDN",
"page_id": "ltnt1pzqrlmg",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T09:51:24.639Z"
},
{
"components": [
"qt36c1nzm6lb",
"lc2xq7nlktyn"
],
"created_at": "2019-08-08T07:59:15.553Z",
"description": null,
"group_id": null,
"id": "9bs61hmlspyq",
"name": "Processing engines",
"page_id": "ltnt1pzqrlmg",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2021-06-01T07:34:55.618Z"
},
{
"created_at": "2019-11-04T18:39:00.802Z",
"description": null,
"group_id": null,
"id": "j2904rh7dsb6",
"name": "Dashboard",
"page_id": "ltnt1pzqrlmg",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.618Z"
},
{
"created_at": "2019-11-04T18:39:15.441Z",
"description": null,
"group_id": null,
"id": "z32z0zbgd3p8",
"name": "Website",
"page_id": "ltnt1pzqrlmg",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.641Z"
}
],
"incidents": [
{
"components": [
{
"created_at": "2019-11-04T18:39:00.802Z",
"description": null,
"group_id": null,
"id": "j2904rh7dsb6",
"name": "Dashboard",
"page_id": "ltnt1pzqrlmg",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.618Z"
},
{
"created_at": "2019-11-04T18:39:15.441Z",
"description": null,
"group_id": null,
"id": "z32z0zbgd3p8",
"name": "Website",
"page_id": "ltnt1pzqrlmg",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.641Z"
}
],
"created_at": "2026-07-16T08:42:22.909Z",
"id": "jyg5gc8d24wb",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "z32z0zbgd3p8",
"name": "Website",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "The issue affecting Website and Dashboard has been resolved as of 11:30 UTC. Users should no longer experience the issue. Our team is continuing to monitor performance to ensure stability.\nWe apologize for any disruption this issue may have caused and appreciate your understanding.\nThank you for your patience, and please reach out to support if you notice anything unusual.",
"created_at": "2026-07-16T12:51:14.675Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-16T11:30:14.000Z",
"id": "n2mlr2g5x0zf",
"incident_id": "jyg5gc8d24wb",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T12:51:37.232Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "z32z0zbgd3p8",
"name": "Website",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Our team is working to implement a permanent solution. Some users may still experience connectivity issues during this time.\nWe will continue to share updates as we make progress toward full resolution.",
"created_at": "2026-07-16T11:11:44.165Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-07-16T11:11:44.165Z",
"id": "yrzx9x8mtr1f",
"incident_id": "jyg5gc8d24wb",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T11:11:44.165Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "partial_outage"
},
{
"code": "z32z0zbgd3p8",
"name": "Website",
"new_status": "degraded_performance",
"old_status": "partial_outage"
}
],
"body": "Our team has identified the cause of the issue impacting Website and Dashboard and is actively working to implement a permanent solution. Some users may still experience connectivity issues during this time.\nWe will continue to share updates as we make progress toward full resolution.\nThank you for your continued patience.",
"created_at": "2026-07-16T08:52:58.247Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-16T08:52:58.247Z",
"id": "l830pwj0r0bs",
"incident_id": "jyg5gc8d24wb",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T08:52:58.247Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "z32z0zbgd3p8",
"name": "Website",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating an issue affecting Website and Dashboard. Users may experience an increased number of errors. Our team is working to identify the root cause, and we will provide an update as soon as more information becomes available.\nWe apologize for any inconvenience and appreciate your patience.",
"created_at": "2026-07-16T08:42:23.021Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-07-16T08:42:23.021Z",
"id": "qzz1yvr445sf",
"incident_id": "jyg5gc8d24wb",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-07-16T08:42:23.021Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Website and Dashboard outage",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": "On July 16, 2026, between 07:55 UTC and 11:15 UTC \\(approximately 3 hours and 20 minutes\\), Uploadcare’s website and customer portal were partially unavailable. During this window, affected requests could fail at our CloudFront distributions with 504 Gateway Timeout errors and never reached our backend services.\n\nThe disruption was caused by a global Amazon Web Services \\(AWS\\) CloudFront outage affecting VPC Origins, the origin type we rely on for website and customer portal. Requests routed through VPC Origins failed, while CloudFront distributions using other origin types were unaffected. AWS later attributed the outage to an internal capacity constraint in the fleet that manages connections to private VPC origins, which caused routing configuration to be distributed incorrectly to its network processors.\n\nImportantly, our core platform services — including file uploading, storage, processing, and delivery of already-cached files — were not affected by this incident and continued to operate normally throughout.\n\nOur follow-up work focuses on keeping a proven, ready-to-deploy fallback for this class of AWS CloudFront failure.\n\n# **Timeline of events**\n\n_All times are in UTC on July 16, 2026._\n\n* **07:55 —** CloudFront metrics begin showing elevated error rates for our website and webclient distributions.\n* **07:59 —** Our monitoring alerts that [uploadcare.com](http://uploadcare.com) is unreachable. Our engineering team begins investigating immediately.\n* **08:02 —** We confirm 504 errors for [uploadcare.com](http://uploadcare.com) and observe that requests are not reaching our backend. Other our CloudFront distributions remain healthy.\n* **08:07 —** Based on the error pattern and the CloudFront-generated 504 error page, CloudFront becomes our primary suspected root cause. At this point AWS had not yet posted any notice, though the wider community had begun reporting CloudFront problems.\n* **08:28 —** We confirm the issue is affecting our public websites and customer portal.\n* **08:42 —** CloudFront metrics show an error rate of approximately 30%.\n* **08:44 —** AWS acknowledges a global CloudFront outage related to VPC Origins — approximately 49 minutes after our impact began.\n* **09:23 —** We begin implementing a fallback: switching affectted origins from VPC Origins to internet-facing Application Load Balancers \\(ALBs\\).\n* **10:58 —** The fallback is deployed to our staging environment for validation.\n* **11:02 —** The fallback passes validation on staging. We begin rolling the same changes toward production.\n* **11:15 —** AWS resolves the underlying outage. Our production websites and customer portall fully recover and error rates return to zero. Because AWS recovered first, the production fallback was not required. We declare the incident resolved.\n\n# **What went well**\n\n* **Fast detection and diagnosis.** Our monitoring detected the failure within minutes, and our team correlated it with a broader AWS issue and identified the likely root cause before AWS acknowledged the outage publicly.\n* **A validated mitigation.** During the incident we designed, implemented, and validated a fallback — switching CloudFront origins from VPC Origins to internet-facing ALBs — on our staging environment. AWS recovered before we needed to apply it to production, but this is now a proven mitigation for future VPC Origins incidents.\n* **Contained impact.** Because the failure was limited to distributions using VPC Origins, our core platform — file uploads, storage, processing, and cached delivery — remained fully operational.\n\n# **What went wrong**\n\n* **A shared origin dependency.** Our public website and customer portal distributions all relied on CloudFront VPC Origins, so a single AWS subsystem failure affected them together with no failover.\n\n# **Action items**\n\n* **Keep a ready-to-deploy CloudFront fallback.** We have prepared and validated infrastructure changes to switch affected distributions from VPC Origins to internet-facing ALBs, so this mitigation can be applied quickly should a similar AWS outage recur.\n\nWe sincerely apologize for the disruption this incident caused, and for the delay in communicating it through our status page. While the root cause was an AWS-side outage outside our direct control, we are committed to reducing our exposure to this class of failure and to communicating more quickly and transparently in the future.",
"postmortem_body_last_updated_at": "2026-07-21T10:52:56.396Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-07-21T10:52:56.396Z",
"reminder_intervals": null,
"resolved_at": "2026-07-16T11:30:14.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/2vjt1qhhv8hx",
"started_at": "2026-07-16T08:42:22.897Z",
"status": "resolved",
"updated_at": "2026-07-21T10:52:56.412Z"
},
{
"components": [
{
"created_at": "2015-08-18T18:37:03.144Z",
"description": null,
"group_id": null,
"id": "430q802wdbyz",
"name": "Upload API",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-19T17:31:49.875Z"
}
],
"created_at": "2025-12-19T15:23:08.000Z",
"id": "z8j9zb2pd30b",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Although Cloudflare's issues persist and the files they distribute take longer to download via `from_url` than others, our systems remain stable. We are closing the incident but will continue to monitor the situation.",
"created_at": "2025-12-19T17:31:49.913Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-19T17:31:49.913Z",
"id": "74q2vw67bxkz",
"incident_id": "z8j9zb2pd30b",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-19T17:31:49.913Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "`from_url` uploads are processing normally. We're continuing to monitor the situation.",
"created_at": "2025-12-19T16:38:46.448Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-19T16:38:46.000Z",
"id": "gwdgn3gbq1fl",
"incident_id": "z8j9zb2pd30b",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-19T16:40:09.014Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "`from_url` uploads are now processing normally. Our team monitors the situation.",
"created_at": "2025-12-19T15:37:10.892Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-19T15:37:10.892Z",
"id": "ymmyph0dbmb1",
"incident_id": "z8j9zb2pd30b",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-19T15:37:10.892Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We're seeing slow `from_url` uploads processing due to network performance issues at Cloudflare https://www.cloudflarestatus.com/\nSince a lot of submitted URLs are hosted by Cloudflare, our `from_url` queue is processing slower than usual.",
"created_at": "2025-12-19T15:23:08.459Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-19T15:23:08.000Z",
"id": "w6cnfr40k9mr",
"incident_id": "z8j9zb2pd30b",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-19T17:28:47.928Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2025-12-19T16:38:46.000Z",
"name": "`from_url` uploads degradation",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-12-19T17:31:49.913Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/y8w5p0v3jvrr",
"started_at": "2025-12-19T15:23:08.000Z",
"status": "resolved",
"updated_at": "2025-12-19T17:31:49.932Z"
},
{
"components": [
{
"created_at": "2015-08-18T18:37:03.144Z",
"description": null,
"group_id": null,
"id": "430q802wdbyz",
"name": "Upload API",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-19T17:31:49.875Z"
},
{
"created_at": "2015-08-18T18:37:03.158Z",
"description": null,
"group_id": null,
"id": "73mzhqs4mzy4",
"name": "REST API",
"page_id": "ltnt1pzqrlmg",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-18T08:53:10.218Z"
},
{
"created_at": "2019-11-04T18:39:00.802Z",
"description": null,
"group_id": null,
"id": "j2904rh7dsb6",
"name": "Dashboard",
"page_id": "ltnt1pzqrlmg",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.618Z"
}
],
"created_at": "2025-12-15T18:05:53.656Z",
"id": "wbb64j910d6l",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The incident is resolved.",
"created_at": "2025-12-16T03:32:21.583Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T03:32:21.583Z",
"id": "ywkrmkdlv9b4",
"incident_id": "wbb64j910d6l",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-16T03:32:21.583Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2025-12-16T03:06:44.607Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T03:06:44.607Z",
"id": "16mqfyt72s2z",
"incident_id": "wbb64j910d6l",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-16T03:06:44.607Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We experience more DB lateny issues.\nInvestigating, looking for the source of the latency.",
"created_at": "2025-12-16T01:15:44.618Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T01:15:44.618Z",
"id": "kg2bfjmx67mk",
"incident_id": "wbb64j910d6l",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-16T01:15:44.618Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We experience more DB lateny issues.",
"created_at": "2025-12-16T01:11:48.548Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T01:11:48.548Z",
"id": "qvc80v9pc249",
"incident_id": "wbb64j910d6l",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-16T01:11:48.548Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The Upload API is still experiencing degradation.\nYou may see longer-than-usual upload processing times, as well as intermittent 5XX responses.\nWe’re actively working to resolve the underlying issue and will continue to share updates.",
"created_at": "2025-12-16T00:30:23.770Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T00:30:23.770Z",
"id": "7ggsvlbw0fb4",
"incident_id": "wbb64j910d6l",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-16T00:30:23.770Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We are continuing to monitor for any further issues.",
"created_at": "2025-12-16T00:10:46.824Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T00:10:46.824Z",
"id": "r6cjcc473jrg",
"incident_id": "wbb64j910d6l",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-16T00:10:46.824Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "DB performance is recovered along with REST API and Dashboard.",
"created_at": "2025-12-15T23:15:38.025Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T23:15:38.025Z",
"id": "z10x5nt2290b",
"incident_id": "wbb64j910d6l",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T23:15:38.025Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "Extensive processing of Upload API backlog affects DB and therefore latency of REST API and Dashboard.",
"created_at": "2025-12-15T22:45:52.845Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T22:45:52.845Z",
"id": "n0h9np8gnyz6",
"incident_id": "wbb64j910d6l",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T22:45:52.845Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A fix has been implemented and we are monitoring the results.\nThe upload \"finish\" wait times are still outside of norm.",
"created_at": "2025-12-15T21:59:22.070Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T21:59:22.000Z",
"id": "9kjc07s3d8bg",
"incident_id": "wbb64j910d6l",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T22:28:51.515Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Currently there is a very big task queue accrued during initial production issue phase.\nWe've fixed original issue and are working through this queue.\n\nAll uploaded files should be accounted for eventually.\nFor end users it still looks like uploads are not working.\n\nAffected endpoints:\n- `/from_url/`\n- `/base/`",
"created_at": "2025-12-15T20:55:07.500Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T20:55:07.500Z",
"id": "h03mvkbwzk2h",
"incident_id": "wbb64j910d6l",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T20:55:07.500Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We are continuing to work on a fix for this issue.",
"created_at": "2025-12-15T20:29:34.486Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T20:29:34.486Z",
"id": "mp46ttbmwtxx",
"incident_id": "wbb64j910d6l",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T20:29:34.486Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2025-12-15T19:24:00.646Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T19:24:00.646Z",
"id": "98hyp8gkw7wh",
"incident_id": "wbb64j910d6l",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T19:24:00.646Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "partial_outage",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2025-12-15T18:50:05.261Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T18:50:05.261Z",
"id": "7xn15z8djbtg",
"incident_id": "wbb64j910d6l",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T18:50:05.261Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We're experiencing performance degradation for Upload API.\nUploads are finished with significantly higher latency.",
"created_at": "2025-12-15T18:07:22.793Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T18:07:22.793Z",
"id": "4mbdw757dg69",
"incident_id": "wbb64j910d6l",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T18:07:22.793Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2025-12-15T18:05:53.738Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-15T18:05:53.738Z",
"id": "zsktmt2ww6zl",
"incident_id": "wbb64j910d6l",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-15T18:05:53.738Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2025-12-15T21:59:22.000Z",
"name": "Upload API Degradation.",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": "# **Analysis of the December 15, 2025, Upload API Disruption**\n\n## 1. Summary\n\nOn December 15, 2025, the Uploadcare platform experienced a significant service disruption affecting the Upload API and REST API subsystems. The incident, spanning approximately ten hours, resulted in elevated error rates for file uploads, increased latency across API endpoints, and interruptions of Dashboard operations.\n\n## 2. Root cause\n\nThe root cause was a resource contention event on Amazon Elastic File System \\(EFS\\), aggravated by an observability side-effect. A monitoring agent, executing a metadata-heavy recursive file scan, monopolized the I/O capacity of the shared NFS mount points.\n\nThis I/O saturation created a cascading failure due to a legacy architectural pattern where storage operations occur within active database transactions. Background Processing Workers, taking long time to read or write to the stalled filesystem, held open database transactions for a long time. This saturated the global connection pool \\(PgBouncer\\), causing a denial of service for Upload API HTTP Workers and other services attempting to run database queries, despite the underlying PostgreSQL engine remaining healthy.\n\n## 3. Architectural context and system design\n\nThe Uploadcare platform is designed for scale, but this incident exposed specific coupling risks between our storage, database, and monitoring layers.\n\n### 3.1 Ingestion pipeline \u0026 worker roles\n\nThe core of the affected system is the Upload API. It utilizes two distinct worker types:\n\n1. Upload API HTTP workers. These synchronous workers handle incoming HTTP requests \\(Direct uploads, Multipart, and `from_url` requests\\). They ingest data and schedule tasks.\n2. Background processing workers. These asynchronous workers pick up tasks from a queue to perform complex operations \\(virus scanning, image validation, property extraction, etc\\).\n\nBoth worker types utilize a shared temporary staging area backed by Amazon EFS to maintain a POSIX-compliant shared state across the distributed fleet.\n\n### 3.2 Observability stack\n\nTo monitor the staging area's health, a Telegraf sidecar agent was configured to track file backlog depth. This was implemented via a standard Linux `find` command sequence.\n\nOn a network filesystem \\(NFS\\), a recursive `find` forces a traversal of the directory tree over the network. As file counts grow, the cost of this operation increases linearly, generating a storm of `READDIRPLUS` and `GETATTR` RPC calls.\n\n## 4. Timeline\n\n_All times are listed in Coordinated Universal Time \\(UTC\\)._\n\n* **17:17:** System begins receiving a massive surge of `from_url` upload requests. This workload is write-heavy.\n* **17:17 – 17:23:** Upload API HTTP workers successfully ingest files and write them to EFS. However, Background processing workers \\(responsible for processing and deleting files\\) begin to slow down due to the increasing I/O load. The rate of file creation exceeds the rate of processing/deletion. The file count on the volume jumps from ~40 \\(healthy\\) to ~7,000.\n* **17:25:** The \"Noisy Neighbor\" effect begins. The Telegraf `find` command, attempting to scan 7,000\\+ files, fails to complete within its 30-second timeout. We lose visibility of metrics related to the file staging area.\n* **17:30:** Multiple instances of the `find` command stack up. The resulting metadata storm heavily affects the NFS client's ability to process requests. Background processing workers become very ineffective.\n* **19:06:** PostgreSQL begins terminating client connections due to `idle-in-transaction` timeouts. This confirms workers are stuck holding connections.\n* **20:28 – 20:51:** Engineering attempts to horizontally scale the PgBouncers to absorb the held connections. This inadvertently pushes the upper layer PgBouncers toward their OS file descriptor limits.\n* **21:07:** Upper layer PgBouncer instances hit the Linux file descriptor limit \\(`ulimit`\\). New connections to this layer begin failing.\n* **21:28:** File descriptor limits are increased. Background processing workers begin to recover slowly as the pool stabilizes.\n* **23:30:** Background processing workers successfully process the backlog. The background task queue becomes empty.\n* **23:41:** Despite the background queue being empty, Upload API HTTP workers continue to exhibit \"overload\" behavior. \n* **01:35 \\(Dec 16\\):** Engineering reconfigures Upload API HTTP workers to connect directly to the upper PgBouncer layer. \n* **02:48:** EFS metrics reappear. \n* **02:58:** Engineers manually removed approximately 4,000 \"orphaned\" files that remained on the disk \\(files where the background job failed during the crash and could not self-heal\\). File manipulations on EFS become normal.\n* **02:59:** Database connection pressure vanished. Latency returned to nominal levels. Incident resolved.\n\n## 5. Response Evaluation\n\n### What went well\n\n* Proactive alerting. Our monitoring systems successfully triggered alerts for elevated wait times before customer reports of degradation began to surface, giving the team an early head start on triage.\n* Core database resilience. Despite the severe saturation of the connection poolers \\(PgBouncer\\), the underlying PostgreSQL database engine demonstrated remarkable stability. Resource utilization \\(CPU/Memory\\) on the database nodes remained nominal, preventing a total database meltdown.\n* URL API isolation. The URL API \\(file processing and delivery\\) remained fully operational throughout the incident. Its architectural isolation from the ingestion pipeline ensured that end-user access to previously uploaded files was never affected.\n\n### What went wrong\n\n* Observability became the bottleneck. The monitoring agent \\(`telegraf`\\), designed to provide visibility, became the primary cause of the outage. The use of an active, heavy command like `find` on a network filesystem was a design flaw that turned a monitoring check into a Denial of Service attack on the NFS client.\n* Misleading initial signals. The initial flood of `idle-in-transaction` connections led the response team to focus on the database layer. It delayed the investigation into the storage layer.\n\n## 5. Action Items\n\n* Eliminate \"Noisy Neighbor\" observability. Replace active, resource-intensive monitoring checks \\(specifically the recursive `find` command\\) with passive metrics. Transition to using AWS CloudWatch metrics or other side-channel indicators that do not consume client-side I/O credits or burden the EFS mount.\n* Enforce connection pool isolation. Reconfigure PgBouncer to implement strict resource bulkheading. Dedicate separate connection pools for the Upload API HTTP workers, REST API HTTP workers, and background workers. This ensures that a resource stall in the background worker fleet cannot starve the customer-facing APIs of database connections.\n* Decouple storage I/O from database state. Refactor the application logic to execute blocking file storage operations \\(reading/writing to EFS\\) _outside_ of atomic database transactions. This ensures that storage latency results only in slower worker throughput, rather than holding database connections open \\(`idle-in-transaction`\\) and exhausting the global pool.\n* Upgrade incident response tooling. Equip production containers and nodes with essential diagnostic capabilities and establish secure access protocols. This ensures engineers can diagnose kernel-level and network-level bottlenecks during active incidents.\n\n## 6. Conclusion\n\nThe December 15 disruption was a complex failure mode involving storage physics, monitoring side-effects, and database concurrency. By identifying the contention on the EFS mount \\(caused by the monitoring agent\\) as the root cause, rather than a hard AWS limit, we have a clear path to resolution. The removal of the invasive monitoring script and the decoupling of transactions will robustly insulate the Uploadcare platform from similar incidents in the future. \n\nWe apologize for the disruption and appreciate the patience of our customers as we hardened our systems.",
"postmortem_body_last_updated_at": "2025-12-17T17:55:21.396Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2025-12-17T17:55:21.396Z",
"reminder_intervals": null,
"resolved_at": "2025-12-16T03:32:21.583Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/qw8n7yg8gh7q",
"started_at": "2025-12-15T18:05:53.651Z",
"status": "resolved",
"updated_at": "2025-12-17T17:55:21.414Z"
},
{
"components": [
{
"created_at": "2015-08-18T18:37:03.144Z",
"description": null,
"group_id": null,
"id": "430q802wdbyz",
"name": "Upload API",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-19T17:31:49.875Z"
},
{
"created_at": "2019-08-08T07:59:15.562Z",
"description": null,
"group_id": "9bs61hmlspyq",
"id": "qt36c1nzm6lb",
"name": "Document processing",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-11T11:00:21.463Z"
},
{
"created_at": "2019-08-08T08:02:58.258Z",
"description": null,
"group_id": "9bs61hmlspyq",
"id": "lc2xq7nlktyn",
"name": "Video processing",
"page_id": "ltnt1pzqrlmg",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-11T11:00:21.521Z"
},
{
"created_at": "2021-06-01T07:34:33.078Z",
"description": "The service sends real-time updates about uploads.",
"group_id": null,
"id": "kf5n2d80ls0j",
"name": "Webhooks",
"page_id": "ltnt1pzqrlmg",
"position": 2,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-12-11T11:00:21.570Z"
},
{
"created_at": "2015-08-18T18:37:03.158Z",
"description": null,
"group_id": null,
"id": "73mzhqs4mzy4",
"name": "REST API",
"page_id": "ltnt1pzqrlmg",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-18T08:53:10.218Z"
},
{
"created_at": "2019-11-04T18:39:00.802Z",
"description": null,
"group_id": null,
"id": "j2904rh7dsb6",
"name": "Dashboard",
"page_id": "ltnt1pzqrlmg",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.618Z"
},
{
"created_at": "2019-11-04T18:39:15.441Z",
"description": null,
"group_id": null,
"id": "z32z0zbgd3p8",
"name": "Website",
"page_id": "ltnt1pzqrlmg",
"position": 7,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.641Z"
}
],
"created_at": "2025-12-05T11:03:46.451Z",
"id": "zcxd5wcxdxk8",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "z32z0zbgd3p8",
"name": "Website",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2025-12-11T11:00:21.863Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-11T11:00:21.863Z",
"id": "28d16zyzj6w9",
"incident_id": "zcxd5wcxdxk8",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-11T11:00:21.863Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "z32z0zbgd3p8",
"name": "Website",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2025-12-11T09:00:21.745Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-11T09:00:21.745Z",
"id": "8wg41pdlj70c",
"incident_id": "zcxd5wcxdxk8",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-11T09:00:21.745Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "z32z0zbgd3p8",
"name": "Website",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing scheduled maintenance on our database on 2025-12-11 at 09:00 UTC. The maintenance window is expected to last approximately 2 hours.\n\nNo downtime is expected during this time, but some operations may experience temporary delays or reduced performance.\n\nWe will monitor the process closely and provide updates here as necessary. Thank you for your understanding.",
"created_at": "2025-12-05T11:03:46.524Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-05T11:03:46.524Z",
"id": "43gnf6l5xl1l",
"incident_id": "zcxd5wcxdxk8",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-05T11:05:51.576Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled Database Maintenance",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-12-11T11:00:21.863Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2025-12-11T09:00:00.000Z",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2025-12-11T11:00:00.000Z",
"shortlink": "https://stspg.io/y4qhrxp5v385",
"started_at": "2025-12-05T11:03:46.439Z",
"status": "completed",
"updated_at": "2025-12-11T11:00:21.924Z"
},
{
"components": [],
"created_at": "2025-10-20T00:30:00.000Z",
"id": "8fgndf3w3l5w",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": null,
"body": "On October 20, 2025, between 06:49 UTC and 09:27 UTC, a portion of our URL API service experienced a major disruption. The service responsible for real-time processing and transformation of uncached files was unavailable, resulting in failed requests for those specific operations. The delivery of already-cached files and all other Uploadcare services, including file uploading and management, were unaffected.",
"created_at": "2025-10-20T09:48:32.349Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-10-20T06:30:00.000Z",
"id": "5q0jw0dq1qvk",
"incident_id": "8fgndf3w3l5w",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-10-20T09:50:04.298Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "URL API Service Disruption",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": "On October 20, 2025, between 07:50 UTC and 09:27 UTC, a portion of our URL API service experienced a major disruption. The service responsible for real-time processing and transformation of uncached files was unavailable, resulting in failed requests for those specific operations. The delivery of already-cached files and all other Uploadcare services, including file uploading and management, were unaffected.\n\nThe direct root cause of this incident was a major service outage in the Amazon Web Services \\(AWS\\) `us-east-1` \\(N. Virginia\\) region, specifically impacting AWS DynamoDB.\n\nDuring the incident, we also identified a critical flaw in our incident communication process: we were unable to access our Atlassian-hosted status page to provide timely updates to our customers. It was affected by AWS services disruption, too.\n\nOur key follow-up actions will focus on improving the architectural resilience of our services to reduce single points of failure and establishing a more robust, independent system for incident communications.\n\n## Timeline of events\n\nAll times are in UTC on October 20, 2025.\n\n* **06:49:** Our internal monitoring systems begin to alert on a significant increase in error rates and latency for the URL API service, specifically for requests involving uncached files. Our engineering team begins an immediate investigation.\n* **07:03**: Our engineering team localizes issue with DynamoDB.\n* **07:37:** The team attempts to update our public status page \\(`status.uploadcare.com`\\) to inform customers but is unable to log in to the third-party provider \\(Atlassian Statuspage\\).\n* **07:46:** We get to conclusion that the issue is related to DNS of DynamoDB.\n* **07:51:** AWS posts its first notification acknowledging increased error rates and latencies for multiple services in the `us-east-1` region. Our team correlates our internal alerts with this broader AWS issue.\n* **08:26:** AWS confirms that the issue is centered on significant error rates for DynamoDB in `us-east-1`, confirming our initial diagnosis of the root cause affecting our service.\n* **09:01:** AWS reports they have identified a potential root cause related to DNS resolution for the DynamoDB API endpoint and are working on mitigations.\n* **09:27:** Our internal monitoring shows that error rates for the URL API have returned to normal levels. The service is fully recovered and operational. We declare the incident resolved internally.\n\n## What went well\n\n* Our internal monitoring systems detected the service failure immediately, allowing for a rapid start to our investigation.\n* Our engineering team was able to quickly correlate the internal issue with the external AWS service outage, preventing wasted time on internal diagnostics.\n\n## What went wrong\n\n* **Architectural single point of failure:** Our URL API for uncached files had a hard dependency on a single service \\(DynamoDB\\) within a single AWS region \\(`us-east-1`\\). The failure of this service led directly to the failure of ours, with no mechanism for graceful degradation or failover.\n* **Failure of incident communication:** Our designated channel for customer communication, the status page hosted at `status.uploadcare.com`, was inaccessible to our team during the outage. This failure prevented us from providing timely and transparent updates to our users, which is a critical part of our incident response protocol.\n\n## Action items\n\n* **Establish a redundant status communication channel:** We will set up a secondary, fully independent channel for incident communication. This will ensure that if our primary status page provider is ever unavailable, we can still communicate effectively with our customers.\n* **Architect for resilience and decentralization:** Our engineering team will conduct a full architectural review of the URL API service. The primary goal is to re-architect the service to remove DynamoDB in `us-east-1` as a single point of failure. This may involve implementing multi-region failover capabilities or introducing a more resilient data caching layer.\n* **Audit critical services for single points of failure:** We will expand our review to other critical services across the Uploadcare platform to identify and mitigate other potential single points of failure related to external, regional dependencies.\n\nWe sincerely apologize for the disruption this incident caused our customers and for our failure to communicate the issue in a timely manner. We are committed to learning from this event and implementing these changes to build a more resilient and reliable platform.",
"postmortem_body_last_updated_at": "2025-10-20T11:30:31.359Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2025-10-20T10:01:56.977Z",
"reminder_intervals": null,
"resolved_at": "2025-10-20T06:30:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/5btg2lfsxc5l",
"started_at": "2025-10-20T06:30:00.000Z",
"status": "resolved",
"updated_at": "2025-10-20T11:30:31.362Z"
},
{
"components": [],
"created_at": "2025-07-03T13:54:33.872Z",
"id": "4rn72v31pmq5",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "The scheduled maintenance has been completed.",
"created_at": "2025-07-08T09:00:21.221Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-08T09:00:21.221Z",
"id": "78py3b4hjmvy",
"incident_id": "4rn72v31pmq5",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-08T09:00:21.221Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2025-07-08T08:00:21.438Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-08T08:00:21.438Z",
"id": "2hy5jjymvnv8",
"incident_id": "4rn72v31pmq5",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-08T08:00:21.438Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We will be performing scheduled maintenance on our Proxy service on 2025-07-08 at 06:00 UTC. The maintenance window is expected to last approximately 1 hour.\n\nNo downtime is expected during this time, but some operations may experience temporary delays or reduced performance.\n\nWe will monitor the process closely and provide updates here as necessary. Thank you for your understanding.",
"created_at": "2025-07-03T13:54:33.909Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-03T13:54:33.909Z",
"id": "f979s1bk7cl7",
"incident_id": "4rn72v31pmq5",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-03T13:54:33.909Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Proxy service maintenance",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-07-08T09:00:21.221Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2025-07-08T08:00:00.000Z",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2025-07-08T09:00:00.000Z",
"shortlink": "https://stspg.io/ydb3zhm0gl3k",
"started_at": "2025-07-03T13:54:33.869Z",
"status": "completed",
"updated_at": "2025-07-08T09:00:21.260Z"
},
{
"components": [
{
"created_at": "2015-08-18T18:37:03.144Z",
"description": null,
"group_id": null,
"id": "430q802wdbyz",
"name": "Upload API",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-19T17:31:49.875Z"
},
{
"created_at": "2019-08-08T07:59:15.562Z",
"description": null,
"group_id": "9bs61hmlspyq",
"id": "qt36c1nzm6lb",
"name": "Document processing",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-11T11:00:21.463Z"
},
{
"created_at": "2019-08-08T08:02:58.258Z",
"description": null,
"group_id": "9bs61hmlspyq",
"id": "lc2xq7nlktyn",
"name": "Video processing",
"page_id": "ltnt1pzqrlmg",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-11T11:00:21.521Z"
},
{
"created_at": "2021-06-01T07:34:33.078Z",
"description": "The service sends real-time updates about uploads.",
"group_id": null,
"id": "kf5n2d80ls0j",
"name": "Webhooks",
"page_id": "ltnt1pzqrlmg",
"position": 2,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-12-11T11:00:21.570Z"
},
{
"created_at": "2015-08-18T18:37:03.158Z",
"description": null,
"group_id": null,
"id": "73mzhqs4mzy4",
"name": "REST API",
"page_id": "ltnt1pzqrlmg",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-18T08:53:10.218Z"
},
{
"created_at": "2019-11-04T18:39:00.802Z",
"description": null,
"group_id": null,
"id": "j2904rh7dsb6",
"name": "Dashboard",
"page_id": "ltnt1pzqrlmg",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.618Z"
}
],
"created_at": "2025-07-03T06:25:52.339Z",
"id": "xp3mxbq84rs8",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2025-07-07T01:00:21.351Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-07T01:00:21.351Z",
"id": "8cfxwszs1m9d",
"incident_id": "xp3mxbq84rs8",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-07T01:00:21.351Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2025-07-07T00:00:21.356Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-07T00:00:21.356Z",
"id": "9y7kd6cmmq88",
"incident_id": "xp3mxbq84rs8",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-07T00:00:21.356Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
}
],
"body": "We will be performing scheduled maintenance on our database on Monday at 06:00 UTC. The maintenance window is expected to last approximately 1 hour.\n\nNo downtime is expected during this time, but some operations may experience temporary delays or reduced performance.\n\nWe will monitor the process closely and provide updates here as necessary. Thank you for your understanding.",
"created_at": "2025-07-03T06:25:52.401Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-03T06:25:52.401Z",
"id": "nj0jhfx2s9w2",
"incident_id": "xp3mxbq84rs8",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-03T06:25:52.401Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled Database Maintenance",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-07-07T01:00:21.351Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2025-07-07T00:00:00.000Z",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2025-07-07T01:00:00.000Z",
"shortlink": "https://stspg.io/8hz03bwjtj6s",
"started_at": "2025-07-03T06:25:52.328Z",
"status": "completed",
"updated_at": "2025-07-07T01:00:21.410Z"
},
{
"components": [
{
"created_at": "2015-08-18T18:37:03.144Z",
"description": null,
"group_id": null,
"id": "430q802wdbyz",
"name": "Upload API",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-19T17:31:49.875Z"
},
{
"created_at": "2019-08-08T07:59:15.562Z",
"description": null,
"group_id": "9bs61hmlspyq",
"id": "qt36c1nzm6lb",
"name": "Document processing",
"page_id": "ltnt1pzqrlmg",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-11T11:00:21.463Z"
},
{
"created_at": "2019-08-08T08:02:58.258Z",
"description": null,
"group_id": "9bs61hmlspyq",
"id": "lc2xq7nlktyn",
"name": "Video processing",
"page_id": "ltnt1pzqrlmg",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-11T11:00:21.521Z"
},
{
"created_at": "2021-06-01T07:34:33.078Z",
"description": "The service sends real-time updates about uploads.",
"group_id": null,
"id": "kf5n2d80ls0j",
"name": "Webhooks",
"page_id": "ltnt1pzqrlmg",
"position": 2,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-12-11T11:00:21.570Z"
},
{
"created_at": "2015-08-18T18:37:03.158Z",
"description": null,
"group_id": null,
"id": "73mzhqs4mzy4",
"name": "REST API",
"page_id": "ltnt1pzqrlmg",
"position": 3,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-12-18T08:53:10.218Z"
},
{
"created_at": "2019-11-04T18:39:00.802Z",
"description": null,
"group_id": null,
"id": "j2904rh7dsb6",
"name": "Dashboard",
"page_id": "ltnt1pzqrlmg",
"position": 6,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-16T12:51:14.618Z"
}
],
"created_at": "2025-07-01T10:54:34.257Z",
"id": "s82c1pm4x7rb",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2025-07-03T07:00:21.689Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-03T07:00:21.689Z",
"id": "9j8l709gmp5p",
"incident_id": "s82c1pm4x7rb",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-03T07:00:21.689Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2025-07-03T06:00:21.647Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-03T06:00:21.647Z",
"id": "mpfbh2np6533",
"incident_id": "s82c1pm4x7rb",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-03T06:00:21.647Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "430q802wdbyz",
"name": "Upload API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "73mzhqs4mzy4",
"name": "REST API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qt36c1nzm6lb",
"name": "Processing engines - Document processing",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "lc2xq7nlktyn",
"name": "Processing engines - Video processing",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "j2904rh7dsb6",
"name": "Dashboard",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kf5n2d80ls0j",
"name": "Webhooks",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be performing scheduled maintenance on our database on Thursday at 06:00 UTC. The maintenance window is expected to last approximately 1 hour.\n\nNo downtime is expected during this time, but some operations may experience temporary delays or reduced performance.\n\nWe will monitor the process closely and provide updates here as necessary. Thank you for your understanding.",
"created_at": "2025-07-01T10:54:34.324Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-07-01T10:54:34.324Z",
"id": "smx27hwtbl0b",
"incident_id": "s82c1pm4x7rb",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-07-01T10:54:34.324Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Scheduled Database Maintenance",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-07-03T07:00:21.689Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2025-07-03T06:00:00.000Z",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2025-07-03T07:00:00.000Z",
"shortlink": "https://stspg.io/qb76b0h9f6hb",
"started_at": "2025-07-01T10:54:34.246Z",
"status": "completed",
"updated_at": "2025-07-03T07:00:21.740Z"
},
{
"components": [
{
"created_at": "2016-08-30T14:15:44.762Z",
"description": null,
"group_id": null,
"id": "sv5vvph3q2jz",
"name": "CDN",
"page_id": "ltnt1pzqrlmg",
"position": 4,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-20T09:51:24.639Z"
}
],
"created_at": "2024-11-26T14:56:34.911Z",
"id": "m35596kt80qy",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "sv5vvph3q2jz",
"name": "CDN",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2024-11-27T09:14:20.502Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2024-11-26T18:00:00.000Z",
"id": "t72tpjg219xd",
"incident_id": "m35596kt80qy",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2024-11-27T09:15:10.052Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sv5vvph3q2jz",
"name": "CDN",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Systems are back to normal",
"created_at": "2024-11-26T15:16:12.063Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2024-11-26T15:16:12.063Z",
"id": "zr2lz91lwqyn",
"incident_id": "m35596kt80qy",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2024-11-26T15:16:12.063Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "sv5vvph3q2jz",
"name": "CDN",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We're experiencing an elevated level of URL API errors and are currently looking into the issue.",
"created_at": "2024-11-26T14:56:34.989Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2024-11-26T14:56:34.989Z",
"id": "7jm8x44scjf7",
"incident_id": "m35596kt80qy",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2024-11-26T14:56:34.989Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Elevated URL API Errors",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": "## Incident Summary\n\nOn 26 November 2024, an issue with our URL API was identified, causing a partial outage of the service from 14:31 to 15:00 UTC.\n\n## Timeline\n\n* **14:20 UTC** – A CDN configuration change was made to improve the Auto Format feature in the URL API.\n* **14:31 UTC** – Service degradation began as increased traffic overwhelmed our origin servers due to cache key changes.\n* **14:37 UTC** – Alert notifications were received by our team.\n* **14:58 UTC** – The CDN configuration was rolled back.\n* **15:00 UTC** – The incident was fully resolved, and service was restored.\n\n## Root Cause\n\nThe issue was caused by a CDN configuration change that altered cache key behavior, significantly increasing traffic to our origin servers. This overwhelmed our autoscaling groups, which hit their scaling limits, resulting in a service outage.\n\n## Impact\n\nURL API degradation lasted for approximately 29 minutes.\n\n## Challenges During Resolution\n\nThis type of issue is difficult to catch in a staging environment because the traffic profile does not reflect production-level demand. Although the alert system worked as intended, delays in responding to the alert contributed to the resolution time.\n\n## Resolution\n\nThe configuration change was identified as the root cause and rolled back. The service stabilized immediately after the rollback, and full functionality was restored by 15:00 UTC.\n\n## Action Items\n\n### Short-term\n\n* Review and improve escalation processes for alert notifications to reduce resolution time.\n* Adjust staging environment testing to better simulate production traffic patterns for similar changes.\n\n## Long-term\n\n* Assess autoscaling limits to ensure sufficient capacity for handling unexpected traffic spikes.\n\n \nWe sincerely apologize for the disruption and are committed to learning from this incident to improve the reliability of our services. Thank you for your understanding and continued support.",
"postmortem_body_last_updated_at": "2024-11-27T09:39:50.463Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2024-11-27T09:39:50.463Z",
"reminder_intervals": "[]",
"resolved_at": "2024-11-26T18:00:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bs1j85yq3f3m",
"started_at": "2024-11-26T14:56:34.906Z",
"status": "resolved",
"updated_at": "2024-11-27T09:39:50.479Z"
},
{
"components": [],
"created_at": "2024-11-25T10:00:00.000Z",
"id": "z3fshcp2v5vd",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": null,
"body": "Issue has been resolved",
"created_at": "2024-11-27T10:18:13.288Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2024-11-25T19:45:00.000Z",
"id": "67ty88vz1f7n",
"incident_id": "z3fshcp2v5vd",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2024-11-27T10:22:32.314Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Unavailability of uploading from 3rd Party Services",
"page_id": "ltnt1pzqrlmg",
"postmortem_body": "## Incident Summary\n\nOn November 25, 2024, from 16:01 to 19:44 UTC, the “Uploading from 3rd Party Services” feature of our platform was unavailable. This feature allows users to upload files from social networks and storage services like Dropbox, Facebook, Google Drive, and Instagram.\n\n## Timeline\n\n* **15:50 UTC** – A server configuration update was initiated to streamline deployments in our Kubernetes environment.\n* **16:01 UTC** – The outage began as requests to the “Uploading from 3rd Party Services” feature failed to be processed.\n* **19:00 UTC** – Investigation revealed that the application networking was incorrectly configured.\n* **19:15 UTC** – A fix was implemented to re-configure the application networking.\n* **19:44 UTC** – The incident was fully resolved, and service was restored.\n\n## Root Cause\n\nThe outage was caused by a synchronization issue between repositories during a server configuration update. This misalignment between repositories, compounded by a lack of explicit configuration, led to the service outage.\n\n## Impact\n\nFor approximately 3 hours and 43 minutes, users were unable to upload files from third-party services.\n\n## Challenges During Resolution\n\nThis type of issue is difficult to catch in a staging environment because the traffic profile does not reflect production-level demand. Although the alert system worked as intended, delays in responding to the alert contributed to the resolution time.\n\n## Resolution\n\nThe application configuration was updated and service functionality was fully restored at 19:44 UTC.\n\n## Action Items\n\n### Short-term\n\n* Improve feature status monitoring and alerting to detect outages faster.\n\n## Long-term\n\n* Improve synchronization processes between repositories to avoid dependency misalignments.\n\n\n\nWe sincerely apologize for the disruption this caused to our users. We take this incident seriously and are committed to implementing the above action items to prevent similar occurrences in the future.\n\nThank you for your patience and continued trust in our platform. If you have any questions or need further details, please don’t hesitate to reach out.",
"postmortem_body_last_updated_at": "2024-11-27T10:34:46.199Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2024-11-27T10:34:46.199Z",
"reminder_intervals": null,
"resolved_at": "2024-11-25T19:45:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/hp6qslrfpr9z",
"started_at": "2024-11-25T18:30:00.000Z",
"status": "resolved",
"updated_at": "2024-11-27T10:34:46.238Z"
}
],
"page": {
"id": "ltnt1pzqrlmg",
"name": "Uploadcare",
"url": "https://status.uploadcare.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}