Snapshot 303894
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"lghg3n9cd6jj",
"ncqf81v9p42j"
],
"created_at": "2020-08-25T06:28:15.153Z",
"description": null,
"group_id": null,
"id": "lvchkfg2wdtn",
"name": "Edlink Core Systems",
"page_id": "90lv07q7m45f",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2020-08-25T06:28:15.153Z"
},
{
"components": [
"yjllq2cb2xrv",
"b4c7k0s4wtrx",
"l1k8c3j5spl8",
"hvcj3cghg1yd",
"9hq4bz1k2pdp",
"w7pdm7kfrr15",
"tzf448873cvc",
"cgn7g2zkf4yy",
"p9tpw4bstfxn"
],
"created_at": "2020-08-25T06:29:42.534Z",
"description": null,
"group_id": null,
"id": "nf4g7l76k9s4",
"name": "Data Sources",
"page_id": "90lv07q7m45f",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2020-08-25T06:29:42.534Z"
},
{
"created_at": "2023-05-17T02:43:37.301Z",
"description": null,
"group_id": null,
"id": "3pkmc72zg8fh",
"name": "Google Cloud Platform Google Kubernetes Engine",
"page_id": "90lv07q7m45f",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-03-12T22:41:46.275Z"
},
{
"created_at": "2023-05-17T04:49:21.834Z",
"description": null,
"group_id": null,
"id": "gl06w2px85vx",
"name": "Google Cloud Platform Google Cloud Storage",
"page_id": "90lv07q7m45f",
"position": 6,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-09T00:17:17.547Z"
},
{
"created_at": "2023-05-17T04:50:43.082Z",
"description": null,
"group_id": null,
"id": "3t37235k4g0f",
"name": "Google Cloud Platform Google Cloud SQL",
"page_id": "90lv07q7m45f",
"position": 7,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2023-11-29T22:00:59.853Z"
},
{
"created_at": "2023-05-17T04:51:16.166Z",
"description": null,
"group_id": null,
"id": "b4dc84hws4zt",
"name": "Google Cloud Platform Cloud Key Management Service",
"page_id": "90lv07q7m45f",
"position": 8,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2023-05-17T04:51:16.166Z"
},
{
"created_at": "2023-05-17T04:52:08.549Z",
"description": null,
"group_id": null,
"id": "8h3f4w7qrq9h",
"name": "GitHub Actions",
"page_id": "90lv07q7m45f",
"position": 9,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-16T16:12:00.607Z"
},
{
"created_at": "2023-05-17T04:52:09.820Z",
"description": null,
"group_id": null,
"id": "3n6kx6m4n5s1",
"name": "GitHub Actions",
"page_id": "90lv07q7m45f",
"position": 10,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-16T16:12:00.926Z"
},
{
"created_at": "2023-05-17T04:52:12.958Z",
"description": null,
"group_id": null,
"id": "zp2p6vnckrqn",
"name": "GitHub Copilot",
"page_id": "90lv07q7m45f",
"position": 11,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-11T21:07:39.011Z"
},
{
"created_at": "2023-05-17T04:52:15.756Z",
"description": null,
"group_id": null,
"id": "hf5rp0289kkb",
"name": "GitHub Pull Requests",
"page_id": "90lv07q7m45f",
"position": 12,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-16T16:14:43.511Z"
},
{
"created_at": "2023-05-17T04:52:18.513Z",
"description": null,
"group_id": null,
"id": "664z6yvl7fwc",
"name": "GitHub Pages",
"page_id": "90lv07q7m45f",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-11T21:09:36.497Z"
},
{
"created_at": "2023-05-17T04:52:21.882Z",
"description": null,
"group_id": null,
"id": "stl3j3b6n53z",
"name": "GitHub Issues",
"page_id": "90lv07q7m45f",
"position": 14,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-06T00:09:40.719Z"
},
{
"created_at": "2023-05-17T04:52:25.100Z",
"description": null,
"group_id": null,
"id": "v01h05xt7c8s",
"name": "GitHub Git Operations",
"page_id": "90lv07q7m45f",
"position": 15,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-04T11:44:44.911Z"
},
{
"created_at": "2023-05-17T04:52:53.239Z",
"description": null,
"group_id": null,
"id": "l4dhzrwpp47b",
"name": "ClassLink Partner Portal",
"page_id": "90lv07q7m45f",
"position": 16,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-07-13T14:43:40.231Z"
},
{
"created_at": "2023-05-17T04:52:54.212Z",
"description": null,
"group_id": null,
"id": "rrs4zpy0w4pp",
"name": "ClassLink Roster Server",
"page_id": "90lv07q7m45f",
"position": 17,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-04-10T19:37:54.822Z"
},
{
"created_at": "2023-05-17T04:53:31.982Z",
"description": null,
"group_id": null,
"id": "8kml4m4rgz3t",
"name": "Clever Apps Dashboard",
"page_id": "90lv07q7m45f",
"position": 18,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-10-15T19:57:37.008Z"
},
{
"created_at": "2023-05-17T04:53:35.545Z",
"description": null,
"group_id": null,
"id": "jnx44s9yx285",
"name": "Clever Data API",
"page_id": "90lv07q7m45f",
"position": 19,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-11-19T01:51:29.062Z"
},
{
"created_at": "2023-05-17T04:53:37.161Z",
"description": null,
"group_id": null,
"id": "38my5k6xm4np",
"name": "Clever Events API",
"page_id": "90lv07q7m45f",
"position": 20,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-09-08T17:53:41.554Z"
},
{
"created_at": "2023-05-17T04:53:38.401Z",
"description": null,
"group_id": null,
"id": "4sggf7yf9qt4",
"name": "Clever Single Sign-On",
"page_id": "90lv07q7m45f",
"position": 21,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-08-05T20:48:38.118Z"
}
],
"incidents": [
{
"components": [
{
"created_at": "2020-08-25T06:25:23.375Z",
"description": "The primary Edlink API servers.",
"group_id": "lvchkfg2wdtn",
"id": "lghg3n9cd6jj",
"name": "Edlink API",
"page_id": "90lv07q7m45f",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.209Z"
},
{
"created_at": "2020-08-25T06:25:23.386Z",
"description": "The website frontend where developers manage their applications and schools manage their data sources.",
"group_id": "lvchkfg2wdtn",
"id": "ncqf81v9p42j",
"name": "Edlink Dashboard",
"page_id": "90lv07q7m45f",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.225Z"
}
],
"created_at": "2026-09-01T15:42:52.597Z",
"id": "506y6c23xn2c",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "Our control plane is back online and we appear to be serving API requests as expected. Syncs and materializations should resume shortly.",
"created_at": "2026-09-01T15:52:54.254Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-01T15:52:54.254Z",
"id": "gf7lydj6xcnq",
"incident_id": "506y6c23xn2c",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-01T15:52:54.254Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently experiencing degraded API performance. We've traced the incident to our primary hosting provider (GCP) and we are currently exploring ways to relocate resources to our API servers.\n\nWe are currently processing an approximately-normal level of login requests, but dashboard performance is severely degraded.",
"created_at": "2026-09-01T15:42:52.688Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-09-01T15:42:52.688Z",
"id": "6shx9h8chmzt",
"incident_id": "506y6c23xn2c",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-09-01T15:42:52.688Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Edlink API Degraded Performance",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-09-01T15:52:54.254Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/p3qglvrr5wk1",
"started_at": "2026-09-01T15:42:52.587Z",
"status": "resolved",
"updated_at": "2026-09-01T15:52:54.270Z"
},
{
"components": [],
"created_at": "2026-05-29T08:00:00.000Z",
"id": "nh8j4f43dz70",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": null,
"body": "This morning, we experienced an issue with the execution of custom transformations during the final materialization phase of our data pipeline. Custom transformations are utilized by approximately 22% of our integrations. Integrations without custom transformations were not impacted. Integrations that utilized \"Edlink-defined\" transformations (such as \"Infer Roles\") were not impacted.\n\nTimeline of Events\n\n- Our transformation execution relies on private NPM packages in our NPM organization.\n- Around 3:12am CT, the first custom transformation failed because it was unable to load packages from NPM.\n- The first client issues were reported around 8:30am CT, but it was not apparent that this was a widespread phenomenon until approximately 9:50am CT when it was escalated to P1.\n- We resolved the issue at approximately 10:10am CT, but due to latency, materializations did not resume until approximately 10:23am CT.\n\nRoot Cause\n\nThe root causes was related to an expired credit card.\nAfter a failed payment, NPM unexpectedly downgraded our account and we were no longer able to access our own packages.\nWe re-upgraded the account and after some brief latency due to their billing system, we are able to access packages again.\n\nChanges Going Forward\n\n1. In response, we plan to introduce additional alerting to detect unusual changes in our materializations.\n2. We are also seeking alternatives to NPM for holding our private packages for materialization execution. Our use of NPM was a technical requirement when we first implemented our custom transformation requirement, but that may no longer be the case.",
"created_at": "2026-05-29T15:46:31.001Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-05-29T08:00:00.000Z",
"id": "2x8yk7xx9lgv",
"incident_id": "nh8j4f43dz70",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-29T15:46:31.001Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Custom Transformation Failure During Materialization",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-05-29T08:00:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/k7wnr1bn2y86",
"started_at": "2026-05-29T15:46:30.935Z",
"status": "resolved",
"updated_at": "2026-05-29T15:46:38.689Z"
},
{
"components": [
{
"created_at": "2020-08-25T06:29:42.544Z",
"description": "The cloud hosted version of Canvas by Instructure.",
"group_id": "nf4g7l76k9s4",
"id": "yjllq2cb2xrv",
"name": "Canvas",
"page_id": "90lv07q7m45f",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-05-08T20:41:28.903Z"
}
],
"created_at": "2026-05-07T23:19:34.806Z",
"id": "6fgx1kzfcyjj",
"impact": "critical",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "yjllq2cb2xrv",
"name": "Data Sources - Canvas",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "Instructure has published a status update indicating that they do not believe any data was compromised in the most recent attack. We have published a longer statement on the topic (including our plans) here: https://www.linkedin.com/pulse/re-instructure-security-incident-dakota-gordon-8nyqc/",
"created_at": "2026-05-08T20:41:28.938Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-08T20:41:28.938Z",
"id": "mfq22nw728tn",
"incident_id": "6fgx1kzfcyjj",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-08T20:41:28.938Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "yjllq2cb2xrv",
"name": "Data Sources - Canvas",
"new_status": "major_outage",
"old_status": "operational"
}
],
"body": "Instructure is currently undergoing a data breach (separate from the one earlier this week) and the attackers have gained control of many Canvas instances. Users of these instances will not be able to sign in or access any content. Data syncs (either to or from Instructure) are presumed non-functional at the moment.\n\nThe situation is developing.",
"created_at": "2026-05-07T23:19:34.868Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-07T23:19:34.868Z",
"id": "z5sp4cnt8z0p",
"incident_id": "6fgx1kzfcyjj",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-07T23:19:34.868Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-05-07T23:19:34.868Z",
"name": "Instructure Data Breach \u0026 Product Outage",
"page_id": "90lv07q7m45f",
"postmortem_body": "You may know by now that Instructure was breached by bad actors twice in the last week. At this time, we have no reason to believe that our clients \\(or Edlink itself\\) was compromised. \n\nHowever, out of an abundance of caution and as a best practice, to have all districts using Canvas to rotate their API and LTI keys. \n\nHere's why: \n\nWhile it is Instructure’s current position that there’s “no evidence” that any data was accessed,we don’t share their level of conviction. This assumption is based on the fact that attackers hijacked substantially all production Canvas instances to show a ransom message, which would require a fairly high level of access to Instructure’s infrastructure and systems. \n\nAs a result, if a bad actor did get access to Instucture core systems, they'd likely have access to API and LTI keys.\n\nFor API \u0026 LTI 1.3 integrations, exposed keys mean that attackers can exfiltrate any data that those keys have access to. Attackers can act as your application and use end user tokens to retrieve data. It is unlikely that attackers will be able to impersonate Canvas users to sign into your platform, assuming you have correctly implemented OAuth 2.0 or OIDC \\(for LTI\\). \n\nFor LTI 1.1 integrations, exposed keys means that attackers can potentially sign into your product as “legitimate” end users. This can lead to possible data exfiltration from your product and it will be difficult or impossible to tell if traffic is legitimate. As such, we recommend that you immediately rotate all LTI 1.1 keys or upgrade to LTI 1.3, if possible.\n\nIn either case, the attackers could “sit” on stolen credentials for months or years before they decide to use them. By the time they do make their move, this incident may be a distant memory and it will be unclear to those affected exactly how the unauthorized access was obtained.\n\nLater today or early tomorrow, we will release a user interface that we can share with you to share with your Canvas school customers to make the key rotation process as seamless as possible for school IT admins. \n\nIf you'd like to rotate keys sooner, we would be happy to work directly with you or your schools to knock this out.",
"postmortem_body_last_updated_at": "2026-05-11T21:12:51.831Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-05-11T21:12:51.831Z",
"reminder_intervals": null,
"resolved_at": "2026-05-08T20:41:28.938Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/rzkhzb1p0syz",
"started_at": "2026-05-07T23:19:34.799Z",
"status": "resolved",
"updated_at": "2026-05-11T21:12:51.848Z"
},
{
"components": [
{
"created_at": "2020-08-25T06:25:23.375Z",
"description": "The primary Edlink API servers.",
"group_id": "lvchkfg2wdtn",
"id": "lghg3n9cd6jj",
"name": "Edlink API",
"page_id": "90lv07q7m45f",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.209Z"
},
{
"created_at": "2020-08-25T06:25:23.386Z",
"description": "The website frontend where developers manage their applications and schools manage their data sources.",
"group_id": "lvchkfg2wdtn",
"id": "ncqf81v9p42j",
"name": "Edlink Dashboard",
"page_id": "90lv07q7m45f",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.225Z"
}
],
"created_at": "2026-05-04T18:16:08.643Z",
"id": "hj0flrrsq2z9",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We've identified the root cause and the pipeline has resumed. We will publish a full post-mortem shortly.",
"created_at": "2026-05-04T19:57:16.793Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-04T19:57:16.793Z",
"id": "cy62vqw1099m",
"incident_id": "hj0flrrsq2z9",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-04T19:57:16.793Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We've isolated the issue to our sync service that retrieves data from school data sources. We are re-enabling enrichment \u0026 materialization. We are going to revert to the previous stable version of the sync application and re-enable it in a limited capacity for monitoring.",
"created_at": "2026-05-04T18:30:00.771Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-05-04T18:30:00.771Z",
"id": "x0l00jzm1r8f",
"incident_id": "hj0flrrsq2z9",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-04T18:30:00.771Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Reports of degraded API \u0026 SSO performance started coming in around 12:30pm CT.\nAfter investigation our logs, it appears that the degraded performance started at exactly 12:07pm Central Time (about 27 minutes prior to first reports).\nAs of 1:04pm Central Time, we have reallocated database resources and the immediate issue has been resolved. SSO \u0026 API requests have returned to expected levels.\nWe are continuing to investigate the root cause of the issue before restoring normal pipeline services.\n\nPlease expect delays in sync and materialization queues while we investigate the issue.",
"created_at": "2026-05-04T18:16:08.694Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-05-04T18:16:08.694Z",
"id": "vfnvxzbwtjxr",
"incident_id": "hj0flrrsq2z9",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-05-04T18:16:08.694Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-05-04T18:16:08.694Z",
"name": "Degraded API Performance",
"page_id": "90lv07q7m45f",
"postmortem_body": "**Date of Incident:** May 4, 2026\n\n**Duration:** 12:07 PM – 1:04 PM \\(57 Minutes\\)\n\n**Impact:** Total interruption of user login capabilities.\n\n**Status:** Resolved\n\n### **Executive Summary**\n\nBetween 12:07 PM and 1:04 PM today, Edlink experienced a severe API degradation that prevented substantially all users from logging into client products via SSO. The incident was caused by database connection starvation, which was triggered by a service deployment that utilized an older pinned version of the PostgreSQL \\(PG\\) library. The issue was mitigated by bringing down the problematic service to free up connections, followed by a full revert of the change. Normal operations have been fully restored.\n\n### **Impact**\n\n* **User Experience:** Substantially all users attempting to access Edlink during the 57-minute window were completely unable to log in or authenticate. A small portion were able to authenticate normally.\n* **System Impact:** Critical services, including the authentication/login service, were starved of database connections and timed out or dropped requests.\n\n### **Root Cause**\n\nThe incident was traced back to a deployment that occurred today at 12:07pm CT where a part of our service was upgraded.\n\n**Background on Dependency Version Pinning:**\n\nTo provide context on _why_ an older version was in use, we recently implemented a strict policy for managing our software dependencies. This proactive security measure was enacted following an industry-wide supply chain attack incident in March of 2026. Edlink was not affected by this incident directly, but we determined that the attack vector was not one that we were comfortable with, and as such, we moved to proactively update our systems.\n\nTo safeguard Edlink's platform and ensure our systems remain secure, we instituted a policy to \"pin\" or lock all our software dependencies to explicitly verified, “safe” versions. This prevents our systems from automatically downloading new, potentially unverified updates. This is the attack vector that was used in the industry-wide attack a few weeks back.\n\nWhile this policy is crucial for preventing malicious, unverified updates from automatically infiltrating our systems, in this specific instance, it had an unintended side effect. In this situation, it forced our deployment to utilize an older version of the PG library than we had intended.\n\nThis older version of the library managed Postgres connections highly inefficiently and would leak unused connections. Upon deployment, the service rapidly consumed the available database connection pool. Because the connections were not being properly released or managed, other critical infrastructure \\(notably the service handling user logins\\) was left without available database connections, leading to the system-wide login failures.\n\n### **Timeline**\n\n* **12:07 PM:** The deployment containing the outdated PG library goes live. Database connection pools immediately begin to spike. Users start experiencing login failures.\n* **~12:45 PM:** Emergency mitigation is enacted: the problematic service is brought down, immediately releasing its held connections back to the pool and restoring baseline functionality for the login service.\n* **1:04 PM:** The deployment is officially reverted to the previous stable state. System stability is confirmed, and the incident is closed.\n\n### **Resolution and Recovery**\n\nThe immediate bleeding was stopped by intentionally spinning down the problematic service, which allowed the authentication services to recover. The underlying code change was then completely reverted in version control and redeployed to ensure the service could be brought back online safely without causing a secondary outage.\n\n### **Preventative Measures**\n\nTo ensure we can more quickly identify and resolve similar issues in the future, we have integrated additional monitoring tools focused on detecting failed logins. This will allow us to immediately detect authentication issues and mitigate them before they cause a prolonged system-wide impact.\n\nAdditionally, we have conducted a review of any other dependency packages that may have been inadvertently downgraded during this “pinning” process.",
"postmortem_body_last_updated_at": "2026-05-04T21:22:51.266Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-05-04T21:22:32.761Z",
"reminder_intervals": null,
"resolved_at": "2026-05-04T19:57:16.793Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/h464pf6b8wh3",
"started_at": "2026-05-04T18:16:08.633Z",
"status": "resolved",
"updated_at": "2026-05-04T21:22:51.269Z"
},
{
"components": [],
"created_at": "2026-03-02T20:30:00.000Z",
"id": "c9lwjwnfpbjq",
"impact": "critical",
"impact_override": "critical",
"incident_updates": [
{
"affected_components": null,
"body": "For 172 seconds, our meta API (including SSO endpoints) were unresponsive. This incident took place around 2:30pm CT. The incident was caused by a lock on a table related to single sign on. The lock was the result of a database maintenance query that we were running at the time. The query was expected to complete in less than one second (and therefore, not cause any service disruption), but maintained the lock longer than expected. We terminated the execution of the query approximately two minutes later.",
"created_at": "2026-03-02T20:41:43.404Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-03-02T20:30:00.000Z",
"id": "gc1t5vmlgtf2",
"incident_id": "c9lwjwnfpbjq",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-03-02T20:41:43.404Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Meta API request timeouts",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-03-02T20:30:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/tfy84xgq5fk2",
"started_at": "2026-03-02T20:41:43.357Z",
"status": "resolved",
"updated_at": "2026-03-02T20:41:56.961Z"
},
{
"components": [
{
"created_at": "2020-08-25T06:25:23.375Z",
"description": "The primary Edlink API servers.",
"group_id": "lvchkfg2wdtn",
"id": "lghg3n9cd6jj",
"name": "Edlink API",
"page_id": "90lv07q7m45f",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.209Z"
}
],
"created_at": "2025-12-28T04:35:32.835Z",
"id": "qdgwv81q6fw8",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "The maintenance is complete and we're back to normal API operations.",
"created_at": "2025-12-28T05:46:27.525Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-28T05:46:27.525Z",
"id": "p07cb2f0dqbg",
"incident_id": "qdgwv81q6fw8",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-28T05:46:27.525Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "We are conducting scheduled maintenance on our US regional database.",
"created_at": "2025-12-28T04:35:32.932Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-28T04:35:32.932Z",
"id": "ybyc10ktzchp",
"incident_id": "qdgwv81q6fw8",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-28T04:35:32.932Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2025-12-28T04:35:32.932Z",
"name": "12/27 - Scheduled Downtime",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": "[]",
"resolved_at": "2025-12-28T05:46:27.525Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/460y5tq0qk4c",
"started_at": "2025-12-28T04:35:32.826Z",
"status": "resolved",
"updated_at": "2025-12-28T05:46:27.543Z"
},
{
"components": [
{
"created_at": "2020-11-25T17:01:15.241Z",
"description": null,
"group_id": "nf4g7l76k9s4",
"id": "b4c7k0s4wtrx",
"name": "Schoology",
"page_id": "90lv07q7m45f",
"position": 2,
"showcase": true,
"start_date": "2020-11-25T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-12-23T05:14:49.143Z"
}
],
"created_at": "2025-12-09T17:41:43.000Z",
"id": "mtbv013cqtr7",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "Although we have not heard any official messages from Schoology, it appears that the API issues have been resolved on their end. Sync have resumed.",
"created_at": "2025-12-23T05:14:49.181Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-23T05:14:49.181Z",
"id": "xx38yfp4cwpz",
"incident_id": "mtbv013cqtr7",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-23T05:14:49.181Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "Error rates have dropped again and we are begging to resume syncs. Last word from Powerschool is that they are \"looking into it\" (as of last Friday).",
"created_at": "2025-12-22T15:37:27.225Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-22T15:37:27.225Z",
"id": "msdzg8shgk11",
"incident_id": "mtbv013cqtr7",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-22T15:37:27.225Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "...and we're back to seeing elevated error rates again after a relatively smooth morning and early afternoon.",
"created_at": "2025-12-17T22:21:12.493Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-17T22:21:12.493Z",
"id": "dw929n2qx930",
"incident_id": "mtbv013cqtr7",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-17T22:21:12.493Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "Everything appears to be back to normal, we're waiting on a few more districts to sync.",
"created_at": "2025-12-17T16:53:12.846Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-17T16:53:12.846Z",
"id": "y23vvq54bf69",
"incident_id": "mtbv013cqtr7",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-17T16:53:12.846Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "We have noticed some API requests begin to succeed again. We are not sure if this is temporary (or if it applies to 100% of API requests) but we are monitoring the situation.",
"created_at": "2025-12-17T01:10:54.185Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-17T01:10:54.185Z",
"id": "wvxrpmqgwzth",
"incident_id": "mtbv013cqtr7",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-17T01:10:54.185Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "major_outage",
"old_status": "degraded_performance"
}
],
"body": "The issue with the Schoology API has been escalated to \"major\". At this point, it appears that close to 100% of API requests to Schoology are failing. This includes SSO, syncs, coursework, and grade requests.\n\nWe are temporarily pausing all Schoology syncs in order to free up queue space for other sources.",
"created_at": "2025-12-16T14:56:14.269Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-16T14:56:14.269Z",
"id": "nfyvc4k10mqc",
"incident_id": "mtbv013cqtr7",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-16T14:56:14.269Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have a deployed an update to our roster sync code to try to workaround this ongoing issue. This should help roster data begin to flow again. This change does not impact the assignments, grades, and coursework APIs, so you may still experience random, intermittent 403 errors from Schoology.",
"created_at": "2025-12-11T21:14:51.195Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-12-11T21:14:51.195Z",
"id": "f9pxhm7vy378",
"incident_id": "mtbv013cqtr7",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-11T21:14:51.195Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "b4c7k0s4wtrx",
"name": "Data Sources - Schoology",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating reports of an unusual number of School API errors. This may affect any Schoology integrations, including ones where you are only accessing class roster data via Edlink. We are receiving random (intermittent) 403 errors from AWS Cloudfront (Schoology's CDN provider). These errors do not actually correlate to permissions issues in Schoology as they appear to be random and will succeed after several retries or a significant time delay.\n\nOur system, by default, replays external requests up to 5 times, but they are in relatively short succession so this retry logic does not provide a workaround in this situation.\n\nRoster data is all still available (the Edlink API itself still returns data as usual), but Edlink may not receive subsequent roster updates from Schoology during this time.\n\nWe are attempting to work with Schoology to help them determine the root cause.",
"created_at": "2025-12-09T17:41:43.577Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-12-09T17:41:43.000Z",
"id": "8sfnk913gqmc",
"incident_id": "mtbv013cqtr7",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-12-09T17:42:21.110Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2025-12-17T01:10:54.185Z",
"name": "Schoology API Elevated Error Rates",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": "24",
"resolved_at": "2025-12-23T05:14:49.181Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/t4mswhhnc73t",
"started_at": "2025-12-09T17:41:43.000Z",
"status": "resolved",
"updated_at": "2025-12-23T05:14:49.200Z"
},
{
"components": [
{
"created_at": "2020-08-25T06:25:23.375Z",
"description": "The primary Edlink API servers.",
"group_id": "lvchkfg2wdtn",
"id": "lghg3n9cd6jj",
"name": "Edlink API",
"page_id": "90lv07q7m45f",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.209Z"
},
{
"created_at": "2020-08-25T06:25:23.386Z",
"description": "The website frontend where developers manage their applications and schools manage their data sources.",
"group_id": "lvchkfg2wdtn",
"id": "ncqf81v9p42j",
"name": "Edlink Dashboard",
"page_id": "90lv07q7m45f",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.225Z"
}
],
"created_at": "2025-11-18T18:06:11.000Z",
"id": "9j49nr0tbq0f",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "Everything has returned to normal and the job queue is back down close to zero.",
"created_at": "2025-11-19T22:10:33.484Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-11-19T22:10:33.484Z",
"id": "jngphgcxnhyc",
"incident_id": "9j49nr0tbq0f",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-11-19T22:10:33.484Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "Update 3:02pm CT:\n\nWe are still working on the core issue, but we are beginning to resume background operations. Currently, we are running at partial capacity for enrichment, materialization, queue, and sync jobs.\n\nManually-initiated jobs (i.e. jobs started by clicking a button in the dashboard) will be prioritized in the queue.\n\nPlease expect longer than usual queue times on sync jobs.",
"created_at": "2025-11-18T21:04:20.042Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-11-18T21:04:20.042Z",
"id": "93qg8wlj6zhw",
"incident_id": "9j49nr0tbq0f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-11-18T21:04:20.042Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently experiencing a database pressure issue in our US Central region (which holds most of our data). In an abundance of caution, while we sort out the issue, we have voluntarily disabled some nonessential background services, including:\n\n- Scheduled syncs\n- Data enrichments\n- Materializations\n- Queued jobs \u0026 tasks\n\n***Core platform functionality (API, SSO, Dashboard, Widgets, etc.) remains fully operational.***\n\nWhat’s Happening:\n\nAn internal database counter exceeded its safe threshold. When this number gets too high, the database temporarily prevents new write operations. In order to slow the growth of this counter while we fix the issue, we've reduced the number of workloads on the database.\n\nWhat We’ve Done So Far:\n\n- Identified the cause\n- Temporarily paused non-critical syncing and materialization tasks during recovery\n- We are manually running certain database maintenance processes to help the database recover more quickly\n\nCurrent Impact:\n\n- Only background data tasks are delayed (intentionally)\n- No data has been lost\n- Primary application operations continue as normal\n\nNext Steps:\n\nWe will monitor the configuration update. If it's enough, we'll bring non-essential services back online. If it isn't, we may need to restart the database which will happen at non-peak time and require \u003c 10 minutes of downtime. \n\nNext Update:\n\nWe will provide another update at the end of the day, or if the situation resolves or escalates, whichever occurs first.",
"created_at": "2025-11-18T18:21:11.323Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-11-18T18:06:11.000Z",
"id": "ckmsy6f509kr",
"incident_id": "9j49nr0tbq0f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-11-18T21:02:12.039Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Database Write Limitation",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-11-19T22:10:33.484Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/5t753l0897m8",
"started_at": "2025-11-18T18:06:11.000Z",
"status": "resolved",
"updated_at": "2025-11-19T22:10:33.502Z"
},
{
"components": [
{
"created_at": "2020-08-25T06:25:23.375Z",
"description": "The primary Edlink API servers.",
"group_id": "lvchkfg2wdtn",
"id": "lghg3n9cd6jj",
"name": "Edlink API",
"page_id": "90lv07q7m45f",
"position": 1,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.209Z"
},
{
"created_at": "2020-08-25T06:25:23.386Z",
"description": "The website frontend where developers manage their applications and schools manage their data sources.",
"group_id": "lvchkfg2wdtn",
"id": "ncqf81v9p42j",
"name": "Edlink Dashboard",
"page_id": "90lv07q7m45f",
"position": 2,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-01T15:52:54.225Z"
},
{
"created_at": "2023-05-17T04:49:21.834Z",
"description": null,
"group_id": null,
"id": "gl06w2px85vx",
"name": "Google Cloud Platform Google Cloud Storage",
"page_id": "90lv07q7m45f",
"position": 6,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-10-09T00:17:17.547Z"
},
{
"created_at": "2025-06-12T18:57:56.036Z",
"description": null,
"group_id": "nf4g7l76k9s4",
"id": "p9tpw4bstfxn",
"name": "SFTP",
"page_id": "90lv07q7m45f",
"position": 9,
"showcase": true,
"start_date": "2025-06-12T00:00:00.000Z",
"status": "operational",
"updated_at": "2025-06-12T22:30:39.457Z"
}
],
"created_at": "2025-06-12T18:59:42.722Z",
"id": "0c98lmmz9qly",
"impact": "major",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "p9tpw4bstfxn",
"name": "Data Sources - SFTP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "gl06w2px85vx",
"name": "Google Cloud Platform Google Cloud Storage",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "Google services are back up and running.",
"created_at": "2025-06-12T22:30:39.495Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-06-12T22:30:39.495Z",
"id": "v5w0lhbj5r7v",
"incident_id": "0c98lmmz9qly",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-06-12T22:30:39.495Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "p9tpw4bstfxn",
"name": "Data Sources - SFTP",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "gl06w2px85vx",
"name": "Google Cloud Platform Google Cloud Storage",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "A fix has been implemented by Google Cloud and we are monitoring the results. We are now able to connect to the Edlink SFTP servers and API requests are stable.",
"created_at": "2025-06-12T20:40:05.798Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-06-12T20:40:05.000Z",
"id": "4n7f6w0y1pyx",
"incident_id": "0c98lmmz9qly",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-06-12T20:41:03.365Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "p9tpw4bstfxn",
"name": "Data Sources - SFTP",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "gl06w2px85vx",
"name": "Google Cloud Platform Google Cloud Storage",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "Here's a link to the Google Cloud page showing their status: https://status.cloud.google.com/regional/americas",
"created_at": "2025-06-12T19:00:35.455Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-06-12T19:00:35.455Z",
"id": "vzrfwnd28khn",
"incident_id": "0c98lmmz9qly",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-06-12T19:00:35.455Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "lghg3n9cd6jj",
"name": "Edlink Core Systems - Edlink API",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "ncqf81v9p42j",
"name": "Edlink Core Systems - Edlink Dashboard",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "p9tpw4bstfxn",
"name": "Data Sources - SFTP",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "gl06w2px85vx",
"name": "Google Cloud Platform Google Cloud Storage",
"new_status": "major_outage",
"old_status": "operational"
}
],
"body": "Currently Google Cloud Storage is down globally. This is an insanely rare event. Edlink's API is currently functional, but some requests may fail. The SFTP transfer (Edlink's SFTP servers) are currently down. So new roster data can't be transferred into Edlink via SFTP. Existing roster data is unaffected. Functions of the Edlink Dashboard that involve SFTP (i.e. CSV mapping) may be affected. In some instances, when loading the Administrator Onboarding Flow, Edlink's clients' logos / icons may not load.",
"created_at": "2025-06-12T18:59:42.863Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2025-06-12T18:59:42.863Z",
"id": "y9d5522xh0lt",
"incident_id": "0c98lmmz9qly",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-06-12T18:59:42.863Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2025-06-12T20:40:05.000Z",
"name": "Google Cloud Storage Outage",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": "[]",
"resolved_at": "2025-06-12T22:30:39.495Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/rmk02p7ntfcn",
"started_at": "2025-06-12T18:59:42.707Z",
"status": "resolved",
"updated_at": "2025-06-12T22:30:39.512Z"
},
{
"components": [],
"created_at": "2025-03-05T22:16:00.000Z",
"id": "qhf86dkldv3x",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": null,
"body": "Issue: \n- We updated our Kubernetes HTTP route configuration at approximately 4:16pm CT.\n- This route configuration was slightly out of date and contained an invalid \"catchall\" route that used to route traffic to the Edlink Dashboard.\n- The issue was identified and fixed at approximately 4:26pm CT.\n\nImpact: \n- This issue only affected the Edlink Dashboard, not our API, SSO, Widgets, Upload, Data Pipeline or other related services.\n- It was likely not noticed by many users as it would only have impacted people who were attempting to load the dashboard in a new tab or window.\n- If you already had a dashboard page open, it would have continued to work uninterrupted (as our API was unaffected).",
"created_at": "2025-03-05T22:41:38.604Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2025-03-05T22:16:00.000Z",
"id": "5vm581ltdny0",
"incident_id": "qhf86dkldv3x",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2025-03-05T22:41:38.604Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Dashboard not loading ~10 minutes",
"page_id": "90lv07q7m45f",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2025-03-05T22:16:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/s2g81qfjtpng",
"started_at": "2025-03-05T22:41:38.549Z",
"status": "resolved",
"updated_at": "2025-03-05T22:41:52.351Z"
}
],
"page": {
"id": "90lv07q7m45f",
"name": "Edlink",
"url": "https://status.ed.link"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}