Snapshot 21717
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"p46xbw6694qj",
"l0n1kl7997c7"
],
"created_at": "2020-05-01T17:47:03.417Z",
"description": null,
"group_id": null,
"id": "18w0fv6lbv3t",
"name": "US Production",
"page_id": "6wx3thb9c4cs",
"position": 1,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2023-05-29T11:06:39.511Z"
},
{
"components": [
"znj98w74cqxf",
"w8mybjk8l866"
],
"created_at": "2023-05-29T13:27:37.422Z",
"description": null,
"group_id": null,
"id": "hrjfhyf2nwjz",
"name": "Canada Production",
"page_id": "6wx3thb9c4cs",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2023-05-29T14:50:44.681Z"
}
],
"incidents": [
{
"components": [],
"created_at": "2023-05-29T11:27:33.000Z",
"id": "4h7lb54n9vsk",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "We are currently investigating this issue.",
"created_at": "2023-05-29T11:27:58.105Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2023-05-29T11:27:33.000Z",
"id": "s82gm59p52sy",
"incident_id": "4h7lb54n9vsk",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-29T11:27:58.105Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Test Incident",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2023-05-29T11:27:33.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/mp21qcm2y8lj",
"started_at": "2023-05-29T11:27:58.078Z",
"status": "resolved",
"updated_at": "2023-05-29T11:27:58.112Z"
},
{
"components": [],
"created_at": "2023-05-05T12:31:54.591Z",
"id": "80l5km6llmh1",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "wql06xh6358z",
"name": "Private Production - UCSF S3",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "This incident has been resolved.",
"created_at": "2023-05-05T12:32:36.059Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2023-05-05T12:32:36.059Z",
"id": "48hlzwhr0xnf",
"incident_id": "80l5km6llmh1",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-05T12:32:36.059Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "wql06xh6358z",
"name": "Private Production - UCSF S3",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "This is a test incident",
"created_at": "2023-05-05T12:31:54.647Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2023-05-05T12:31:54.647Z",
"id": "5jhd7rzcszwj",
"incident_id": "80l5km6llmh1",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-05T12:31:54.647Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Test Incident",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2023-05-05T12:32:36.059Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/pf8pzdpkxnpr",
"started_at": "2023-05-05T12:31:54.585Z",
"status": "resolved",
"updated_at": "2023-05-05T12:32:36.077Z"
},
{
"components": [],
"created_at": "2023-05-03T11:46:33.985Z",
"id": "krzf81f5h1zz",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "US Shared Production - Login Page",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "p7gmdrfwbh75",
"name": "Private Production - UCSF Login Page",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "pxcxrqrkz5dy",
"name": "Canada - Canada Login Page",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2023-05-04T22:39:52.750Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2023-05-04T22:39:52.750Z",
"id": "cwykckydt5r9",
"incident_id": "krzf81f5h1zz",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-04T22:39:52.750Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "US Shared Production - Login Page",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "p7gmdrfwbh75",
"name": "Private Production - UCSF Login Page",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "pxcxrqrkz5dy",
"name": "Canada - Canada Login Page",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Greetings!\n\nThe new Ovrture platform is here and live and working like a dream! Thank you for your patience.\n\nTo get started, visit your organization's url to log in with your existing credentials. You will need to authorize your device the first time you log in to the new platform. From there, you can carry on as usual and explore what's new and different.\n\nYour feedback makes the platform and our support articles stronger, so please share your thoughts. Team leaders, please share this information with your staff. Feel free to email us at support@ovrture.com or reach out directly to Gideon or Sarah\n\nTHANK YOU!\n\nOnward,\n\nThe Ovrture Team",
"created_at": "2023-05-04T22:39:30.656Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2023-05-04T22:39:30.656Z",
"id": "xb38nmtr19ww",
"incident_id": "krzf81f5h1zz",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-04T22:39:30.656Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "US Shared Production - Login Page",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "p7gmdrfwbh75",
"name": "Private Production - UCSF Login Page",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "pxcxrqrkz5dy",
"name": "Canada - Canada Login Page",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "We are continuing to push!",
"created_at": "2023-05-04T15:25:03.210Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2023-05-04T15:25:03.000Z",
"id": "tsgcln1n10m9",
"incident_id": "krzf81f5h1zz",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-04T15:32:46.451Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "US Shared Production - Login Page",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "p7gmdrfwbh75",
"name": "Private Production - UCSF Login Page",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
},
{
"code": "pxcxrqrkz5dy",
"name": "Canada - Canada Login Page",
"new_status": "under_maintenance",
"old_status": "under_maintenance"
}
],
"body": "Good morning everyone,\n\nOur team is hard at work migrating all your data and systems to the new platform, and it turns out migrating 10 million assets(!) takes a really long time. What an incredible amount of work that represents for all of you!\n\nWe are projecting that your access will be restored by noon (EDT) today. We are sorry for the disruption and are doing everything we can to get you back into the platform.\n\nWe will write again when we're up and running. Thank you for your patience and your partnership.\n\nOnward,\n\nThe Ovrture Team",
"created_at": "2023-05-04T13:19:28.047Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2023-05-04T13:19:28.047Z",
"id": "z1bpnhqdcfsw",
"incident_id": "krzf81f5h1zz",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-04T13:19:28.047Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "US Shared Production - Login Page",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "p7gmdrfwbh75",
"name": "Private Production - UCSF Login Page",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "pxcxrqrkz5dy",
"name": "Canada - Canada Login Page",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "After more than a year of heavy effort, we have everything in place to push our new platform to the production environment. Please note this will take no more than 24 hours to complete, and the availability of existing sites and reports will not be disrupted.\n\nWhat does this mean for you? Beginning 11pm ET on May 2, through the early morning hours of May 4, no users or admins will have login access to the backend of the platform.\n \nNo later than the morning of May 4, we will email you with confirmation that we're live and reminders of all the ways we're here to support you during this transition. Team leaders, please share this information with your staff.\n\nThank you for your patience and your partnership.\n\nOnward,\n\nThe Ovrture Team",
"created_at": "2023-05-03T11:46:34.078Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2023-05-03T11:46:34.078Z",
"id": "51xxw4t006b8",
"incident_id": "krzf81f5h1zz",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2023-05-03T11:46:34.078Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2023-05-03T11:46:34.078Z",
"name": "Platform Downtime // New Ovrture Version Launch",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2023-05-04T22:39:52.750Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/v26kbtngrfgs",
"started_at": "2023-05-04T15:25:03.000Z",
"status": "resolved",
"updated_at": "2023-05-04T22:39:52.765Z"
},
{
"components": [],
"created_at": "2022-04-07T13:06:59.141Z",
"id": "ngkzbvmkkp8j",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "US Shared Production - Login Page",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "p7gmdrfwbh75",
"name": "Private Production - UCSF Login Page",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved.",
"created_at": "2022-04-07T13:46:30.677Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2022-04-07T13:46:30.677Z",
"id": "3d3qjg0hq5vr",
"incident_id": "ngkzbvmkkp8j",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2022-04-07T13:46:30.677Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "US Shared Production - Login Page",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "p7gmdrfwbh75",
"name": "Private Production - UCSF Login Page",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2022-04-07T13:06:59.234Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2022-04-07T13:06:59.234Z",
"id": "zfwkh3mhdw9l",
"incident_id": "ngkzbvmkkp8j",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2022-04-07T13:06:59.234Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Platform Login Page Navigating to 404",
"page_id": "6wx3thb9c4cs",
"postmortem_body": "Good morning Ovrture users,\n\nThis document details the cause and events that occurred during Ovrture’s incident on April 7, 2022 as well as the steps we are taking to mitigate the impact of future incidents like this one in the future.\n\nOn April 7, 2022, there was a partial outage impacting the Ovrture platform login page. This outage occurred from 8:40am - 9:50am EST.\n\nWhen the issue was found, the ODT notified DevOps and the Engagement team of the issue via Slack. ODT, DevOps, and the Engagement team connected via Zoom to diagnose the issue.\n\nEngagement helped investigate the issue and determined that the client using the private production environment and clients using the shared production environment and using [mysite.ovrture.com](http://mysite.ovrture.com) instead of their unique URL were the only tenants being affected. Clients using their own unique URLs and clients using the Canada production environment did not have any issues. No live donor/prospect sites or reports were being affected. Engagement then updated the Ovrture Status Page accordingly. \n\nODT and DevOps continued to investigate the issue to determine what could have caused the incident. Through collaborative testing, they were able to pinpoint and fix the issue.\n\nThe confirmed root cause was a configuration error that caused the application to malfunction in the way that it handled some requests. We have rolled back the code dependent on this configuration and will build a new solution and test cases that evaluate the configuration prior to production code deployment.\n\nOnward,\n\nThe Ovrture Team",
"postmortem_body_last_updated_at": "2022-04-08T12:56:27.316Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2022-04-08T12:56:27.316Z",
"reminder_intervals": null,
"resolved_at": "2022-04-07T13:46:30.677Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/qmhjvwj8250d",
"started_at": "2022-04-07T13:06:59.130Z",
"status": "resolved",
"updated_at": "2022-04-08T12:56:27.329Z"
},
{
"components": [],
"created_at": "2022-03-02T12:02:48.097Z",
"id": "jzbb9zbq3f38",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This issue has been resolved.",
"created_at": "2022-03-03T12:40:16.283Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2022-03-03T12:40:16.283Z",
"id": "ntry3fqs24tm",
"incident_id": "jzbb9zbq3f38",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2022-03-03T12:40:16.283Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "There is an error occurring with Vimeo which is affecting the capability of consistently uploading video files into Ovrture. We are currently working with Vimeo to resolve the issue.",
"created_at": "2022-03-02T12:02:48.135Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2022-03-02T12:02:48.135Z",
"id": "0bwsp9jntl60",
"incident_id": "jzbb9zbq3f38",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2022-03-02T12:02:48.135Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Degraded Performance // Video Asset Upload",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2022-03-03T12:40:16.283Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/v672xb1brpff",
"started_at": "2022-03-02T12:02:48.090Z",
"status": "resolved",
"updated_at": "2022-03-03T12:40:16.297Z"
},
{
"components": [],
"created_at": "2021-09-20T11:47:38.675Z",
"id": "tq23bw3n42gn",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "q87gcf3wn03s",
"name": "US Shared Production - Microsites",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "09szgm5xmx6j",
"name": "Private Production - UCSF Microsites",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2021-09-22T16:05:05.033Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2021-09-22T16:05:05.033Z",
"id": "wz1v2fnz2d7y",
"incident_id": "tq23bw3n42gn",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2021-09-22T16:05:05.033Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "q87gcf3wn03s",
"name": "US Shared Production - Microsites",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "09szgm5xmx6j",
"name": "Private Production - UCSF Microsites",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently experiencing an issue in regard to the Printbridge functionality within the platform. This issue has been identified and a fix is being implemented.",
"created_at": "2021-09-20T11:47:38.775Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2021-09-20T11:47:38.775Z",
"id": "0rqj2lb7nm3l",
"incident_id": "tq23bw3n42gn",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2021-09-20T11:47:38.775Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Degraded Performance // Printbridge Functionality",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2021-09-22T16:05:05.033Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/gcjkpgtxm2fv",
"started_at": "2021-09-20T11:47:38.665Z",
"status": "resolved",
"updated_at": "2021-09-22T16:05:05.051Z"
},
{
"components": [],
"created_at": "2021-04-09T12:55:12.423Z",
"id": "sjl94tvg0spr",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2021-05-20T12:11:24.838Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2021-05-20T12:11:24.838Z",
"id": "fdw4vy9ssyys",
"incident_id": "sjl94tvg0spr",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2021-05-20T12:11:24.838Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are currently experiencing an issue in regard to the Reports Printbridge Exports functionality. This issue has been identified and a fix is being implemented. For the time being, we have disabled the Exports functionality. This will not affect the Printbridge functionality on a singular report level or for a donor.",
"created_at": "2021-04-09T12:55:12.457Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2021-04-09T12:55:12.457Z",
"id": "dh0l7xl6zc13",
"incident_id": "sjl94tvg0spr",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2021-04-09T12:55:12.457Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Degraded Performance // Printbridge Functionality",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2021-05-20T12:11:24.838Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/2j8zndbjk6fl",
"started_at": "2021-04-09T12:55:12.417Z",
"status": "resolved",
"updated_at": "2021-05-20T12:11:24.853Z"
},
{
"components": [],
"created_at": "2020-10-27T13:31:13.724Z",
"id": "vsy86vbf9qyd",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "The scheduled maintenance has been completed.",
"created_at": "2020-10-29T13:10:35.039Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2020-10-29T13:10:35.039Z",
"id": "5115hhxwtg55",
"incident_id": "vsy86vbf9qyd",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-10-29T13:10:35.039Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2020-10-27T13:32:19.978Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2020-10-27T13:32:19.978Z",
"id": "7tcw6ttc1x27",
"incident_id": "vsy86vbf9qyd",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-10-27T13:32:19.978Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The Exports FY Reports Printbridge Functionality is undergoing maintenance and will be disabled until completed.",
"created_at": "2020-10-27T13:31:13.772Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2020-10-27T13:31:13.772Z",
"id": "2py8xly4xnl4",
"incident_id": "vsy86vbf9qyd",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-10-27T13:31:13.772Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Maintenance // Exports - FY Reports Printbridge",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2020-10-29T13:10:35.039Z",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2020-10-27T13:32:00.000Z",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2020-10-31T17:32:00.000Z",
"shortlink": "http://stspg.io/ttkcnt6plq9f",
"started_at": "2020-10-27T13:31:13.719Z",
"status": "completed",
"updated_at": "2020-10-29T13:10:35.057Z"
},
{
"components": [],
"created_at": "2020-10-19T14:23:47.797Z",
"id": "4lgffns0btjb",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2020-10-19T14:41:23.539Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2020-10-19T14:41:23.539Z",
"id": "7798h5802hc6",
"incident_id": "4lgffns0btjb",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-10-19T14:41:23.539Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are currently experiencing an issue in regard to the Vimeo Video Player within Ovrture. This issue has been identified, a fix has been implemented, and results are being monitored. \n\nFor more information: https://www.vimeostatus.com/#",
"created_at": "2020-10-19T14:23:47.843Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2020-10-19T14:23:47.843Z",
"id": "5ys73ggcxwfk",
"incident_id": "4lgffns0btjb",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-10-19T14:23:47.843Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2020-10-19T14:23:47.843Z",
"name": "Degraded Performance // Vimeo Video Player",
"page_id": "6wx3thb9c4cs",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2020-10-19T14:41:23.539Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "http://stspg.io/ybsrwsbrspl4",
"started_at": "2020-10-19T14:23:47.788Z",
"status": "resolved",
"updated_at": "2020-10-19T14:41:23.553Z"
},
{
"components": [],
"created_at": "2020-01-28T12:50:58.467Z",
"id": "60hd7hf7kqh9",
"impact": "critical",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "Login Page",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czpw4cxz9r3q",
"name": "Database",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q87gcf3wn03s",
"name": "Microsites",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2020-01-28T14:14:17.148Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2020-01-28T14:14:17.148Z",
"id": "xc4hfk743nnj",
"incident_id": "60hd7hf7kqh9",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-01-28T14:14:17.148Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "Login Page",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "czpw4cxz9r3q",
"name": "Database",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "q87gcf3wn03s",
"name": "Microsites",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "This incident has been resolved. You can now use the platform as usual. Thank you for your patience as this was resolved. We will continue to monitor the situation and will publish a postmortem in the next 24 hours.",
"created_at": "2020-01-28T14:13:32.269Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2020-01-28T14:13:32.269Z",
"id": "n37m2wl80vbp",
"incident_id": "60hd7hf7kqh9",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-01-28T14:13:32.269Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "Login Page",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "czpw4cxz9r3q",
"name": "Database",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "q87gcf3wn03s",
"name": "Microsites",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are continuing to investigate the issue. There are some live sites/reports that are operational at this point.",
"created_at": "2020-01-28T13:19:56.148Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2020-01-28T13:19:56.148Z",
"id": "mw4vrl2w6pym",
"incident_id": "60hd7hf7kqh9",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-01-28T13:19:56.148Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "Login Page",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "czpw4cxz9r3q",
"name": "Database",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "q87gcf3wn03s",
"name": "Microsites",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "We are continuing to investigate the issue. Live sites/reports are operational.",
"created_at": "2020-01-28T13:17:58.934Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2020-01-28T13:17:58.934Z",
"id": "xsd1xkd2bqhb",
"incident_id": "60hd7hf7kqh9",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-01-28T13:17:58.934Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "Login Page",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "czpw4cxz9r3q",
"name": "Database",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "q87gcf3wn03s",
"name": "Microsites",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "We are continuing to investigate the issue. Live sites/reports are operational.",
"created_at": "2020-01-28T13:15:46.108Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2020-01-28T13:15:46.108Z",
"id": "rt1kl97kgy4v",
"incident_id": "60hd7hf7kqh9",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-01-28T13:15:46.108Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "96x5sgdtqh9y",
"name": "Login Page",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "czpw4cxz9r3q",
"name": "Database",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "q87gcf3wn03s",
"name": "Microsites",
"new_status": "major_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating the issue.",
"created_at": "2020-01-28T12:50:58.543Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2020-01-28T12:50:58.543Z",
"id": "yzhf7s4ymk9m",
"incident_id": "60hd7hf7kqh9",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2020-01-28T12:50:58.543Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Major Outage",
"page_id": "6wx3thb9c4cs",
"postmortem_body": "Good afternoon Ovrture users,\n\nThis document details the cause and events occurring immediately after Ovrture’s incident on January 28, 2020 as well as the steps we are taking to mitigate the impact of future outages like this one in the future.\n\nOn January 28, 2020, there was a major outage regarding microsites, login pages, and the database. This outage occurred from 6:00 AM. to 9:00 AM.\n\nThe Ovrture development team was the first to notice this issue through notification from the Ovrture status page. Ovrture investigated the issue and notified the Ovrture development operations team, who began looking into the outage immediately. It was discovered that the outages were related to a race condition in the reboot process between EFS, Tomcat, and Ubuntu 18's new security separation layers. These caused the reboot sequence to be unpredictable. When regular nightly updates were applied last night, the updates themselves were actually okay, it was the reboot that hurt. Subsequent reboots also had the same issue.\n\nThe underlying issues regarding this have been fixed and the race condition on reboot was resolved. That being said, reboots are now safe.\n\nThings we will improve to make sure issues like this do not happen again in the future:\n\n1. We will be improving the production environment's infrastructure to ensure this specific issue does not happen again.\n2. We will reduce downtime by developing an \"off-hours\" notification protocol.\n3. We will be auditing and possibly adjusting our Incident Response Plan \\(IRP\\) to provide better early warning, coordination, and resolution of issues.\n4. To prevent slowdowns during debugging of future incidents, our Nagios site is now accessible from all of the Ovrture VPNs.\n\nWe are very sorry for any inconvenience this incident may have caused on January 28 and we will continue to work hard to make sure something like this doesn’t happen again.\n\nOnward,\n\nGideon and the Ovrture Team",
"postmortem_body_last_updated_at": "2020-02-09T08:44:47.316Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2020-01-28T22:07:56.066Z",
"reminder_intervals": null,
"resolved_at": "2020-01-28T14:14:17.148Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "http://stspg.io/bn8nbvd3gs8v",
"started_at": "2020-01-28T12:50:58.461Z",
"status": "resolved",
"updated_at": "2020-02-09T08:44:47.321Z"
}
],
"page": {
"id": "6wx3thb9c4cs",
"name": "Ovrture",
"url": "https://status.ovrture.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}