Snapshot 24759
Normalized text
Scripts and page chrome removed; this is what change detection compares.
{
"data": {
"attributes": {
"aggregate_state": "operational",
"announcement": null,
"company_name": "Hatchet",
"company_url": "https://hatchet.run",
"contact_url": "mailto:contact@hatchet.run",
"created_at": "2024-12-02T17:17:42.300Z",
"custom_domain": "status.hatchet.run",
"logo_remote_url": null,
"logo_url": "https://d1lppblt9t2x15.cloudfront.net/logos/0f7bb28a2e044dec7ac360534525bc0a.png",
"subdomain": "hatchet",
"timezone": "Eastern Time (US \u0026 Canada)",
"updated_at": "2026-09-30T19:18:21.646Z"
},
"id": "202555",
"relationships": {
"resources": {
"data": [
{
"id": "8498343",
"type": "status_page_resource"
},
{
"id": "8498341",
"type": "status_page_resource"
},
{
"id": "8498342",
"type": "status_page_resource"
}
]
},
"sections": {
"data": [
{
"id": "240438",
"type": "status_page_section"
},
{
"id": "240439",
"type": "status_page_section"
}
]
},
"status_reports": {
"data": [
{
"id": "1027970",
"type": "status_report"
},
{
"id": "984451",
"type": "status_report"
},
{
"id": "976671",
"type": "status_report"
},
{
"id": "940638",
"type": "status_report"
},
{
"id": "940181",
"type": "status_report"
}
]
}
},
"type": "status_page"
},
"included": [
{
"attributes": {
"name": "API",
"position": 0
},
"id": "240438",
"type": "status_page_section"
},
{
"attributes": {
"name": "Website",
"position": 1
},
"id": "240439",
"type": "status_page_section"
},
{
"attributes": {
"availability": 1,
"explanation": "",
"position": 0,
"public_name": "cloud.hatchet.run",
"resource_id": 2832016,
"resource_type": "Monitor",
"status": "operational",
"status_history": [
{
"day": "2026-09-01",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-02",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-03",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-04",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-05",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-06",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-07",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-08",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-09",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-10",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-11",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-12",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-13",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-14",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-15",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-16",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-17",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-18",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-19",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-20",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-21",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-22",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-23",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-24",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-25",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-26",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-27",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-28",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-29",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-30",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
}
],
"status_page_section_id": 240438
},
"id": "8498343",
"type": "status_page_resource"
},
{
"attributes": {
"availability": 1,
"explanation": "",
"position": 1,
"public_name": "Hatchet API",
"resource_id": 2832028,
"resource_type": "Monitor",
"status": "not_monitored",
"status_history": [
{
"day": "2026-09-01",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-02",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-03",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-04",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-05",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-06",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-07",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-08",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-09",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-10",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-11",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-12",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-13",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-14",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-15",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-16",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-17",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-18",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-19",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-20",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-21",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-22",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-23",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-24",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-25",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-26",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-27",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-28",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-29",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-30",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
}
],
"status_page_section_id": 240438
},
"id": "8498341",
"type": "status_page_resource"
},
{
"attributes": {
"availability": 1,
"explanation": "",
"position": 0,
"public_name": "hatchet.run",
"resource_id": 2832030,
"resource_type": "Monitor",
"status": "operational",
"status_history": [
{
"day": "2026-09-01",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-02",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-03",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-04",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-05",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-06",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-07",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-08",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-09",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-10",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-11",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-12",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-13",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-14",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-15",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-16",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-17",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-18",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-19",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-20",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-21",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-22",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-23",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-24",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-25",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-26",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-27",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-28",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-29",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
},
{
"day": "2026-09-30",
"downtime_duration": 0,
"maintenance_duration": 0,
"status": "operational"
}
],
"status_page_section_id": 240439
},
"id": "8498342",
"type": "status_page_resource"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8498341"
},
{
"status": "degraded",
"status_page_resource_id": "8498343"
}
],
"message": "On August 22, 2026, starting around 12:00 UTC, the shared us-west-2 cloud environment experienced elevated database CPU and elevated scheduling latency. A high volume of concurrent workflow runs with stacked concurrency controls on a shared shard caused lock contention and delayed task processing for tenants on that environment. We took action to throttle the work driving saturation and constraining scheduler capacity while the system recovered. Database load and scheduling latency returned to normal later the same afternoon. No data loss occurred.",
"notify_subscribers": false,
"published_at": "2026-08-22T16:24:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5637688",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"message": "On August 22, 2026, starting around 12:00 UTC, one shared shard in the us-west-2 cloud environment experienced elevated database CPU and elevated scheduling latency. A high volume of concurrent workflow runs with stacked concurrency controls caused lock contention and delayed task processing for tenants on that shard. \n\n**Other shared shards in us-west-2, shared environments in other regions, and dedicated customer shards were not affected.**\n\nTook steps to throttle the workload and reduce contention. Database load and scheduling latency on the affected shard returned to normal later the same afternoon. No data loss occurred.",
"notify_subscribers": false,
"published_at": "2026-08-22T21:27:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5637689",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-08-22T16:24:00.000Z",
"title": "[us-west-2] Elevated latency on us-west-2 cloud"
},
"id": "1027970",
"relationships": {
"status_updates": {
"data": [
{
"id": "5637688",
"type": "status_update"
},
{
"id": "5637689",
"type": "status_update"
}
]
}
},
"type": "status_report"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8498341"
},
{
"status": "degraded",
"status_page_resource_id": "8498343"
}
],
"message": "We have pushed some configuration changes and are seeing initial signs of improvement. We will continue to monitor the situation.",
"notify_subscribers": false,
"published_at": "2026-07-29T08:20:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5485609",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"message": "We saw elevated database load on our shared production environment beginning around 05:00 UTC, which caused some multi-step workflows to experience scheduling delays and increased API/dashboard latency for affected tenants.\n\nWe reduced connection pressure on the shared database and latency has returned to normal. We’re continuing to monitor closely.",
"notify_subscribers": false,
"published_at": "2026-07-29T08:27:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5485680",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8498341"
},
{
"status": "degraded",
"status_page_resource_id": "8498343"
}
],
"message": "We're monitoring increased latency on a single shared cloud shard in our us-west-2 region experienced elevated database lock contention, which is causing higher API and scheduling latency and some intermittent task-dispatch delays for tenants on that shard. Tasks are continuing to be ingested and assigned.\n",
"notify_subscribers": false,
"published_at": "2026-07-29T07:56:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5485449",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-07-29T07:56:00.000Z",
"title": "[us-west-2] Increased latency on single shard"
},
"id": "984451",
"relationships": {
"status_updates": {
"data": [
{
"id": "5485609",
"type": "status_update"
},
{
"id": "5485680",
"type": "status_update"
},
{
"id": "5485449",
"type": "status_update"
}
]
}
},
"type": "status_report"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"message": "Between approximately 05:00–10:00 UTC on July 24, the shared cloud shard in our us-west-2 region experienced elevated database lock contention, which caused higher API and scheduling latency and some intermittent task-dispatch delays for tenants on that shard.\n\nThis was isolated to a single shard in us-west-2 (cloud). Other regions and shards were not affected. We have no evidence of dropped workload, and all assignments were eventually completed.\n\nWe are continuing to investigate and monitor.",
"notify_subscribers": false,
"published_at": "2026-07-24T11:14:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5456483",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-07-24T11:14:00.000Z",
"title": "Elevated latency on US West (Oregon) cloud shard"
},
"id": "976671",
"relationships": {
"status_updates": {
"data": [
{
"id": "5456483",
"type": "status_update"
}
]
}
},
"type": "status_report"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8498341"
},
{
"status": "degraded",
"status_page_resource_id": "8498342"
},
{
"status": "degraded",
"status_page_resource_id": "8498343"
}
],
"message": "We're monitoring some degradation in performance on the Hatchet dashboard and REST APIs, and the scheduler, and are working on a fix.\n\nWe believe about 30% of tenants are affected",
"notify_subscribers": false,
"published_at": "2026-07-01T14:43:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5318390",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8498341"
},
{
"status": "degraded",
"status_page_resource_id": "8498342"
},
{
"status": "degraded",
"status_page_resource_id": "8498343"
}
],
"message": "We're continuing to monitor affected gRPC APIs as well. We're seeing some reports of `StatusCode.UNAVAILABLE` being returned to SDKs from the API",
"notify_subscribers": false,
"published_at": "2026-07-01T15:00:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5318472",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498342"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"message": "We're continuing to monitor the fix. We've been seeing significant recovery since about 35 minutes ago (15:15 UTC).\n\nIf you're seeing tasks that are stuck during execution, cancelling and replaying will likely get them moving again.",
"notify_subscribers": false,
"published_at": "2026-07-01T15:50:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5318678",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8498341"
},
{
"status": "degraded",
"status_page_resource_id": "8498342"
},
{
"status": "degraded",
"status_page_resource_id": "8498343"
}
],
"message": "We've applied a candidate fix and are seeing some services begin to recover, and are continuing to monitor",
"notify_subscribers": false,
"published_at": "2026-07-01T15:15:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5318537",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498342"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-07-01T14:43:00.000Z",
"title": "[us-west-2] Increased Latency on Dashboard and APIs"
},
"id": "940638",
"relationships": {
"status_updates": {
"data": [
{
"id": "5318390",
"type": "status_update"
},
{
"id": "5318472",
"type": "status_update"
},
{
"id": "5318678",
"type": "status_update"
},
{
"id": "5318537",
"type": "status_update"
}
]
}
},
"type": "status_report"
},
{
"attributes": {
"affected_resources": [
{
"status": "degraded",
"status_page_resource_id": "8498341"
},
{
"status": "degraded",
"status_page_resource_id": "8498343"
}
],
"message": "We are investigating reports of elevated latency on our OLAP database, which powers parts of the Hatchet UI and some API requests.\n\nThe core database is not affected. Workflow execution and workloads are continuing to run normally.",
"notify_subscribers": false,
"published_at": "2026-07-01T04:14:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5316221",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"message": "We have identified the source of the latency and have pushed initial configuration changes to mitigate. We will continue to monitor for degraded performance.",
"notify_subscribers": false,
"published_at": "2026-07-01T05:42:00.000Z",
"published_at_timezone": "Eastern Time (US \u0026 Canada)"
},
"id": "5316315",
"type": "status_update"
},
{
"attributes": {
"affected_resources": [
{
"status": "resolved",
"status_page_resource_id": "8498341"
},
{
"status": "resolved",
"status_page_resource_id": "8498343"
}
],
"aggregate_state": "resolved",
"ends_at": null,
"report_type": "manual",
"starts_at": "2026-07-01T04:14:00.000Z",
"title": "Dashboard Latency in US-WEST-2"
},
"id": "940181",
"relationships": {
"status_updates": {
"data": [
{
"id": "5316221",
"type": "status_update"
},
{
"id": "5316315",
"type": "status_update"
}
]
}
},
"type": "status_report"
}
]
}